目录
eBPF in Android
Android eBPF kprobe dma代码
定义一个MAP
定义一个 PROG
bpfprogs/Android.bp
测试程序
bpfprogs/memstats/Android.bp
bpfprogs/memstats/MemStats.h
bpfprogs/memstats/MemStats.cpp
bpfprogs/memstats/MemStatsMain.cpp
编译运行
结果分析
小结
eBPF in Android
官网:https://source.android.com/docs/core/architecture/kernel/bpf?hl=zh-cn#tracepoints
Andoid官方网站有比较详细的介绍,在Android中使用eBPF的官方示例,可以参考以上链接。
Android eBPF kprobe dma代码
在 源目录/system/bpfprogs,可以开发我们自己的 eBPF程序。
在system/bpfprogs 源码目录下,新建一个 memStats.c 示例程序,在次程序中有几个关键点,下面我逐一进行讲解。
memStats.c
#include <bpf_helpers.h>
#include <sys/types.h>struct pt_regs {unsigned long long regs[31];unsigned long long sp;unsigned long long pc;unsigned long long pstate;
};#define DMABUF_MAP_SIZE 4096struct dmabuf_info {unsigned long pid;uint64_t inode;char comm[16];uint64_t size;
};DEFINE_BPF_MAP_GRW(dmabuf_mem_map, HASH, uint64_t, struct dmabuf_info, DMABUF_MAP_SIZE,AID_SYSTEM);DEFINE_BPF_PROG("kprobe/dmabuf_setup", AID_ROOT, AID_SYSTEM, kp_dmabuf_setup)
(struct pt_regs* regs) {const int ALLOW = 1;struct dmabuf_info cur_val = { 0 };unsigned long long tempAddr;size_t size;unsigned long inode;pid_t pid = (bpf_get_current_pid_tgid() >> 32) & 0xffffffff;bpf_probe_read(&size, sizeof(size), (void*)regs->regs[0]); // size = [x0]; dmabuf->sizebpf_probe_read(&tempAddr, sizeof(tempAddr), ((void*)regs->regs[1] + 32)); // tempAddr = [x1 + 32];bpf_probe_read(&inode, sizeof(inode), ((void*)tempAddr + 64)); // inode = [[x1 + 32] + 64];cur_val.pid = pid;cur_val.size = size;cur_val.inode = inode;bpf_get_current_comm(cur_val.comm, sizeof(cur_val.comm));bpf_dmabuf_mem_map_update_elem(&(cur_val.inode), &cur_val, BPF_ANY);return ALLOW;
}LICENSE("GPL");
定义一个MAP
DEFINE_BPF_MAP_GRW(dmabuf_mem_map, HASH, uint64_t, struct dmabuf_info, DMABUF_MAP_SIZE,AID_SYSTEM);
其中:
- MAP的名字为:dmabuf_mem_map
- MAP使用HASH进行存储,常见的还有 ARRAY。
- MAP的key的类型为 uint64_t,value的类型为 struct dmabuf_info。后续会使用 这个函数 bpf_dmabuf_mem_map_update_elem(&key, &value, flags) 进行key和value的映射。
-
MAP的大小 DMABUF_MAP_SIZE 设置为 4096
-
权限是 AID_SYSTEM
定义一个 PROG
DEFINE_BPF_PROG("kprobe/dmabuf_setup", AID_ROOT, AID_SYSTEM, kp_dmabuf_setup)
(struct pt_regs* regs) {const int ALLOW = 1;struct dmabuf_info cur_val = { 0 };unsigned long long tempAddr;size_t size;unsigned long inode;pid_t pid = (bpf_get_current_pid_tgid() >> 32) & 0xffffffff;bpf_probe_read(&size, sizeof(size), (void*)regs->regs[0]); // size = [x0]; dmabuf->sizebpf_probe_read(&tempAddr, sizeof(tempAddr), ((void*)regs->regs[1] + 32)); // tempAddr = [x1 + 32];bpf_probe_read(&inode, sizeof(inode), ((void*)tempAddr + 64)); // inode = [[x1 + 32] + 64];cur_val.pid = pid;cur_val.size = size;cur_val.inode = inode;bpf_get_current_comm(cur_val.comm, sizeof(cur_val.comm));bpf_dmabuf_mem_map_update_elem(&(cur_val.inode), &cur_val, BPF_ANY);return ALLOW;
}
其中:
-
pid_t pid = (bpf_get_current_pid_tgid() >> 32) & 0xffffffff; 获取进程的 pid号
-
三个 bpf_probe_read(...) 获取对应的 dmabuf的 size 和 inode。这个比较那难理解,我这边详细解释一下。首先需要知道 关于aarch64调用传入参数规则:
怎样获取kprobe对应函数的参数? ARM64中参数1~参数8 分别保存到 X0~X7 寄存器中 即x0存储参数1.....
详细参考这篇文章:第16部分- Linux ARM汇编 ARM64调用标准 - 掘金 -
那么寄存器中的参数偏移是怎么样计算得到的呢?主要是通过 gdb 调试得到的。
怎么获取函数输入变量偏移?aarch64-linux-android-gdb vmlinux
(gdb) ptype/T struct dma_buf
type = struct dma_buf {size_t size;struct file *file;struct list_head attachments;const struct dma_buf_ops *ops;struct mutex lock;unsigned int vmapping_counter;struct iosys_map vmap_ptr;const char *exp_name;const char *name;spinlock_t name_lock;struct module *owner;struct list_head list_node;void *priv;struct dma_resv *resv;wait_queue_head_t poll;struct dma_buf_poll_cb_t cb_in;struct dma_buf_poll_cb_t cb_out;struct dma_buf_sysfs_entry *sysfs_entry;u64 android_kabi_reserved1;u64 android_kabi_reserved2;
}
(gdb) print (int)&((struct dma_buf *)0)->name
$2 = 120
(gdb) print (int)&((struct dma_buf *)0)->exp_name
$3 = 112
(gdb) print (int)&((struct dma_buf *)0)->file
$6 = 8
bpf_get_current_comm(cur_val.comm, sizeof(cur_val.comm)); 获得进程的名字。
bpf_dmabuf_mem_map_update_elem(&(cur_val.inode), &cur_val, BPF_ANY); 建立 key 和 value的 MAP映射。
bpfprogs/Android.bp
需要在 Android.bp中添加以下代码:
bpf {name: "memStats.o",srcs: ["memStats.c"],btf: true,cflags: ["-Wall","-Werror",],
}
怎么样写一个测试程序,验证上面的 eBPF程序是没有没有问题的呢?
测试程序
主要参考安卓开源代码: frameworks/native/services/gpuservice/ 中的实现部分。
bpfprogs/memstats/Android.bp
// Copyright 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package {default_applicable_licenses: ["system_bpfprogs_license"],
}
cc_binary {name: "memStats",srcs: ["MemStatsMain.cpp","MemStats.cpp",],header_libs: ["bpf_headers"],shared_libs: ["libbase","libbpf_bcc","libcutils","liblog","libutils",],export_header_lib_headers: ["bpf_headers"],export_shared_lib_headers: ["libbase"],cppflags: ["-Wall","-Werror","-Wformat","-Wthread-safety","-Wunused","-Wunreachable-code",],
}
bpfprogs/memstats/MemStats.h
/** Copyright 2020 The Android Open Source Project** Licensed under the Apache License, Version 2.0 (the "License");* you may not use this file except in compliance with the License.* You may obtain a copy of the License at** http://www.apache.org/licenses/LICENSE-2.0** Unless required by applicable law or agreed to in writing, software* distributed under the License is distributed on an "AS IS" BASIS,* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.* See the License for the specific language governing permissions and* limitations under the License.*/
#pragma once
#include <bpf/BpfMap.h>
#include <utils/String16.h>
#include <utils/Vector.h>
#include <functional>class MemStats {
public:MemStats() = default;~MemStats();// initialize eBPF program and mapvoid initialize();bool isInitialized() { return mInitialized.load(); }
private:std::atomic<bool> mInitialized = false;// tracepoint event category// tracepoint event categorystatic constexpr char * kMemStatsKprobeTraceGroup[] = {"kprobes"};// tracepointstatic constexpr char * kMemStatsDmabufKprobe[] ={"dmabuf_setup"};// pinned bpf c program path in bpf sysfsstatic constexpr char * kMemStatsDmabufProgPath[] ={"/sys/fs/bpf/prog_memStats_kprobe_dmabuf_setup"};// 30 seconds timeout for trying to attach bpf program to tracepointstatic constexpr int kWaitTimeout = 30;
};
bpfprogs/memstats/MemStats.cpp
/** Copyright 2020 The Android Open Source Project** Licensed under the Apache License, Version 2.0 (the "License");* you may not use this file except in compliance with the License.* You may obtain a copy of the License at** http://www.apache.org/licenses/LICENSE-2.0** Unless required by applicable law or agreed to in writing, software* distributed under the License is distributed on an "AS IS" BASIS,* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.* See the License for the specific language governing permissions and* limitations under the License.*/
#undef LOG_TAG
#define LOG_TAG "MemStats"
#define ATRACE_TAG ATRACE_TAG_GRAPHICSS3
#include "MemStats.h"
#include <android-base/stringprintf.h>
#include <libbpf.h>
#include <bpf/WaitForProgsLoaded.h>
#include <log/log.h>
#include <unistd.h>
#include <utils/Timers.h>
#include <utils/Trace.h>
#include <unordered_map>
#include <vector>#include <android-base/file.h>MemStats::~MemStats() {for (int i = 0; i < sizeof(kMemStatsDmabufKprobe)/sizeof(char *); i++){bpf_detach_tracepoint(kMemStatsKprobeTraceGroup[0], kMemStatsDmabufKprobe[i]);}
}void MemStats::initialize() {int count = 0;int fd = 0;// Make sure bpf programs are loadedandroid::bpf::waitForProgsLoaded();errno = 0;android::base::WriteStringToFile("p:dmabuf_setup dma_buf_stats_setup","/sys/kernel/debug/tracing/kprobe_events");for (int i = 0; i < sizeof(kMemStatsDmabufProgPath)/sizeof(char *); i++){fd = android::bpf::retrieveProgram(kMemStatsDmabufProgPath[i]);if (fd < 0) {ALOGE("Failed to retrieve pinned program from %s [%d(%s)]", kMemStatsDmabufProgPath[i], errno,strerror(errno));}// Attach the program to the tracepoint, and the tracepoint is automatically enabled here.errno = 0;count = 0;while (bpf_attach_tracepoint(fd, kMemStatsKprobeTraceGroup[0], kMemStatsDmabufKprobe[i]) < 0) {if (++count > kWaitTimeout) {ALOGE("Failed to attach bpf program to %s/%s tracepoint [%d(%s)]", kMemStatsKprobeTraceGroup[0],kMemStatsDmabufKprobe[i], errno, strerror(errno));}// Retry until loaded or timeout.sleep(1);}}mInitialized.store(true);
}
其中:
android::base::WriteStringToFile("p:dmabuf_setup dma_buf_stats_setup","/sys/kernel/debug/tracing/kprobe_events");
在 bpf_attach_tracepoint(...)之前,将 "p:dmabuf_setup dma_buf_stats_setup" 写入到 "/sys/kernel/debug/tracing/kprobe_events" 这个事件中去。如果不使用这行代码,也可以在 Linux终端使用,adb shell; echo 'p:dmabuf_setup dma_buf_stats_setup' > /sys/kernel/debug/tracing/kprobe_events
bpfprogs/memstats/MemStatsMain.cpp
#include "MemStats.h"
#include <unistd.h>
int main()
{class MemStats *memHandle = new MemStats;memHandle->initialize();while(1){sleep(10);}return 0;
}
编译运行
- make memStats.o
-
adb push system/etc/bpf/memStats.o /system/etc/bpf/
-
adb reboot; #重启之后就可以在 /sys/fs/bpf/ 目录下,看到生成了 这个文件 map_memStats_dmabuf_mem_map
-
make memStats
-
adb push memStats /data/local/tmp/
-
adb shell ; /data/local/tmp/memStats
-
在另外一个Linux终端:cat /sys/fs/bpf/map_memStats_dmabuf_mem_map
可以看到输出以下信息:
/ # cat /sys/fs/bpf/map_memStats_dmabuf_mem_map
# WARNING!! The output is for debug purpose only
# WARNING!! The output format will change
697: {1512,697,['b','i','n','d','e','r',':','1','5','1','2','_','3',],188416,}
1344: {1512,1344,['P','r','e','v','i','e','w','_','4',],22020096,}
752: {1512,752,['P','r','e','v','i','e','w','_','0',],4096,}
589: {1525,589,['b','i','n','d','e','r',':','1','5','2','5','_','1',],10522624,}
1175: {1512,1175,['P','r','e','v','i','e','w','_','2',],462848,}
1498: {1512,1498,['P','r','e','v','i','e','w','_','4',],4096,}
1539: {1525,1539,['b','i','n','d','e','r',':','1','5','2','5','_','1',],73728,}
712: {1512,712,['P','r','e','v','i','e','w','_','2',],4096,}
1469: {1525,1469,['b','i','n','d','e','r',':','1','5','2','5','_','1',],2506752,}
1520: {2553,1520,['.','v','o','r','b','i','s','.','d','e','c','o','d','e','r',],32768,}
745: {1512,745,['P','r','e','v','i','e','w','_','1',],4096,}
1137: {1512,1137,['P','r','e','v','i','e','w','_','7',],4096,}
1186: {1512,1186,['b','i','n','d','e','r',':','1','5','1','2','_','3',],6451200,}
724: {1512,724,['P','r','e','v','i','e','w','_','5',],4096,}
791: {1512,791,['P','r','e','v','i','e','w','_','7',],4096,}
1159: {1512,1159,['P','r','e','v','i','e','w','_','4',],462848,}
810: {1512,810,['P','r','e','v','i','e','w','_','7',],4096,}
1285: {1512,1285,['P','r','e','v','i','e','w','_','0',],4096,}
761: {1512,761,['P','r','e','v','i','e','w','_','8',],4096,}
735: {1512,735,['P','r','e','v','i','e','w','_','1',],4096,}
630: {1512,630,['b','i','n','d','e','r',':','1','5','1','2','_','3',],4096,}
1012: {1512,1012,['b','i','n','d','e','r',':','1','5','1','2','_','3',],45056,}
结果分析
解释以下输出结果,以这个为例子:697: {1512,697,['b','i','n','d','e','r',':','1','5','1','2','_','3',],188416,}
其中 697 是 inode的值,{1512,697,['b','i','n','d','e','r',':','1','5','1','2','_','3',],188416,} 是struct dmabuf_info 结构体的输出。具体输出每一项为对应结构体中的值,如下所示:
struct dmabuf_info {unsigned long pid; //1512uint64_t inode; //697char comm[16]; //['b','i','n','d','e','r',':','1','5','1','2','_','3',]uint64_t size; //188416
};
小结
以上是一个具体的 eBPF实例程序。通过 kprobe 监控内核中的 dma_buf_stats_setup 接口。
kernel_platform/common/drivers/dma-buf/dma-buf-sysfs-stats.c
int dma_buf_stats_setup(struct dma_buf *dmabuf, struct file *file)
172 {
173 struct dma_buf_sysfs_entry *sysfs_entry;
174 int ret;
175
176 if (!dmabuf->exp_name) {
177 pr_err("exporter name must not be empty if stats needed\n");
178 return -EINVAL;
179 }
180
181 sysfs_entry = kzalloc(sizeof(struct dma_buf_sysfs_entry), GFP_KERNEL);
182 if (!sysfs_entry)
183 return -ENOMEM;
184
185 sysfs_entry->kobj.kset = dma_buf_per_buffer_stats_kset;
186 sysfs_entry->dmabuf = dmabuf;
187
188 dmabuf->sysfs_entry = sysfs_entry;
189
190 /* create the directory for buffer stats */
191 ret = kobject_init_and_add(&sysfs_entry->kobj, &dma_buf_ktype, NULL,
192 "%lu", file_inode(file)->i_ino);
193 if (ret)
194 goto err_sysfs_dmabuf;
195
196 return 0;
197
198 err_sysfs_dmabuf:
199 kobject_put(&sysfs_entry->kobj);
200 dmabuf->sysfs_entry = NULL;
201 return ret;
202 }
后续将在 Android上输出更多的 eBPF demo程序。