2019-05-28 20:10:09 +03:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2016-02-18 06:58:58 +03:00
|
|
|
/* Copyright (c) 2016 Facebook
|
|
|
|
|
*/
|
|
|
|
|
#include <linux/bpf.h>
|
|
|
|
|
#include <linux/jhash.h>
|
|
|
|
|
#include <linux/filter.h>
|
|
|
|
|
#include <linux/stacktrace.h>
|
|
|
|
|
#include <linux/perf_event.h>
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
#include <linux/elf.h>
|
|
|
|
|
#include <linux/pagemap.h>
|
2018-05-07 20:50:48 +03:00
|
|
|
#include <linux/irq_work.h>
|
2016-03-08 08:57:17 +03:00
|
|
|
#include "percpu_freelist.h"
|
2016-02-18 06:58:58 +03:00
|
|
|
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
#define STACK_CREATE_FLAG_MASK \
|
|
|
|
|
(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY | \
|
|
|
|
|
BPF_F_STACK_BUILD_ID)
|
2017-10-18 23:00:22 +03:00
|
|
|
|
2016-02-18 06:58:58 +03:00
|
|
|
struct stack_map_bucket {
|
2016-03-08 08:57:17 +03:00
|
|
|
struct pcpu_freelist_node fnode;
|
2016-02-18 06:58:58 +03:00
|
|
|
u32 hash;
|
|
|
|
|
u32 nr;
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
u64 data[];
|
2016-02-18 06:58:58 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct bpf_stack_map {
|
|
|
|
|
struct bpf_map map;
|
2016-03-08 08:57:17 +03:00
|
|
|
void *elems;
|
|
|
|
|
struct pcpu_freelist freelist;
|
2016-02-18 06:58:58 +03:00
|
|
|
u32 n_buckets;
|
2016-03-08 08:57:17 +03:00
|
|
|
struct stack_map_bucket *buckets[];
|
2016-02-18 06:58:58 +03:00
|
|
|
};
|
|
|
|
|
|
2018-05-07 20:50:48 +03:00
|
|
|
/* irq_work to run up_read() for build_id lookup in nmi context */
|
|
|
|
|
struct stack_map_irq_work {
|
|
|
|
|
struct irq_work irq_work;
|
|
|
|
|
struct rw_semaphore *sem;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static void do_up_read(struct irq_work *entry)
|
|
|
|
|
{
|
|
|
|
|
struct stack_map_irq_work *work;
|
|
|
|
|
|
2020-02-24 17:01:53 +03:00
|
|
|
if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT)))
|
|
|
|
|
return;
|
|
|
|
|
|
2018-05-07 20:50:48 +03:00
|
|
|
work = container_of(entry, struct stack_map_irq_work, irq_work);
|
2019-02-10 23:52:35 +03:00
|
|
|
up_read_non_owner(work->sem);
|
2018-05-07 20:50:48 +03:00
|
|
|
work->sem = NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static DEFINE_PER_CPU(struct stack_map_irq_work, up_read_work);
|
|
|
|
|
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
static inline bool stack_map_use_build_id(struct bpf_map *map)
|
|
|
|
|
{
|
|
|
|
|
return (map->map_flags & BPF_F_STACK_BUILD_ID);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int stack_map_data_size(struct bpf_map *map)
|
|
|
|
|
{
|
|
|
|
|
return stack_map_use_build_id(map) ?
|
|
|
|
|
sizeof(struct bpf_stack_build_id) : sizeof(u64);
|
|
|
|
|
}
|
|
|
|
|
|
2016-03-08 08:57:17 +03:00
|
|
|
static int prealloc_elems_and_freelist(struct bpf_stack_map *smap)
|
|
|
|
|
{
|
|
|
|
|
u32 elem_size = sizeof(struct stack_map_bucket) + smap->map.value_size;
|
|
|
|
|
int err;
|
|
|
|
|
|
2017-08-18 21:28:00 +03:00
|
|
|
smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,
|
|
|
|
|
smap->map.numa_node);
|
2016-03-08 08:57:17 +03:00
|
|
|
if (!smap->elems)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
err = pcpu_freelist_init(&smap->freelist);
|
|
|
|
|
if (err)
|
|
|
|
|
goto free_elems;
|
|
|
|
|
|
|
|
|
|
pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size,
|
|
|
|
|
smap->map.max_entries);
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
free_elems:
|
bpf: don't trigger OOM killer under pressure with map alloc
This patch adds two helpers, bpf_map_area_alloc() and bpf_map_area_free(),
that are to be used for map allocations. Using kmalloc() for very large
allocations can cause excessive work within the page allocator, so i) fall
back earlier to vmalloc() when the attempt is considered costly anyway,
and even more importantly ii) don't trigger OOM killer with any of the
allocators.
Since this is based on a user space request, for example, when creating
maps with element pre-allocation, we really want such requests to fail
instead of killing other user space processes.
Also, don't spam the kernel log with warnings should any of the allocations
fail under pressure. Given that, we can make backend selection in
bpf_map_area_alloc() generic, and convert all maps over to use this API
for spots with potentially large allocation requests.
Note, replacing the one kmalloc_array() is fine as overflow checks happen
earlier in htab_map_alloc(), since it must also protect the multiplication
for vmalloc() should kmalloc_array() fail.
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-01-18 17:14:17 +03:00
|
|
|
bpf_map_area_free(smap->elems);
|
2016-03-08 08:57:17 +03:00
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
2016-02-18 06:58:58 +03:00
|
|
|
/* Called from syscall */
|
|
|
|
|
static struct bpf_map *stack_map_alloc(union bpf_attr *attr)
|
|
|
|
|
{
|
|
|
|
|
u32 value_size = attr->value_size;
|
|
|
|
|
struct bpf_stack_map *smap;
|
2019-05-30 04:03:58 +03:00
|
|
|
struct bpf_map_memory mem;
|
2016-02-18 06:58:58 +03:00
|
|
|
u64 cost, n_buckets;
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
if (!capable(CAP_SYS_ADMIN))
|
|
|
|
|
return ERR_PTR(-EPERM);
|
|
|
|
|
|
2017-10-18 23:00:22 +03:00
|
|
|
if (attr->map_flags & ~STACK_CREATE_FLAG_MASK)
|
2016-03-08 08:57:16 +03:00
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
2016-02-18 06:58:58 +03:00
|
|
|
/* check sanity of attributes */
|
|
|
|
|
if (attr->max_entries == 0 || attr->key_size != 4 ||
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
value_size < 8 || value_size % 8)
|
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
|
|
BUILD_BUG_ON(sizeof(struct bpf_stack_build_id) % sizeof(u64));
|
|
|
|
|
if (attr->map_flags & BPF_F_STACK_BUILD_ID) {
|
|
|
|
|
if (value_size % sizeof(struct bpf_stack_build_id) ||
|
|
|
|
|
value_size / sizeof(struct bpf_stack_build_id)
|
|
|
|
|
> sysctl_perf_event_max_stack)
|
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
} else if (value_size / 8 > sysctl_perf_event_max_stack)
|
2016-02-18 06:58:58 +03:00
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
|
|
/* hash table size must be power of 2 */
|
|
|
|
|
n_buckets = roundup_pow_of_two(attr->max_entries);
|
|
|
|
|
|
|
|
|
|
cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);
|
2019-05-30 04:03:58 +03:00
|
|
|
cost += n_buckets * (value_size + sizeof(struct stack_map_bucket));
|
2019-05-30 04:03:59 +03:00
|
|
|
err = bpf_map_charge_init(&mem, cost);
|
2019-05-30 04:03:58 +03:00
|
|
|
if (err)
|
|
|
|
|
return ERR_PTR(err);
|
|
|
|
|
|
2017-08-18 21:28:00 +03:00
|
|
|
smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));
|
2019-05-30 04:03:58 +03:00
|
|
|
if (!smap) {
|
|
|
|
|
bpf_map_charge_finish(&mem);
|
bpf: don't trigger OOM killer under pressure with map alloc
This patch adds two helpers, bpf_map_area_alloc() and bpf_map_area_free(),
that are to be used for map allocations. Using kmalloc() for very large
allocations can cause excessive work within the page allocator, so i) fall
back earlier to vmalloc() when the attempt is considered costly anyway,
and even more importantly ii) don't trigger OOM killer with any of the
allocators.
Since this is based on a user space request, for example, when creating
maps with element pre-allocation, we really want such requests to fail
instead of killing other user space processes.
Also, don't spam the kernel log with warnings should any of the allocations
fail under pressure. Given that, we can make backend selection in
bpf_map_area_alloc() generic, and convert all maps over to use this API
for spots with potentially large allocation requests.
Note, replacing the one kmalloc_array() is fine as overflow checks happen
earlier in htab_map_alloc(), since it must also protect the multiplication
for vmalloc() should kmalloc_array() fail.
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-01-18 17:14:17 +03:00
|
|
|
return ERR_PTR(-ENOMEM);
|
2019-05-30 04:03:58 +03:00
|
|
|
}
|
2016-02-18 06:58:58 +03:00
|
|
|
|
2018-01-12 07:29:06 +03:00
|
|
|
bpf_map_init_from_attr(&smap->map, attr);
|
2016-02-18 06:58:58 +03:00
|
|
|
smap->map.value_size = value_size;
|
|
|
|
|
smap->n_buckets = n_buckets;
|
2016-03-08 08:57:17 +03:00
|
|
|
|
2016-04-28 19:16:33 +03:00
|
|
|
err = get_callchain_buffers(sysctl_perf_event_max_stack);
|
2016-02-18 06:58:58 +03:00
|
|
|
if (err)
|
2019-05-30 04:03:58 +03:00
|
|
|
goto free_charge;
|
2016-02-18 06:58:58 +03:00
|
|
|
|
2016-03-08 08:57:17 +03:00
|
|
|
err = prealloc_elems_and_freelist(smap);
|
|
|
|
|
if (err)
|
|
|
|
|
goto put_buffers;
|
|
|
|
|
|
2019-05-30 04:03:58 +03:00
|
|
|
bpf_map_charge_move(&smap->map.memory, &mem);
|
|
|
|
|
|
2016-02-18 06:58:58 +03:00
|
|
|
return &smap->map;
|
|
|
|
|
|
2016-03-08 08:57:17 +03:00
|
|
|
put_buffers:
|
|
|
|
|
put_callchain_buffers();
|
2019-05-30 04:03:58 +03:00
|
|
|
free_charge:
|
|
|
|
|
bpf_map_charge_finish(&mem);
|
bpf: don't trigger OOM killer under pressure with map alloc
This patch adds two helpers, bpf_map_area_alloc() and bpf_map_area_free(),
that are to be used for map allocations. Using kmalloc() for very large
allocations can cause excessive work within the page allocator, so i) fall
back earlier to vmalloc() when the attempt is considered costly anyway,
and even more importantly ii) don't trigger OOM killer with any of the
allocators.
Since this is based on a user space request, for example, when creating
maps with element pre-allocation, we really want such requests to fail
instead of killing other user space processes.
Also, don't spam the kernel log with warnings should any of the allocations
fail under pressure. Given that, we can make backend selection in
bpf_map_area_alloc() generic, and convert all maps over to use this API
for spots with potentially large allocation requests.
Note, replacing the one kmalloc_array() is fine as overflow checks happen
earlier in htab_map_alloc(), since it must also protect the multiplication
for vmalloc() should kmalloc_array() fail.
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-01-18 17:14:17 +03:00
|
|
|
bpf_map_area_free(smap);
|
2016-02-18 06:58:58 +03:00
|
|
|
return ERR_PTR(err);
|
|
|
|
|
}
|
|
|
|
|
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
#define BPF_BUILD_ID 3
|
|
|
|
|
/*
|
|
|
|
|
* Parse build id from the note segment. This logic can be shared between
|
|
|
|
|
* 32-bit and 64-bit system, because Elf32_Nhdr and Elf64_Nhdr are
|
|
|
|
|
* identical.
|
|
|
|
|
*/
|
|
|
|
|
static inline int stack_map_parse_build_id(void *page_addr,
|
|
|
|
|
unsigned char *build_id,
|
|
|
|
|
void *note_start,
|
|
|
|
|
Elf32_Word note_size)
|
|
|
|
|
{
|
|
|
|
|
Elf32_Word note_offs = 0, new_offs;
|
|
|
|
|
|
|
|
|
|
/* check for overflow */
|
|
|
|
|
if (note_start < page_addr || note_start + note_size < note_start)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
/* only supports note that fits in the first page */
|
|
|
|
|
if (note_start + note_size > page_addr + PAGE_SIZE)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
while (note_offs + sizeof(Elf32_Nhdr) < note_size) {
|
|
|
|
|
Elf32_Nhdr *nhdr = (Elf32_Nhdr *)(note_start + note_offs);
|
|
|
|
|
|
|
|
|
|
if (nhdr->n_type == BPF_BUILD_ID &&
|
|
|
|
|
nhdr->n_namesz == sizeof("GNU") &&
|
2019-01-17 01:03:15 +03:00
|
|
|
nhdr->n_descsz > 0 &&
|
|
|
|
|
nhdr->n_descsz <= BPF_BUILD_ID_SIZE) {
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
memcpy(build_id,
|
|
|
|
|
note_start + note_offs +
|
|
|
|
|
ALIGN(sizeof("GNU"), 4) + sizeof(Elf32_Nhdr),
|
2019-01-17 01:03:15 +03:00
|
|
|
nhdr->n_descsz);
|
|
|
|
|
memset(build_id + nhdr->n_descsz, 0,
|
|
|
|
|
BPF_BUILD_ID_SIZE - nhdr->n_descsz);
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
new_offs = note_offs + sizeof(Elf32_Nhdr) +
|
|
|
|
|
ALIGN(nhdr->n_namesz, 4) + ALIGN(nhdr->n_descsz, 4);
|
|
|
|
|
if (new_offs <= note_offs) /* overflow */
|
|
|
|
|
break;
|
|
|
|
|
note_offs = new_offs;
|
|
|
|
|
}
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Parse build ID from 32-bit ELF */
|
|
|
|
|
static int stack_map_get_build_id_32(void *page_addr,
|
|
|
|
|
unsigned char *build_id)
|
|
|
|
|
{
|
|
|
|
|
Elf32_Ehdr *ehdr = (Elf32_Ehdr *)page_addr;
|
|
|
|
|
Elf32_Phdr *phdr;
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
/* only supports phdr that fits in one page */
|
|
|
|
|
if (ehdr->e_phnum >
|
|
|
|
|
(PAGE_SIZE - sizeof(Elf32_Ehdr)) / sizeof(Elf32_Phdr))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
phdr = (Elf32_Phdr *)(page_addr + sizeof(Elf32_Ehdr));
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < ehdr->e_phnum; ++i)
|
|
|
|
|
if (phdr[i].p_type == PT_NOTE)
|
|
|
|
|
return stack_map_parse_build_id(page_addr, build_id,
|
|
|
|
|
page_addr + phdr[i].p_offset,
|
|
|
|
|
phdr[i].p_filesz);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Parse build ID from 64-bit ELF */
|
|
|
|
|
static int stack_map_get_build_id_64(void *page_addr,
|
|
|
|
|
unsigned char *build_id)
|
|
|
|
|
{
|
|
|
|
|
Elf64_Ehdr *ehdr = (Elf64_Ehdr *)page_addr;
|
|
|
|
|
Elf64_Phdr *phdr;
|
|
|
|
|
int i;
|
|
|
|
|
|
|
|
|
|
/* only supports phdr that fits in one page */
|
|
|
|
|
if (ehdr->e_phnum >
|
|
|
|
|
(PAGE_SIZE - sizeof(Elf64_Ehdr)) / sizeof(Elf64_Phdr))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
phdr = (Elf64_Phdr *)(page_addr + sizeof(Elf64_Ehdr));
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < ehdr->e_phnum; ++i)
|
|
|
|
|
if (phdr[i].p_type == PT_NOTE)
|
|
|
|
|
return stack_map_parse_build_id(page_addr, build_id,
|
|
|
|
|
page_addr + phdr[i].p_offset,
|
|
|
|
|
phdr[i].p_filesz);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Parse build ID of ELF file mapped to vma */
|
|
|
|
|
static int stack_map_get_build_id(struct vm_area_struct *vma,
|
|
|
|
|
unsigned char *build_id)
|
|
|
|
|
{
|
|
|
|
|
Elf32_Ehdr *ehdr;
|
|
|
|
|
struct page *page;
|
|
|
|
|
void *page_addr;
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
/* only works for page backed storage */
|
|
|
|
|
if (!vma->vm_file)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
page = find_get_page(vma->vm_file->f_mapping, 0);
|
|
|
|
|
if (!page)
|
|
|
|
|
return -EFAULT; /* page not mapped */
|
|
|
|
|
|
|
|
|
|
ret = -EINVAL;
|
2019-01-09 01:20:44 +03:00
|
|
|
page_addr = kmap_atomic(page);
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
ehdr = (Elf32_Ehdr *)page_addr;
|
|
|
|
|
|
|
|
|
|
/* compare magic x7f "ELF" */
|
|
|
|
|
if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0)
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
/* only support executable file and shared object file */
|
|
|
|
|
if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN)
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
if (ehdr->e_ident[EI_CLASS] == ELFCLASS32)
|
|
|
|
|
ret = stack_map_get_build_id_32(page_addr, build_id);
|
|
|
|
|
else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64)
|
|
|
|
|
ret = stack_map_get_build_id_64(page_addr, build_id);
|
|
|
|
|
out:
|
2019-01-09 01:20:44 +03:00
|
|
|
kunmap_atomic(page_addr);
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
put_page(page);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-29 08:28:07 +03:00
|
|
|
static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
u64 *ips, u32 trace_nr, bool user)
|
|
|
|
|
{
|
|
|
|
|
int i;
|
|
|
|
|
struct vm_area_struct *vma;
|
2018-05-07 20:50:48 +03:00
|
|
|
bool irq_work_busy = false;
|
2018-05-26 00:33:20 +03:00
|
|
|
struct stack_map_irq_work *work = NULL;
|
2018-05-07 20:50:48 +03:00
|
|
|
|
bpf/stackmap: Fix deadlock with rq_lock in bpf_get_stack()
bpf stackmap with build-id lookup (BPF_F_STACK_BUILD_ID) can trigger A-A
deadlock on rq_lock():
rcu: INFO: rcu_sched detected stalls on CPUs/tasks:
[...]
Call Trace:
try_to_wake_up+0x1ad/0x590
wake_up_q+0x54/0x80
rwsem_wake+0x8a/0xb0
bpf_get_stack+0x13c/0x150
bpf_prog_fbdaf42eded9fe46_on_event+0x5e3/0x1000
bpf_overflow_handler+0x60/0x100
__perf_event_overflow+0x4f/0xf0
perf_swevent_overflow+0x99/0xc0
___perf_sw_event+0xe7/0x120
__schedule+0x47d/0x620
schedule+0x29/0x90
futex_wait_queue_me+0xb9/0x110
futex_wait+0x139/0x230
do_futex+0x2ac/0xa50
__x64_sys_futex+0x13c/0x180
do_syscall_64+0x42/0x100
entry_SYSCALL_64_after_hwframe+0x44/0xa9
This can be reproduced by:
1. Start a multi-thread program that does parallel mmap() and malloc();
2. taskset the program to 2 CPUs;
3. Attach bpf program to trace_sched_switch and gather stackmap with
build-id, e.g. with trace.py from bcc tools:
trace.py -U -p <pid> -s <some-bin,some-lib> t:sched:sched_switch
A sample reproducer is attached at the end.
This could also trigger deadlock with other locks that are nested with
rq_lock.
Fix this by checking whether irqs are disabled. Since rq_lock and all
other nested locks are irq safe, it is safe to do up_read() when irqs are
not disable. If the irqs are disabled, postpone up_read() in irq_work.
Fixes: 615755a77b24 ("bpf: extend stackmap to save binary_build_id+offset instead of address")
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20191014171223.357174-1-songliubraving@fb.com
Reproducer:
============================ 8< ============================
char *filename;
void *worker(void *p)
{
void *ptr;
int fd;
char *pptr;
fd = open(filename, O_RDONLY);
if (fd < 0)
return NULL;
while (1) {
struct timespec ts = {0, 1000 + rand() % 2000};
ptr = mmap(NULL, 4096 * 64, PROT_READ, MAP_PRIVATE, fd, 0);
usleep(1);
if (ptr == MAP_FAILED) {
printf("failed to mmap\n");
break;
}
munmap(ptr, 4096 * 64);
usleep(1);
pptr = malloc(1);
usleep(1);
pptr[0] = 1;
usleep(1);
free(pptr);
usleep(1);
nanosleep(&ts, NULL);
}
close(fd);
return NULL;
}
int main(int argc, char *argv[])
{
void *ptr;
int i;
pthread_t threads[THREAD_COUNT];
if (argc < 2)
return 0;
filename = argv[1];
for (i = 0; i < THREAD_COUNT; i++) {
if (pthread_create(threads + i, NULL, worker, NULL)) {
fprintf(stderr, "Error creating thread\n");
return 0;
}
}
for (i = 0; i < THREAD_COUNT; i++)
pthread_join(threads[i], NULL);
return 0;
}
============================ 8< ============================
2019-10-14 20:12:23 +03:00
|
|
|
if (irqs_disabled()) {
|
2020-02-24 17:01:53 +03:00
|
|
|
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
|
|
|
|
|
work = this_cpu_ptr(&up_read_work);
|
|
|
|
|
if (atomic_read(&work->irq_work.flags) & IRQ_WORK_BUSY) {
|
|
|
|
|
/* cannot queue more up_read, fallback */
|
|
|
|
|
irq_work_busy = true;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
/*
|
|
|
|
|
* PREEMPT_RT does not allow to trylock mmap sem in
|
|
|
|
|
* interrupt disabled context. Force the fallback code.
|
|
|
|
|
*/
|
2018-05-07 20:50:48 +03:00
|
|
|
irq_work_busy = true;
|
2020-02-24 17:01:53 +03:00
|
|
|
}
|
2018-05-07 20:50:48 +03:00
|
|
|
}
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
|
|
|
|
|
/*
|
bpf/stackmap: Fix deadlock with rq_lock in bpf_get_stack()
bpf stackmap with build-id lookup (BPF_F_STACK_BUILD_ID) can trigger A-A
deadlock on rq_lock():
rcu: INFO: rcu_sched detected stalls on CPUs/tasks:
[...]
Call Trace:
try_to_wake_up+0x1ad/0x590
wake_up_q+0x54/0x80
rwsem_wake+0x8a/0xb0
bpf_get_stack+0x13c/0x150
bpf_prog_fbdaf42eded9fe46_on_event+0x5e3/0x1000
bpf_overflow_handler+0x60/0x100
__perf_event_overflow+0x4f/0xf0
perf_swevent_overflow+0x99/0xc0
___perf_sw_event+0xe7/0x120
__schedule+0x47d/0x620
schedule+0x29/0x90
futex_wait_queue_me+0xb9/0x110
futex_wait+0x139/0x230
do_futex+0x2ac/0xa50
__x64_sys_futex+0x13c/0x180
do_syscall_64+0x42/0x100
entry_SYSCALL_64_after_hwframe+0x44/0xa9
This can be reproduced by:
1. Start a multi-thread program that does parallel mmap() and malloc();
2. taskset the program to 2 CPUs;
3. Attach bpf program to trace_sched_switch and gather stackmap with
build-id, e.g. with trace.py from bcc tools:
trace.py -U -p <pid> -s <some-bin,some-lib> t:sched:sched_switch
A sample reproducer is attached at the end.
This could also trigger deadlock with other locks that are nested with
rq_lock.
Fix this by checking whether irqs are disabled. Since rq_lock and all
other nested locks are irq safe, it is safe to do up_read() when irqs are
not disable. If the irqs are disabled, postpone up_read() in irq_work.
Fixes: 615755a77b24 ("bpf: extend stackmap to save binary_build_id+offset instead of address")
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20191014171223.357174-1-songliubraving@fb.com
Reproducer:
============================ 8< ============================
char *filename;
void *worker(void *p)
{
void *ptr;
int fd;
char *pptr;
fd = open(filename, O_RDONLY);
if (fd < 0)
return NULL;
while (1) {
struct timespec ts = {0, 1000 + rand() % 2000};
ptr = mmap(NULL, 4096 * 64, PROT_READ, MAP_PRIVATE, fd, 0);
usleep(1);
if (ptr == MAP_FAILED) {
printf("failed to mmap\n");
break;
}
munmap(ptr, 4096 * 64);
usleep(1);
pptr = malloc(1);
usleep(1);
pptr[0] = 1;
usleep(1);
free(pptr);
usleep(1);
nanosleep(&ts, NULL);
}
close(fd);
return NULL;
}
int main(int argc, char *argv[])
{
void *ptr;
int i;
pthread_t threads[THREAD_COUNT];
if (argc < 2)
return 0;
filename = argv[1];
for (i = 0; i < THREAD_COUNT; i++) {
if (pthread_create(threads + i, NULL, worker, NULL)) {
fprintf(stderr, "Error creating thread\n");
return 0;
}
}
for (i = 0; i < THREAD_COUNT; i++)
pthread_join(threads[i], NULL);
return 0;
}
============================ 8< ============================
2019-10-14 20:12:23 +03:00
|
|
|
* We cannot do up_read() when the irq is disabled, because of
|
|
|
|
|
* risk to deadlock with rq_lock. To do build_id lookup when the
|
|
|
|
|
* irqs are disabled, we need to run up_read() in irq_work. We use
|
2018-05-07 20:50:48 +03:00
|
|
|
* a percpu variable to do the irq_work. If the irq_work is
|
|
|
|
|
* already used by another lookup, we fall back to report ips.
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
*
|
|
|
|
|
* Same fallback is used for kernel stack (!user) on a stackmap
|
|
|
|
|
* with build_id.
|
|
|
|
|
*/
|
2018-05-07 20:50:48 +03:00
|
|
|
if (!user || !current || !current->mm || irq_work_busy ||
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
down_read_trylock(¤t->mm->mmap_sem) == 0) {
|
|
|
|
|
/* cannot access current->mm, fall back to ips */
|
|
|
|
|
for (i = 0; i < trace_nr; i++) {
|
|
|
|
|
id_offs[i].status = BPF_STACK_BUILD_ID_IP;
|
|
|
|
|
id_offs[i].ip = ips[i];
|
2019-01-17 01:03:16 +03:00
|
|
|
memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE);
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
}
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < trace_nr; i++) {
|
|
|
|
|
vma = find_vma(current->mm, ips[i]);
|
|
|
|
|
if (!vma || stack_map_get_build_id(vma, id_offs[i].build_id)) {
|
|
|
|
|
/* per entry fall back to ips */
|
|
|
|
|
id_offs[i].status = BPF_STACK_BUILD_ID_IP;
|
|
|
|
|
id_offs[i].ip = ips[i];
|
2019-01-17 01:03:16 +03:00
|
|
|
memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE);
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ips[i]
|
|
|
|
|
- vma->vm_start;
|
|
|
|
|
id_offs[i].status = BPF_STACK_BUILD_ID_VALID;
|
|
|
|
|
}
|
2018-05-07 20:50:48 +03:00
|
|
|
|
2018-05-26 00:33:20 +03:00
|
|
|
if (!work) {
|
2018-05-07 20:50:48 +03:00
|
|
|
up_read(¤t->mm->mmap_sem);
|
|
|
|
|
} else {
|
|
|
|
|
work->sem = ¤t->mm->mmap_sem;
|
|
|
|
|
irq_work_queue(&work->irq_work);
|
2019-02-10 23:52:35 +03:00
|
|
|
/*
|
|
|
|
|
* The irq_work will release the mmap_sem with
|
|
|
|
|
* up_read_non_owner(). The rwsem_release() is called
|
|
|
|
|
* here to release the lock from lockdep's perspective.
|
|
|
|
|
*/
|
2019-09-19 19:09:40 +03:00
|
|
|
rwsem_release(¤t->mm->mmap_sem.dep_map, _RET_IP_);
|
2018-05-07 20:50:48 +03:00
|
|
|
}
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
}
|
|
|
|
|
|
bpf: add BPF_CALL_x macros for declaring helpers
This work adds BPF_CALL_<n>() macros and converts all the eBPF helper functions
to use them, in a similar fashion like we do with SYSCALL_DEFINE<n>() macros
that are used today. Motivation for this is to hide all the register handling
and all necessary casts from the user, so that it is done automatically in the
background when adding a BPF_CALL_<n>() call.
This makes current helpers easier to review, eases to write future helpers,
avoids getting the casting mess wrong, and allows for extending all helpers at
once (f.e. build time checks, etc). It also helps detecting more easily in
code reviews that unused registers are not instrumented in the code by accident,
breaking compatibility with existing programs.
BPF_CALL_<n>() internals are quite similar to SYSCALL_DEFINE<n>() ones with some
fundamental differences, for example, for generating the actual helper function
that carries all u64 regs, we need to fill unused regs, so that we always end up
with 5 u64 regs as an argument.
I reviewed several 0-5 generated BPF_CALL_<n>() variants of the .i results and
they look all as expected. No sparse issue spotted. We let this also sit for a
few days with Fengguang's kbuild test robot, and there were no issues seen. On
s390, it barked on the "uses dynamic stack allocation" notice, which is an old
one from bpf_perf_event_output{,_tp}() reappearing here due to the conversion
to the call wrapper, just telling that the perf raw record/frag sits on stack
(gcc with s390's -mwarn-dynamicstack), but that's all. Did various runtime tests
and they were fine as well. All eBPF helpers are now converted to use these
macros, getting rid of a good chunk of all the raw castings.
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-09-09 03:45:31 +03:00
|
|
|
BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
|
|
|
|
|
u64, flags)
|
2016-02-18 06:58:58 +03:00
|
|
|
{
|
|
|
|
|
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
|
|
|
|
|
struct perf_callchain_entry *trace;
|
|
|
|
|
struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
u32 max_depth = map->value_size / stack_map_data_size(map);
|
2016-04-21 18:28:50 +03:00
|
|
|
/* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */
|
|
|
|
|
u32 init_nr = sysctl_perf_event_max_stack - max_depth;
|
2016-02-18 06:58:58 +03:00
|
|
|
u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
|
|
|
|
|
u32 hash, id, trace_nr, trace_len;
|
|
|
|
|
bool user = flags & BPF_F_USER_STACK;
|
|
|
|
|
bool kernel = !user;
|
|
|
|
|
u64 *ips;
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
bool hash_matches;
|
2016-02-18 06:58:58 +03:00
|
|
|
|
|
|
|
|
if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
|
|
|
|
|
BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
2016-04-28 18:30:53 +03:00
|
|
|
trace = get_perf_callchain(regs, init_nr, kernel, user,
|
|
|
|
|
sysctl_perf_event_max_stack, false, false);
|
2016-02-18 06:58:58 +03:00
|
|
|
|
|
|
|
|
if (unlikely(!trace))
|
|
|
|
|
/* couldn't fetch the stack trace */
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
/* get_perf_callchain() guarantees that trace->nr >= init_nr
|
2016-04-21 18:28:50 +03:00
|
|
|
* and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth
|
2016-02-18 06:58:58 +03:00
|
|
|
*/
|
|
|
|
|
trace_nr = trace->nr - init_nr;
|
|
|
|
|
|
|
|
|
|
if (trace_nr <= skip)
|
|
|
|
|
/* skipping more than usable stack trace */
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
|
|
trace_nr -= skip;
|
|
|
|
|
trace_len = trace_nr * sizeof(u64);
|
|
|
|
|
ips = trace->ip + skip + init_nr;
|
|
|
|
|
hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
|
|
|
|
|
id = hash & (smap->n_buckets - 1);
|
2016-03-08 08:57:17 +03:00
|
|
|
bucket = READ_ONCE(smap->buckets[id]);
|
2016-02-18 06:58:58 +03:00
|
|
|
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
hash_matches = bucket && bucket->hash == hash;
|
|
|
|
|
/* fast cmp */
|
|
|
|
|
if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
|
|
|
|
|
return id;
|
|
|
|
|
|
|
|
|
|
if (stack_map_use_build_id(map)) {
|
|
|
|
|
/* for build_id+offset, pop a bucket before slow cmp */
|
|
|
|
|
new_bucket = (struct stack_map_bucket *)
|
|
|
|
|
pcpu_freelist_pop(&smap->freelist);
|
|
|
|
|
if (unlikely(!new_bucket))
|
|
|
|
|
return -ENOMEM;
|
2018-04-29 08:28:07 +03:00
|
|
|
new_bucket->nr = trace_nr;
|
|
|
|
|
stack_map_get_build_id_offset(
|
|
|
|
|
(struct bpf_stack_build_id *)new_bucket->data,
|
|
|
|
|
ips, trace_nr, user);
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
trace_len = trace_nr * sizeof(struct bpf_stack_build_id);
|
|
|
|
|
if (hash_matches && bucket->nr == trace_nr &&
|
|
|
|
|
memcmp(bucket->data, new_bucket->data, trace_len) == 0) {
|
|
|
|
|
pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
|
2016-02-18 06:58:58 +03:00
|
|
|
return id;
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
}
|
|
|
|
|
if (bucket && !(flags & BPF_F_REUSE_STACKID)) {
|
|
|
|
|
pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
|
|
|
|
|
return -EEXIST;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if (hash_matches && bucket->nr == trace_nr &&
|
|
|
|
|
memcmp(bucket->data, ips, trace_len) == 0)
|
2016-02-18 06:58:58 +03:00
|
|
|
return id;
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
if (bucket && !(flags & BPF_F_REUSE_STACKID))
|
|
|
|
|
return -EEXIST;
|
|
|
|
|
|
|
|
|
|
new_bucket = (struct stack_map_bucket *)
|
|
|
|
|
pcpu_freelist_pop(&smap->freelist);
|
|
|
|
|
if (unlikely(!new_bucket))
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
memcpy(new_bucket->data, ips, trace_len);
|
2016-02-18 06:58:58 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
new_bucket->hash = hash;
|
|
|
|
|
new_bucket->nr = trace_nr;
|
|
|
|
|
|
|
|
|
|
old_bucket = xchg(&smap->buckets[id], new_bucket);
|
|
|
|
|
if (old_bucket)
|
2016-03-08 08:57:17 +03:00
|
|
|
pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
|
2016-02-18 06:58:58 +03:00
|
|
|
return id;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const struct bpf_func_proto bpf_get_stackid_proto = {
|
|
|
|
|
.func = bpf_get_stackid,
|
|
|
|
|
.gpl_only = true,
|
|
|
|
|
.ret_type = RET_INTEGER,
|
|
|
|
|
.arg1_type = ARG_PTR_TO_CTX,
|
|
|
|
|
.arg2_type = ARG_CONST_MAP_PTR,
|
|
|
|
|
.arg3_type = ARG_ANYTHING,
|
|
|
|
|
};
|
|
|
|
|
|
2018-04-29 08:28:08 +03:00
|
|
|
BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,
|
|
|
|
|
u64, flags)
|
|
|
|
|
{
|
|
|
|
|
u32 init_nr, trace_nr, copy_len, elem_size, num_elem;
|
|
|
|
|
bool user_build_id = flags & BPF_F_USER_BUILD_ID;
|
|
|
|
|
u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
|
|
|
|
|
bool user = flags & BPF_F_USER_STACK;
|
|
|
|
|
struct perf_callchain_entry *trace;
|
|
|
|
|
bool kernel = !user;
|
|
|
|
|
int err = -EINVAL;
|
|
|
|
|
u64 *ips;
|
|
|
|
|
|
|
|
|
|
if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
|
|
|
|
|
BPF_F_USER_BUILD_ID)))
|
|
|
|
|
goto clear;
|
|
|
|
|
if (kernel && user_build_id)
|
|
|
|
|
goto clear;
|
|
|
|
|
|
|
|
|
|
elem_size = (user && user_build_id) ? sizeof(struct bpf_stack_build_id)
|
|
|
|
|
: sizeof(u64);
|
|
|
|
|
if (unlikely(size % elem_size))
|
|
|
|
|
goto clear;
|
|
|
|
|
|
|
|
|
|
num_elem = size / elem_size;
|
|
|
|
|
if (sysctl_perf_event_max_stack < num_elem)
|
|
|
|
|
init_nr = 0;
|
|
|
|
|
else
|
|
|
|
|
init_nr = sysctl_perf_event_max_stack - num_elem;
|
|
|
|
|
trace = get_perf_callchain(regs, init_nr, kernel, user,
|
|
|
|
|
sysctl_perf_event_max_stack, false, false);
|
|
|
|
|
if (unlikely(!trace))
|
|
|
|
|
goto err_fault;
|
|
|
|
|
|
|
|
|
|
trace_nr = trace->nr - init_nr;
|
|
|
|
|
if (trace_nr < skip)
|
|
|
|
|
goto err_fault;
|
|
|
|
|
|
|
|
|
|
trace_nr -= skip;
|
|
|
|
|
trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
|
|
|
|
|
copy_len = trace_nr * elem_size;
|
|
|
|
|
ips = trace->ip + skip + init_nr;
|
|
|
|
|
if (user && user_build_id)
|
|
|
|
|
stack_map_get_build_id_offset(buf, ips, trace_nr, user);
|
|
|
|
|
else
|
|
|
|
|
memcpy(buf, ips, copy_len);
|
|
|
|
|
|
|
|
|
|
if (size > copy_len)
|
|
|
|
|
memset(buf + copy_len, 0, size - copy_len);
|
|
|
|
|
return copy_len;
|
|
|
|
|
|
|
|
|
|
err_fault:
|
|
|
|
|
err = -EFAULT;
|
|
|
|
|
clear:
|
|
|
|
|
memset(buf, 0, size);
|
|
|
|
|
return err;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const struct bpf_func_proto bpf_get_stack_proto = {
|
|
|
|
|
.func = bpf_get_stack,
|
|
|
|
|
.gpl_only = true,
|
|
|
|
|
.ret_type = RET_INTEGER,
|
|
|
|
|
.arg1_type = ARG_PTR_TO_CTX,
|
|
|
|
|
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
|
|
|
|
|
.arg3_type = ARG_CONST_SIZE_OR_ZERO,
|
|
|
|
|
.arg4_type = ARG_ANYTHING,
|
|
|
|
|
};
|
|
|
|
|
|
2016-03-08 08:57:17 +03:00
|
|
|
/* Called from eBPF program */
|
2016-02-18 06:58:58 +03:00
|
|
|
static void *stack_map_lookup_elem(struct bpf_map *map, void *key)
|
2016-03-08 08:57:17 +03:00
|
|
|
{
|
2018-10-09 04:04:50 +03:00
|
|
|
return ERR_PTR(-EOPNOTSUPP);
|
2016-03-08 08:57:17 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Called from syscall */
|
|
|
|
|
int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
|
2016-02-18 06:58:58 +03:00
|
|
|
{
|
|
|
|
|
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
|
2016-03-08 08:57:17 +03:00
|
|
|
struct stack_map_bucket *bucket, *old_bucket;
|
|
|
|
|
u32 id = *(u32 *)key, trace_len;
|
2016-02-18 06:58:58 +03:00
|
|
|
|
|
|
|
|
if (unlikely(id >= smap->n_buckets))
|
2016-03-08 08:57:17 +03:00
|
|
|
return -ENOENT;
|
|
|
|
|
|
|
|
|
|
bucket = xchg(&smap->buckets[id], NULL);
|
|
|
|
|
if (!bucket)
|
|
|
|
|
return -ENOENT;
|
|
|
|
|
|
bpf: extend stackmap to save binary_build_id+offset instead of address
Currently, bpf stackmap store address for each entry in the call trace.
To map these addresses to user space files, it is necessary to maintain
the mapping from these virtual address to symbols in the binary. Usually,
the user space profiler (such as perf) has to scan /proc/pid/maps at the
beginning of profiling, and monitor mmap2() calls afterwards. Given the
cost of maintaining the address map, this solution is not practical for
system wide profiling that is always on.
This patch tries to solve this problem with a variation of stackmap. This
variation is enabled by flag BPF_F_STACK_BUILD_ID. Instead of storing
addresses, the variation stores ELF file build_id + offset.
Build ID is a 20-byte unique identifier for ELF files. The following
command shows the Build ID of /bin/bash:
[user@]$ readelf -n /bin/bash
...
Build ID: XXXXXXXXXX
...
With BPF_F_STACK_BUILD_ID, bpf_get_stackid() tries to parse Build ID
for each entry in the call trace, and translate it into the following
struct:
struct bpf_stack_build_id_offset {
__s32 status;
unsigned char build_id[BPF_BUILD_ID_SIZE];
union {
__u64 offset;
__u64 ip;
};
};
The search of build_id is limited to the first page of the file, and this
page should be in page cache. Otherwise, we fallback to store ip for this
entry (ip field in struct bpf_stack_build_id_offset). This requires the
build_id to be stored in the first page. A quick survey of binary and
dynamic library files in a few different systems shows that almost all
binary and dynamic library files have build_id in the first page.
Build_id is only meaningful for user stack. If a kernel stack is added to
a stackmap with BPF_F_STACK_BUILD_ID, it will automatically fallback to
only store ip (status == BPF_STACK_BUILD_ID_IP). Similarly, if build_id
lookup failed for some reason, it will also fallback to store ip.
User space can access struct bpf_stack_build_id_offset with bpf
syscall BPF_MAP_LOOKUP_ELEM. It is necessary for user space to
maintain mapping from build id to binary files. This mostly static
mapping is much easier to maintain than per process address maps.
Note: Stackmap with build_id only works in non-nmi context at this time.
This is because we need to take mm->mmap_sem for find_vma(). If this
changes, we would like to allow build_id lookup in nmi context.
Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-03-14 20:23:21 +03:00
|
|
|
trace_len = bucket->nr * stack_map_data_size(map);
|
|
|
|
|
memcpy(value, bucket->data, trace_len);
|
2016-03-08 08:57:17 +03:00
|
|
|
memset(value + trace_len, 0, map->value_size - trace_len);
|
|
|
|
|
|
|
|
|
|
old_bucket = xchg(&smap->buckets[id], bucket);
|
|
|
|
|
if (old_bucket)
|
|
|
|
|
pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
|
|
|
|
|
return 0;
|
2016-02-18 06:58:58 +03:00
|
|
|
}
|
|
|
|
|
|
2018-01-05 00:55:03 +03:00
|
|
|
static int stack_map_get_next_key(struct bpf_map *map, void *key,
|
|
|
|
|
void *next_key)
|
2016-02-18 06:58:58 +03:00
|
|
|
{
|
2018-01-05 00:55:03 +03:00
|
|
|
struct bpf_stack_map *smap = container_of(map,
|
|
|
|
|
struct bpf_stack_map, map);
|
|
|
|
|
u32 id;
|
|
|
|
|
|
|
|
|
|
WARN_ON_ONCE(!rcu_read_lock_held());
|
|
|
|
|
|
|
|
|
|
if (!key) {
|
|
|
|
|
id = 0;
|
|
|
|
|
} else {
|
|
|
|
|
id = *(u32 *)key;
|
|
|
|
|
if (id >= smap->n_buckets || !smap->buckets[id])
|
|
|
|
|
id = 0;
|
|
|
|
|
else
|
|
|
|
|
id++;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
while (id < smap->n_buckets && !smap->buckets[id])
|
|
|
|
|
id++;
|
|
|
|
|
|
|
|
|
|
if (id >= smap->n_buckets)
|
|
|
|
|
return -ENOENT;
|
|
|
|
|
|
|
|
|
|
*(u32 *)next_key = id;
|
|
|
|
|
return 0;
|
2016-02-18 06:58:58 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int stack_map_update_elem(struct bpf_map *map, void *key, void *value,
|
|
|
|
|
u64 map_flags)
|
|
|
|
|
{
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Called from syscall or from eBPF program */
|
|
|
|
|
static int stack_map_delete_elem(struct bpf_map *map, void *key)
|
|
|
|
|
{
|
|
|
|
|
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
|
|
|
|
|
struct stack_map_bucket *old_bucket;
|
|
|
|
|
u32 id = *(u32 *)key;
|
|
|
|
|
|
|
|
|
|
if (unlikely(id >= smap->n_buckets))
|
|
|
|
|
return -E2BIG;
|
|
|
|
|
|
|
|
|
|
old_bucket = xchg(&smap->buckets[id], NULL);
|
|
|
|
|
if (old_bucket) {
|
2016-03-08 08:57:17 +03:00
|
|
|
pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
|
2016-02-18 06:58:58 +03:00
|
|
|
return 0;
|
|
|
|
|
} else {
|
|
|
|
|
return -ENOENT;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Called when map->refcnt goes to zero, either from workqueue or from syscall */
|
|
|
|
|
static void stack_map_free(struct bpf_map *map)
|
|
|
|
|
{
|
|
|
|
|
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
|
|
|
|
|
|
2016-03-08 08:57:17 +03:00
|
|
|
/* wait for bpf programs to complete before freeing stack map */
|
2016-02-18 06:58:58 +03:00
|
|
|
synchronize_rcu();
|
|
|
|
|
|
bpf: don't trigger OOM killer under pressure with map alloc
This patch adds two helpers, bpf_map_area_alloc() and bpf_map_area_free(),
that are to be used for map allocations. Using kmalloc() for very large
allocations can cause excessive work within the page allocator, so i) fall
back earlier to vmalloc() when the attempt is considered costly anyway,
and even more importantly ii) don't trigger OOM killer with any of the
allocators.
Since this is based on a user space request, for example, when creating
maps with element pre-allocation, we really want such requests to fail
instead of killing other user space processes.
Also, don't spam the kernel log with warnings should any of the allocations
fail under pressure. Given that, we can make backend selection in
bpf_map_area_alloc() generic, and convert all maps over to use this API
for spots with potentially large allocation requests.
Note, replacing the one kmalloc_array() is fine as overflow checks happen
earlier in htab_map_alloc(), since it must also protect the multiplication
for vmalloc() should kmalloc_array() fail.
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-01-18 17:14:17 +03:00
|
|
|
bpf_map_area_free(smap->elems);
|
2016-03-08 08:57:17 +03:00
|
|
|
pcpu_freelist_destroy(&smap->freelist);
|
bpf: don't trigger OOM killer under pressure with map alloc
This patch adds two helpers, bpf_map_area_alloc() and bpf_map_area_free(),
that are to be used for map allocations. Using kmalloc() for very large
allocations can cause excessive work within the page allocator, so i) fall
back earlier to vmalloc() when the attempt is considered costly anyway,
and even more importantly ii) don't trigger OOM killer with any of the
allocators.
Since this is based on a user space request, for example, when creating
maps with element pre-allocation, we really want such requests to fail
instead of killing other user space processes.
Also, don't spam the kernel log with warnings should any of the allocations
fail under pressure. Given that, we can make backend selection in
bpf_map_area_alloc() generic, and convert all maps over to use this API
for spots with potentially large allocation requests.
Note, replacing the one kmalloc_array() is fine as overflow checks happen
earlier in htab_map_alloc(), since it must also protect the multiplication
for vmalloc() should kmalloc_array() fail.
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-01-18 17:14:17 +03:00
|
|
|
bpf_map_area_free(smap);
|
2016-02-18 06:58:58 +03:00
|
|
|
put_callchain_buffers();
|
|
|
|
|
}
|
|
|
|
|
|
2018-10-18 16:16:09 +03:00
|
|
|
const struct bpf_map_ops stack_trace_map_ops = {
|
2016-02-18 06:58:58 +03:00
|
|
|
.map_alloc = stack_map_alloc,
|
|
|
|
|
.map_free = stack_map_free,
|
|
|
|
|
.map_get_next_key = stack_map_get_next_key,
|
|
|
|
|
.map_lookup_elem = stack_map_lookup_elem,
|
|
|
|
|
.map_update_elem = stack_map_update_elem,
|
|
|
|
|
.map_delete_elem = stack_map_delete_elem,
|
2018-08-12 02:59:17 +03:00
|
|
|
.map_check_btf = map_check_no_btf,
|
2016-02-18 06:58:58 +03:00
|
|
|
};
|
2018-05-07 20:50:48 +03:00
|
|
|
|
|
|
|
|
static int __init stack_map_init(void)
|
|
|
|
|
{
|
|
|
|
|
int cpu;
|
|
|
|
|
struct stack_map_irq_work *work;
|
|
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
|
work = per_cpu_ptr(&up_read_work, cpu);
|
|
|
|
|
init_irq_work(&work->irq_work, do_up_read);
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
subsys_initcall(stack_map_init);
|
