2018-05-18 15:00:21 +03:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
/* AF_XDP internal functions
|
2018-05-02 14:01:23 +03:00
|
|
|
* Copyright(c) 2018 Intel Corporation.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _LINUX_XDP_SOCK_H
|
|
|
|
#define _LINUX_XDP_SOCK_H
|
|
|
|
|
2018-06-04 15:05:51 +03:00
|
|
|
#include <linux/workqueue.h>
|
|
|
|
#include <linux/if_xdp.h>
|
2018-05-02 14:01:23 +03:00
|
|
|
#include <linux/mutex.h>
|
2018-06-04 15:05:57 +03:00
|
|
|
#include <linux/spinlock.h>
|
2018-06-04 15:05:51 +03:00
|
|
|
#include <linux/mm.h>
|
2018-05-02 14:01:23 +03:00
|
|
|
#include <net/sock.h>
|
|
|
|
|
2018-05-02 14:01:25 +03:00
|
|
|
struct net_device;
|
|
|
|
struct xsk_queue;
|
2020-05-20 22:20:51 +03:00
|
|
|
struct xdp_buff;
|
2018-06-04 15:05:51 +03:00
|
|
|
|
|
|
|
struct xdp_umem {
|
2020-08-28 11:26:21 +03:00
|
|
|
void *addrs;
|
2018-08-31 14:40:02 +03:00
|
|
|
u64 size;
|
2018-06-04 15:05:51 +03:00
|
|
|
u32 headroom;
|
2020-05-20 22:20:53 +03:00
|
|
|
u32 chunk_size;
|
2020-08-28 11:26:17 +03:00
|
|
|
u32 chunks;
|
2020-08-28 11:26:23 +03:00
|
|
|
u32 npgs;
|
2018-06-04 15:05:51 +03:00
|
|
|
struct user_struct *user;
|
|
|
|
refcount_t users;
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 10:27:17 +03:00
|
|
|
u8 flags;
|
2018-06-04 15:05:55 +03:00
|
|
|
bool zc;
|
2020-08-28 11:26:23 +03:00
|
|
|
struct page **pgs;
|
|
|
|
int id;
|
2020-08-28 11:26:22 +03:00
|
|
|
struct list_head xsk_dma_list;
|
xsk: Fix umem cleanup bug at socket destruct
Fix a bug that is triggered when a partially setup socket is
destroyed. For a fully setup socket, a socket that has been bound to a
device, the cleanup of the umem is performed at the end of the buffer
pool's cleanup work queue item. This has to be performed in a work
queue, and not in RCU cleanup, as it is doing a vunmap that cannot
execute in interrupt context. However, when a socket has only been
partially set up so that a umem has been created but the buffer pool
has not, the code erroneously directly calls the umem cleanup function
instead of using a work queue, and this leads to a BUG_ON() in
vunmap().
As there in this case is no buffer pool, we cannot use its work queue,
so we need to introduce a work queue for the umem and schedule this for
the cleanup. So in the case there is no pool, we are going to use the
umem's own work queue to schedule the cleanup. But if there is a
pool, the cleanup of the umem is still being performed by the pool's
work queue, as it is important that the umem is cleaned up after the
pool.
Fixes: e5e1a4bc916d ("xsk: Fix possible memory leak at socket close")
Reported-by: Marek Majtyka <marekx.majtyka@intel.com>
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Tested-by: Marek Majtyka <marekx.majtyka@intel.com>
Link: https://lore.kernel.org/bpf/1605873219-21629-1-git-send-email-magnus.karlsson@gmail.com
2020-11-20 14:53:39 +03:00
|
|
|
struct work_struct work;
|
2018-06-04 15:05:51 +03:00
|
|
|
};
|
2018-05-02 14:01:23 +03:00
|
|
|
|
2019-11-01 14:03:46 +03:00
|
|
|
struct xsk_map {
|
|
|
|
struct bpf_map map;
|
|
|
|
spinlock_t lock; /* Synchronize map updates */
|
xdp: Add proper __rcu annotations to redirect map entries
XDP_REDIRECT works by a three-step process: the bpf_redirect() and
bpf_redirect_map() helpers will lookup the target of the redirect and store
it (along with some other metadata) in a per-CPU struct bpf_redirect_info.
Next, when the program returns the XDP_REDIRECT return code, the driver
will call xdp_do_redirect() which will use the information thus stored to
actually enqueue the frame into a bulk queue structure (that differs
slightly by map type, but shares the same principle). Finally, before
exiting its NAPI poll loop, the driver will call xdp_do_flush(), which will
flush all the different bulk queues, thus completing the redirect.
Pointers to the map entries will be kept around for this whole sequence of
steps, protected by RCU. However, there is no top-level rcu_read_lock() in
the core code; instead drivers add their own rcu_read_lock() around the XDP
portions of the code, but somewhat inconsistently as Martin discovered[0].
However, things still work because everything happens inside a single NAPI
poll sequence, which means it's between a pair of calls to
local_bh_disable()/local_bh_enable(). So Paul suggested[1] that we could
document this intention by using rcu_dereference_check() with
rcu_read_lock_bh_held() as a second parameter, thus allowing sparse and
lockdep to verify that everything is done correctly.
This patch does just that: we add an __rcu annotation to the map entry
pointers and remove the various comments explaining the NAPI poll assurance
strewn through devmap.c in favour of a longer explanation in filter.c. The
goal is to have one coherent documentation of the entire flow, and rely on
the RCU annotations as a "standard" way of communicating the flow in the
map code (which can additionally be understood by sparse and lockdep).
The RCU annotation replacements result in a fairly straight-forward
replacement where READ_ONCE() becomes rcu_dereference_check(), WRITE_ONCE()
becomes rcu_assign_pointer() and xchg() and cmpxchg() gets wrapped in the
proper constructs to cast the pointer back and forth between __rcu and
__kernel address space (for the benefit of sparse). The one complication is
that xskmap has a few constructions where double-pointers are passed back
and forth; these simply all gain __rcu annotations, and only the final
reference/dereference to the inner-most pointer gets changed.
With this, everything can be run through sparse without eliciting
complaints, and lockdep can verify correctness even without the use of
rcu_read_lock() in the drivers. Subsequent patches will clean these up from
the drivers.
[0] https://lore.kernel.org/bpf/20210415173551.7ma4slcbqeyiba2r@kafai-mbp.dhcp.thefacebook.com/
[1] https://lore.kernel.org/bpf/20210419165837.GA975577@paulmck-ThinkPad-P17-Gen-1/
Signed-off-by: Toke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20210624160609.292325-6-toke@redhat.com
2021-06-24 19:05:55 +03:00
|
|
|
struct xdp_sock __rcu *xsk_map[];
|
2019-11-01 14:03:46 +03:00
|
|
|
};
|
|
|
|
|
2018-05-02 14:01:23 +03:00
|
|
|
struct xdp_sock {
|
|
|
|
/* struct sock must be the first member of struct xdp_sock */
|
|
|
|
struct sock sk;
|
2020-08-28 11:26:23 +03:00
|
|
|
struct xsk_queue *rx ____cacheline_aligned_in_smp;
|
2018-05-02 14:01:25 +03:00
|
|
|
struct net_device *dev;
|
2018-05-02 14:01:23 +03:00
|
|
|
struct xdp_umem *umem;
|
2018-05-02 14:01:28 +03:00
|
|
|
struct list_head flush_node;
|
2020-08-28 11:26:16 +03:00
|
|
|
struct xsk_buff_pool *pool;
|
2018-05-02 14:01:26 +03:00
|
|
|
u16 queue_id;
|
2018-06-04 15:05:57 +03:00
|
|
|
bool zc;
|
2019-06-28 11:04:07 +03:00
|
|
|
enum {
|
|
|
|
XSK_READY = 0,
|
|
|
|
XSK_BOUND,
|
|
|
|
XSK_UNBOUND,
|
|
|
|
} state;
|
2020-08-28 11:26:23 +03:00
|
|
|
|
2019-06-06 23:59:40 +03:00
|
|
|
struct xsk_queue *tx ____cacheline_aligned_in_smp;
|
2020-08-28 11:26:20 +03:00
|
|
|
struct list_head tx_list;
|
2019-07-03 15:09:16 +03:00
|
|
|
/* Protects generic receive. */
|
|
|
|
spinlock_t rx_lock;
|
2020-07-08 10:28:33 +03:00
|
|
|
|
|
|
|
/* Statistics */
|
2018-05-02 14:01:27 +03:00
|
|
|
u64 rx_dropped;
|
2020-07-08 10:28:33 +03:00
|
|
|
u64 rx_queue_full;
|
|
|
|
|
2019-08-15 12:30:13 +03:00
|
|
|
struct list_head map_list;
|
|
|
|
/* Protects map_list */
|
|
|
|
spinlock_t map_list_lock;
|
2020-08-28 11:26:23 +03:00
|
|
|
/* Protects multiple processes in the control path */
|
|
|
|
struct mutex mutex;
|
2020-08-28 11:26:18 +03:00
|
|
|
struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */
|
|
|
|
struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
|
2018-05-02 14:01:23 +03:00
|
|
|
};
|
|
|
|
|
2018-05-02 14:01:27 +03:00
|
|
|
#ifdef CONFIG_XDP_SOCKETS
|
2018-08-28 15:44:27 +03:00
|
|
|
|
2020-05-20 22:20:51 +03:00
|
|
|
int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
|
2019-12-19 09:10:02 +03:00
|
|
|
int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
|
|
|
|
void __xsk_map_flush(void);
|
2019-11-01 14:03:46 +03:00
|
|
|
|
2018-05-02 14:01:27 +03:00
|
|
|
#else
|
2020-05-20 22:20:51 +03:00
|
|
|
|
2018-05-02 14:01:27 +03:00
|
|
|
static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
|
|
|
|
{
|
|
|
|
return -ENOTSUPP;
|
|
|
|
}
|
|
|
|
|
2020-05-20 22:20:51 +03:00
|
|
|
static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
|
2018-09-07 11:18:46 +03:00
|
|
|
{
|
2020-05-20 22:20:51 +03:00
|
|
|
return -EOPNOTSUPP;
|
2018-09-07 11:18:46 +03:00
|
|
|
}
|
|
|
|
|
2020-05-20 22:20:51 +03:00
|
|
|
static inline void __xsk_map_flush(void)
|
2018-09-07 11:18:46 +03:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2018-05-02 14:01:27 +03:00
|
|
|
#endif /* CONFIG_XDP_SOCKETS */
|
|
|
|
|
2018-05-02 14:01:23 +03:00
|
|
|
#endif /* _LINUX_XDP_SOCK_H */
|