2018-05-18 15:00:21 +03:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
/* XDP user-space ring structure
|
2018-05-02 14:01:24 +03:00
|
|
|
* Copyright(c) 2018 Intel Corporation.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _LINUX_XSK_QUEUE_H
|
|
|
|
#define _LINUX_XSK_QUEUE_H
|
|
|
|
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/if_xdp.h>
|
2018-06-04 15:05:51 +03:00
|
|
|
#include <net/xdp_sock.h>
|
2020-05-20 22:20:53 +03:00
|
|
|
#include <net/xsk_buff_pool.h>
|
2018-05-02 14:01:24 +03:00
|
|
|
|
2020-05-20 22:20:52 +03:00
|
|
|
#include "xsk.h"
|
|
|
|
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 10:34:59 +03:00
|
|
|
struct xdp_ring {
|
|
|
|
u32 producer ____cacheline_aligned_in_smp;
|
2020-10-08 17:12:18 +03:00
|
|
|
/* Hinder the adjacent cache prefetcher to prefetch the consumer
|
|
|
|
* pointer if the producer pointer is touched and vice versa.
|
|
|
|
*/
|
2020-11-16 14:12:45 +03:00
|
|
|
u32 pad1 ____cacheline_aligned_in_smp;
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 10:34:59 +03:00
|
|
|
u32 consumer ____cacheline_aligned_in_smp;
|
2020-11-16 14:12:45 +03:00
|
|
|
u32 pad2 ____cacheline_aligned_in_smp;
|
xsk: add support for need_wakeup flag in AF_XDP rings
This commit adds support for a new flag called need_wakeup in the
AF_XDP Tx and fill rings. When this flag is set, it means that the
application has to explicitly wake up the kernel Rx (for the bit in
the fill ring) or kernel Tx (for bit in the Tx ring) processing by
issuing a syscall. Poll() can wake up both depending on the flags
submitted and sendto() will wake up tx processing only.
The main reason for introducing this new flag is to be able to
efficiently support the case when application and driver is executing
on the same core. Previously, the driver was just busy-spinning on the
fill ring if it ran out of buffers in the HW and there were none on
the fill ring. This approach works when the application is running on
another core as it can replenish the fill ring while the driver is
busy-spinning. Though, this is a lousy approach if both of them are
running on the same core as the probability of the fill ring getting
more entries when the driver is busy-spinning is zero. With this new
feature the driver now sets the need_wakeup flag and returns to the
application. The application can then replenish the fill queue and
then explicitly wake up the Rx processing in the kernel using the
syscall poll(). For Tx, the flag is only set to one if the driver has
no outstanding Tx completion interrupts. If it has some, the flag is
zero as it will be woken up by a completion interrupt anyway.
As a nice side effect, this new flag also improves the performance of
the case where application and driver are running on two different
cores as it reduces the number of syscalls to the kernel. The kernel
tells user space if it needs to be woken up by a syscall, and this
eliminates many of the syscalls.
This flag needs some simple driver support. If the driver does not
support this, the Rx flag is always zero and the Tx flag is always
one. This makes any application relying on this feature default to the
old behaviour of not requiring any syscalls in the Rx path and always
having to call sendto() in the Tx path.
For backwards compatibility reasons, this feature has to be explicitly
turned on using a new bind flag (XDP_USE_NEED_WAKEUP). I recommend
that you always turn it on as it so far always have had a positive
performance impact.
The name and inspiration of the flag has been taken from io_uring by
Jens Axboe. Details about this feature in io_uring can be found in
http://kernel.dk/io_uring.pdf, section 8.3.
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-14 10:27:17 +03:00
|
|
|
u32 flags;
|
2020-11-16 14:12:45 +03:00
|
|
|
u32 pad3 ____cacheline_aligned_in_smp;
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 10:34:59 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
/* Used for the RX and TX queues for packets */
|
|
|
|
struct xdp_rxtx_ring {
|
|
|
|
struct xdp_ring ptrs;
|
2020-02-28 16:19:07 +03:00
|
|
|
struct xdp_desc desc[] ____cacheline_aligned_in_smp;
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 10:34:59 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
/* Used for the fill and completion queues for buffers */
|
|
|
|
struct xdp_umem_ring {
|
|
|
|
struct xdp_ring ptrs;
|
2020-02-28 16:19:07 +03:00
|
|
|
u64 desc[] ____cacheline_aligned_in_smp;
|
xsk: remove explicit ring structure from uapi
In this commit we remove the explicit ring structure from the the
uapi. It is tricky for an uapi to depend on a certain L1 cache line
size, since it can differ for variants of the same architecture. Now,
we let the user application determine the offsets of the producer,
consumer and descriptors by asking the socket via getsockopt.
A typical flow would be (Rx ring):
struct xdp_mmap_offsets off;
struct xdp_desc *ring;
u32 *prod, *cons;
void *map;
...
getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);
map = mmap(NULL, off.rx.desc +
NUM_DESCS * sizeof(struct xdp_desc),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, sfd,
XDP_PGOFF_RX_RING);
prod = map + off.rx.producer;
cons = map + off.rx.consumer;
ring = map + off.rx.desc;
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-05-22 10:34:59 +03:00
|
|
|
};
|
|
|
|
|
2018-05-02 14:01:24 +03:00
|
|
|
struct xsk_queue {
|
|
|
|
u32 ring_mask;
|
|
|
|
u32 nentries;
|
2019-12-19 15:39:22 +03:00
|
|
|
u32 cached_prod;
|
2019-12-19 15:39:26 +03:00
|
|
|
u32 cached_cons;
|
2018-05-02 14:01:24 +03:00
|
|
|
struct xdp_ring *ring;
|
|
|
|
u64 invalid_descs;
|
2020-07-08 10:28:33 +03:00
|
|
|
u64 queue_empty_descs;
|
2023-02-16 11:30:47 +03:00
|
|
|
size_t ring_vmalloc_size;
|
2018-05-02 14:01:24 +03:00
|
|
|
};
|
|
|
|
|
xsk: Update rings for load-acquire/store-release barriers
Currently, the AF_XDP rings uses general smp_{r,w,}mb() barriers on
the kernel-side. On most modern architectures
load-acquire/store-release barriers perform better, and results in
simpler code for circular ring buffers.
This change updates the XDP socket rings to use
load-acquire/store-release barriers.
It is important to note that changing from the old smp_{r,w,}mb()
barriers, to load-acquire/store-release barriers does not break
compatibility. The old semantics work with the new one, and vice
versa.
As pointed out by "Documentation/memory-barriers.txt" in the "SMP
BARRIER PAIRING" section:
"General barriers pair with each other, though they also pair with
most other types of barriers, albeit without multicopy atomicity.
An acquire barrier pairs with a release barrier, but both may also
pair with other barriers, including of course general barriers."
How different barriers behaves and pairs is outlined in
"tools/memory-model/Documentation/cheatsheet.txt".
In order to make sure that compatibility is not broken, LKMM herd7
based litmus tests can be constructed and verified.
We generalize the XDP socket ring to a one entry ring, and create two
scenarios; One where the ring is full, where only the consumer can
proceed, followed by the producer. One where the ring is empty, where
only the producer can proceed, followed by the consumer. Each scenario
is then expanded to four different tests: general producer/general
consumer, general producer/acqrel consumer, acqrel producer/general
consumer, acqrel producer/acqrel consumer. In total eight tests.
The empty ring test:
C spsc-rb+empty
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{}
// We start at prod==0, cons==0, data==0, i.e. nothing has been
// written to the ring. From here only the producer can start, and
// should write 1. Afterwards, consumer can continue and read 1 to
// data. Can we enter state prod==1, cons==1, but consumer observed
// the incorrect value of 0?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=0 /\ prod=1 /\ cons=1 );
The full ring test:
C spsc-rb+full
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{ prod = 1; }
// We start at prod==1, cons==0, data==1, i.e. producer has
// written 0, so from here only the consumer can start, and should
// consume 0. Afterwards, producer can continue and write 1 to
// data. Can we enter state prod==0, cons==1, but consumer observed
// the write of 1?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=1 /\ prod=0 /\ cons=1 );
where P0 and P1 are:
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_wmb();
WRITE_ONCE(*prod, p ^ 1);
}
}
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_store_release(prod, p ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (READ_ONCE(*prod) != c) {
smp_rmb();
d = READ_ONCE(*data);
smp_mb();
WRITE_ONCE(*cons, c ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (smp_load_acquire(prod) != c) {
d = READ_ONCE(*data);
smp_store_release(cons, c ^ 1);
}
}
The full LKMM litmus tests are found at [1].
On x86-64 systems the l2fwd AF_XDP xdpsock sample performance
increases by 1%. This is mostly due to that the smp_mb() is removed,
which is a relatively expensive operation on these
platforms. Weakly-ordered platforms, such as ARM64 might benefit even
more.
[1] https://github.com/bjoto/litmus-xsk
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://lore.kernel.org/bpf/20210305094113.413544-2-bjorn.topel@gmail.com
2021-03-05 12:41:12 +03:00
|
|
|
/* The structure of the shared state of the rings are a simple
|
|
|
|
* circular buffer, as outlined in
|
|
|
|
* Documentation/core-api/circular-buffers.rst. For the Rx and
|
|
|
|
* completion ring, the kernel is the producer and user space is the
|
|
|
|
* consumer. For the Tx and fill rings, the kernel is the consumer and
|
|
|
|
* user space is the producer.
|
2019-04-16 15:58:08 +03:00
|
|
|
*
|
|
|
|
* producer consumer
|
|
|
|
*
|
xsk: Update rings for load-acquire/store-release barriers
Currently, the AF_XDP rings uses general smp_{r,w,}mb() barriers on
the kernel-side. On most modern architectures
load-acquire/store-release barriers perform better, and results in
simpler code for circular ring buffers.
This change updates the XDP socket rings to use
load-acquire/store-release barriers.
It is important to note that changing from the old smp_{r,w,}mb()
barriers, to load-acquire/store-release barriers does not break
compatibility. The old semantics work with the new one, and vice
versa.
As pointed out by "Documentation/memory-barriers.txt" in the "SMP
BARRIER PAIRING" section:
"General barriers pair with each other, though they also pair with
most other types of barriers, albeit without multicopy atomicity.
An acquire barrier pairs with a release barrier, but both may also
pair with other barriers, including of course general barriers."
How different barriers behaves and pairs is outlined in
"tools/memory-model/Documentation/cheatsheet.txt".
In order to make sure that compatibility is not broken, LKMM herd7
based litmus tests can be constructed and verified.
We generalize the XDP socket ring to a one entry ring, and create two
scenarios; One where the ring is full, where only the consumer can
proceed, followed by the producer. One where the ring is empty, where
only the producer can proceed, followed by the consumer. Each scenario
is then expanded to four different tests: general producer/general
consumer, general producer/acqrel consumer, acqrel producer/general
consumer, acqrel producer/acqrel consumer. In total eight tests.
The empty ring test:
C spsc-rb+empty
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{}
// We start at prod==0, cons==0, data==0, i.e. nothing has been
// written to the ring. From here only the producer can start, and
// should write 1. Afterwards, consumer can continue and read 1 to
// data. Can we enter state prod==1, cons==1, but consumer observed
// the incorrect value of 0?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=0 /\ prod=1 /\ cons=1 );
The full ring test:
C spsc-rb+full
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{ prod = 1; }
// We start at prod==1, cons==0, data==1, i.e. producer has
// written 0, so from here only the consumer can start, and should
// consume 0. Afterwards, producer can continue and write 1 to
// data. Can we enter state prod==0, cons==1, but consumer observed
// the write of 1?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=1 /\ prod=0 /\ cons=1 );
where P0 and P1 are:
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_wmb();
WRITE_ONCE(*prod, p ^ 1);
}
}
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_store_release(prod, p ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (READ_ONCE(*prod) != c) {
smp_rmb();
d = READ_ONCE(*data);
smp_mb();
WRITE_ONCE(*cons, c ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (smp_load_acquire(prod) != c) {
d = READ_ONCE(*data);
smp_store_release(cons, c ^ 1);
}
}
The full LKMM litmus tests are found at [1].
On x86-64 systems the l2fwd AF_XDP xdpsock sample performance
increases by 1%. This is mostly due to that the smp_mb() is removed,
which is a relatively expensive operation on these
platforms. Weakly-ordered platforms, such as ARM64 might benefit even
more.
[1] https://github.com/bjoto/litmus-xsk
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://lore.kernel.org/bpf/20210305094113.413544-2-bjorn.topel@gmail.com
2021-03-05 12:41:12 +03:00
|
|
|
* if (LOAD ->consumer) { (A) LOAD.acq ->producer (C)
|
2019-04-16 15:58:08 +03:00
|
|
|
* STORE $data LOAD $data
|
xsk: Update rings for load-acquire/store-release barriers
Currently, the AF_XDP rings uses general smp_{r,w,}mb() barriers on
the kernel-side. On most modern architectures
load-acquire/store-release barriers perform better, and results in
simpler code for circular ring buffers.
This change updates the XDP socket rings to use
load-acquire/store-release barriers.
It is important to note that changing from the old smp_{r,w,}mb()
barriers, to load-acquire/store-release barriers does not break
compatibility. The old semantics work with the new one, and vice
versa.
As pointed out by "Documentation/memory-barriers.txt" in the "SMP
BARRIER PAIRING" section:
"General barriers pair with each other, though they also pair with
most other types of barriers, albeit without multicopy atomicity.
An acquire barrier pairs with a release barrier, but both may also
pair with other barriers, including of course general barriers."
How different barriers behaves and pairs is outlined in
"tools/memory-model/Documentation/cheatsheet.txt".
In order to make sure that compatibility is not broken, LKMM herd7
based litmus tests can be constructed and verified.
We generalize the XDP socket ring to a one entry ring, and create two
scenarios; One where the ring is full, where only the consumer can
proceed, followed by the producer. One where the ring is empty, where
only the producer can proceed, followed by the consumer. Each scenario
is then expanded to four different tests: general producer/general
consumer, general producer/acqrel consumer, acqrel producer/general
consumer, acqrel producer/acqrel consumer. In total eight tests.
The empty ring test:
C spsc-rb+empty
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{}
// We start at prod==0, cons==0, data==0, i.e. nothing has been
// written to the ring. From here only the producer can start, and
// should write 1. Afterwards, consumer can continue and read 1 to
// data. Can we enter state prod==1, cons==1, but consumer observed
// the incorrect value of 0?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=0 /\ prod=1 /\ cons=1 );
The full ring test:
C spsc-rb+full
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{ prod = 1; }
// We start at prod==1, cons==0, data==1, i.e. producer has
// written 0, so from here only the consumer can start, and should
// consume 0. Afterwards, producer can continue and write 1 to
// data. Can we enter state prod==0, cons==1, but consumer observed
// the write of 1?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=1 /\ prod=0 /\ cons=1 );
where P0 and P1 are:
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_wmb();
WRITE_ONCE(*prod, p ^ 1);
}
}
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_store_release(prod, p ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (READ_ONCE(*prod) != c) {
smp_rmb();
d = READ_ONCE(*data);
smp_mb();
WRITE_ONCE(*cons, c ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (smp_load_acquire(prod) != c) {
d = READ_ONCE(*data);
smp_store_release(cons, c ^ 1);
}
}
The full LKMM litmus tests are found at [1].
On x86-64 systems the l2fwd AF_XDP xdpsock sample performance
increases by 1%. This is mostly due to that the smp_mb() is removed,
which is a relatively expensive operation on these
platforms. Weakly-ordered platforms, such as ARM64 might benefit even
more.
[1] https://github.com/bjoto/litmus-xsk
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://lore.kernel.org/bpf/20210305094113.413544-2-bjorn.topel@gmail.com
2021-03-05 12:41:12 +03:00
|
|
|
* STORE.rel ->producer (B) STORE.rel ->consumer (D)
|
2019-04-16 15:58:08 +03:00
|
|
|
* }
|
|
|
|
*
|
|
|
|
* (A) pairs with (D), and (B) pairs with (C).
|
|
|
|
*
|
|
|
|
* Starting with (B), it protects the data from being written after
|
|
|
|
* the producer pointer. If this barrier was missing, the consumer
|
|
|
|
* could observe the producer pointer being set and thus load the data
|
|
|
|
* before the producer has written the new data. The consumer would in
|
|
|
|
* this case load the old data.
|
|
|
|
*
|
|
|
|
* (C) protects the consumer from speculatively loading the data before
|
|
|
|
* the producer pointer actually has been read. If we do not have this
|
|
|
|
* barrier, some architectures could load old data as speculative loads
|
|
|
|
* are not discarded as the CPU does not know there is a dependency
|
|
|
|
* between ->producer and data.
|
|
|
|
*
|
|
|
|
* (A) is a control dependency that separates the load of ->consumer
|
|
|
|
* from the stores of $data. In case ->consumer indicates there is no
|
xsk: Update rings for load-acquire/store-release barriers
Currently, the AF_XDP rings uses general smp_{r,w,}mb() barriers on
the kernel-side. On most modern architectures
load-acquire/store-release barriers perform better, and results in
simpler code for circular ring buffers.
This change updates the XDP socket rings to use
load-acquire/store-release barriers.
It is important to note that changing from the old smp_{r,w,}mb()
barriers, to load-acquire/store-release barriers does not break
compatibility. The old semantics work with the new one, and vice
versa.
As pointed out by "Documentation/memory-barriers.txt" in the "SMP
BARRIER PAIRING" section:
"General barriers pair with each other, though they also pair with
most other types of barriers, albeit without multicopy atomicity.
An acquire barrier pairs with a release barrier, but both may also
pair with other barriers, including of course general barriers."
How different barriers behaves and pairs is outlined in
"tools/memory-model/Documentation/cheatsheet.txt".
In order to make sure that compatibility is not broken, LKMM herd7
based litmus tests can be constructed and verified.
We generalize the XDP socket ring to a one entry ring, and create two
scenarios; One where the ring is full, where only the consumer can
proceed, followed by the producer. One where the ring is empty, where
only the producer can proceed, followed by the consumer. Each scenario
is then expanded to four different tests: general producer/general
consumer, general producer/acqrel consumer, acqrel producer/general
consumer, acqrel producer/acqrel consumer. In total eight tests.
The empty ring test:
C spsc-rb+empty
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{}
// We start at prod==0, cons==0, data==0, i.e. nothing has been
// written to the ring. From here only the producer can start, and
// should write 1. Afterwards, consumer can continue and read 1 to
// data. Can we enter state prod==1, cons==1, but consumer observed
// the incorrect value of 0?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=0 /\ prod=1 /\ cons=1 );
The full ring test:
C spsc-rb+full
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{ prod = 1; }
// We start at prod==1, cons==0, data==1, i.e. producer has
// written 0, so from here only the consumer can start, and should
// consume 0. Afterwards, producer can continue and write 1 to
// data. Can we enter state prod==0, cons==1, but consumer observed
// the write of 1?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=1 /\ prod=0 /\ cons=1 );
where P0 and P1 are:
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_wmb();
WRITE_ONCE(*prod, p ^ 1);
}
}
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_store_release(prod, p ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (READ_ONCE(*prod) != c) {
smp_rmb();
d = READ_ONCE(*data);
smp_mb();
WRITE_ONCE(*cons, c ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (smp_load_acquire(prod) != c) {
d = READ_ONCE(*data);
smp_store_release(cons, c ^ 1);
}
}
The full LKMM litmus tests are found at [1].
On x86-64 systems the l2fwd AF_XDP xdpsock sample performance
increases by 1%. This is mostly due to that the smp_mb() is removed,
which is a relatively expensive operation on these
platforms. Weakly-ordered platforms, such as ARM64 might benefit even
more.
[1] https://github.com/bjoto/litmus-xsk
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://lore.kernel.org/bpf/20210305094113.413544-2-bjorn.topel@gmail.com
2021-03-05 12:41:12 +03:00
|
|
|
* room in the buffer to store $data we do not. The dependency will
|
|
|
|
* order both of the stores after the loads. So no barrier is needed.
|
2019-04-16 15:58:08 +03:00
|
|
|
*
|
|
|
|
* (D) protects the load of the data to be observed to happen after the
|
|
|
|
* store of the consumer pointer. If we did not have this memory
|
|
|
|
* barrier, the producer could observe the consumer pointer being set
|
|
|
|
* and overwrite the data with a new value before the consumer got the
|
|
|
|
* chance to read the old value. The consumer would thus miss reading
|
|
|
|
* the old entry and very likely read the new entry twice, once right
|
|
|
|
* now and again after circling through the ring.
|
|
|
|
*/
|
|
|
|
|
2019-12-19 15:39:30 +03:00
|
|
|
/* The operations on the rings are the following:
|
|
|
|
*
|
|
|
|
* producer consumer
|
|
|
|
*
|
|
|
|
* RESERVE entries PEEK in the ring for entries
|
|
|
|
* WRITE data into the ring READ data from the ring
|
|
|
|
* SUBMIT entries RELEASE entries
|
|
|
|
*
|
|
|
|
* The producer reserves one or more entries in the ring. It can then
|
|
|
|
* fill in these entries and finally submit them so that they can be
|
|
|
|
* seen and read by the consumer.
|
|
|
|
*
|
|
|
|
* The consumer peeks into the ring to see if the producer has written
|
2020-09-28 11:23:44 +03:00
|
|
|
* any new entries. If so, the consumer can then read these entries
|
2019-12-19 15:39:30 +03:00
|
|
|
* and when it is done reading them release them back to the producer
|
|
|
|
* so that the producer can use these slots to fill in new entries.
|
|
|
|
*
|
|
|
|
* The function names below reflect these operations.
|
|
|
|
*/
|
2019-06-26 17:35:24 +03:00
|
|
|
|
2019-12-19 15:39:30 +03:00
|
|
|
/* Functions that read and validate content from consumer rings. */
|
2018-05-02 14:01:27 +03:00
|
|
|
|
xsk: Batched buffer allocation for the pool
Add a new driver interface xsk_buff_alloc_batch() offering batched
buffer allocations to improve performance. The new interface takes
three arguments: the buffer pool to allocated from, a pointer to an
array of struct xdp_buff pointers which will contain pointers to the
allocated xdp_buffs, and an unsigned integer specifying the max number
of buffers to allocate. The return value is the actual number of
buffers that the allocator managed to allocate and it will be in the
range 0 <= N <= max, where max is the third parameter to the function.
u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp,
u32 max);
A second driver interface is also introduced that need to be used in
conjunction with xsk_buff_alloc_batch(). It is a helper that sets the
size of struct xdp_buff and is used by the NIC Rx irq routine when
receiving a packet. This helper sets the three struct members data,
data_meta, and data_end. The two first ones is in the xsk_buff_alloc()
case set in the allocation routine and data_end is set when a packet
is received in the receive irq function. This unfortunately leads to
worse performance since the xdp_buff is touched twice with a long time
period in between leading to an extra cache miss. Instead, we fill out
the xdp_buff with all 3 fields at one single point in time in the
driver, when the size of the packet is known. Hence this helper. Note
that the driver has to use this helper (or set all three fields
itself) when using xsk_buff_alloc_batch(). xsk_buff_alloc() works as
before and does not require this.
void xsk_buff_set_size(struct xdp_buff *xdp, u32 size);
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20210922075613.12186-3-magnus.karlsson@gmail.com
2021-09-22 10:56:02 +03:00
|
|
|
static inline void __xskq_cons_read_addr_unchecked(struct xsk_queue *q, u32 cached_cons, u64 *addr)
|
2020-05-20 22:20:53 +03:00
|
|
|
{
|
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
xsk: Batched buffer allocation for the pool
Add a new driver interface xsk_buff_alloc_batch() offering batched
buffer allocations to improve performance. The new interface takes
three arguments: the buffer pool to allocated from, a pointer to an
array of struct xdp_buff pointers which will contain pointers to the
allocated xdp_buffs, and an unsigned integer specifying the max number
of buffers to allocate. The return value is the actual number of
buffers that the allocator managed to allocate and it will be in the
range 0 <= N <= max, where max is the third parameter to the function.
u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp,
u32 max);
A second driver interface is also introduced that need to be used in
conjunction with xsk_buff_alloc_batch(). It is a helper that sets the
size of struct xdp_buff and is used by the NIC Rx irq routine when
receiving a packet. This helper sets the three struct members data,
data_meta, and data_end. The two first ones is in the xsk_buff_alloc()
case set in the allocation routine and data_end is set when a packet
is received in the receive irq function. This unfortunately leads to
worse performance since the xdp_buff is touched twice with a long time
period in between leading to an extra cache miss. Instead, we fill out
the xdp_buff with all 3 fields at one single point in time in the
driver, when the size of the packet is known. Hence this helper. Note
that the driver has to use this helper (or set all three fields
itself) when using xsk_buff_alloc_batch(). xsk_buff_alloc() works as
before and does not require this.
void xsk_buff_set_size(struct xdp_buff *xdp, u32 size);
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20210922075613.12186-3-magnus.karlsson@gmail.com
2021-09-22 10:56:02 +03:00
|
|
|
u32 idx = cached_cons & q->ring_mask;
|
2020-05-20 22:20:53 +03:00
|
|
|
|
xsk: Batched buffer allocation for the pool
Add a new driver interface xsk_buff_alloc_batch() offering batched
buffer allocations to improve performance. The new interface takes
three arguments: the buffer pool to allocated from, a pointer to an
array of struct xdp_buff pointers which will contain pointers to the
allocated xdp_buffs, and an unsigned integer specifying the max number
of buffers to allocate. The return value is the actual number of
buffers that the allocator managed to allocate and it will be in the
range 0 <= N <= max, where max is the third parameter to the function.
u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp,
u32 max);
A second driver interface is also introduced that need to be used in
conjunction with xsk_buff_alloc_batch(). It is a helper that sets the
size of struct xdp_buff and is used by the NIC Rx irq routine when
receiving a packet. This helper sets the three struct members data,
data_meta, and data_end. The two first ones is in the xsk_buff_alloc()
case set in the allocation routine and data_end is set when a packet
is received in the receive irq function. This unfortunately leads to
worse performance since the xdp_buff is touched twice with a long time
period in between leading to an extra cache miss. Instead, we fill out
the xdp_buff with all 3 fields at one single point in time in the
driver, when the size of the packet is known. Hence this helper. Note
that the driver has to use this helper (or set all three fields
itself) when using xsk_buff_alloc_batch(). xsk_buff_alloc() works as
before and does not require this.
void xsk_buff_set_size(struct xdp_buff *xdp, u32 size);
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20210922075613.12186-3-magnus.karlsson@gmail.com
2021-09-22 10:56:02 +03:00
|
|
|
*addr = ring->desc[idx];
|
|
|
|
}
|
2020-05-20 22:20:53 +03:00
|
|
|
|
xsk: Batched buffer allocation for the pool
Add a new driver interface xsk_buff_alloc_batch() offering batched
buffer allocations to improve performance. The new interface takes
three arguments: the buffer pool to allocated from, a pointer to an
array of struct xdp_buff pointers which will contain pointers to the
allocated xdp_buffs, and an unsigned integer specifying the max number
of buffers to allocate. The return value is the actual number of
buffers that the allocator managed to allocate and it will be in the
range 0 <= N <= max, where max is the third parameter to the function.
u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp,
u32 max);
A second driver interface is also introduced that need to be used in
conjunction with xsk_buff_alloc_batch(). It is a helper that sets the
size of struct xdp_buff and is used by the NIC Rx irq routine when
receiving a packet. This helper sets the three struct members data,
data_meta, and data_end. The two first ones is in the xsk_buff_alloc()
case set in the allocation routine and data_end is set when a packet
is received in the receive irq function. This unfortunately leads to
worse performance since the xdp_buff is touched twice with a long time
period in between leading to an extra cache miss. Instead, we fill out
the xdp_buff with all 3 fields at one single point in time in the
driver, when the size of the packet is known. Hence this helper. Note
that the driver has to use this helper (or set all three fields
itself) when using xsk_buff_alloc_batch(). xsk_buff_alloc() works as
before and does not require this.
void xsk_buff_set_size(struct xdp_buff *xdp, u32 size);
Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20210922075613.12186-3-magnus.karlsson@gmail.com
2021-09-22 10:56:02 +03:00
|
|
|
static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue *q, u64 *addr)
|
|
|
|
{
|
|
|
|
if (q->cached_cons != q->cached_prod) {
|
|
|
|
__xskq_cons_read_addr_unchecked(q, q->cached_cons, addr);
|
xsk: add support to allow unaligned chunk placement
Currently, addresses are chunk size aligned. This means, we are very
restricted in terms of where we can place chunk within the umem. For
example, if we have a chunk size of 2k, then our chunks can only be placed
at 0,2k,4k,6k,8k... and so on (ie. every 2k starting from 0).
This patch introduces the ability to use unaligned chunks. With these
changes, we are no longer bound to having to place chunks at a 2k (or
whatever your chunk size is) interval. Since we are no longer dealing with
aligned chunks, they can now cross page boundaries. Checks for page
contiguity have been added in order to keep track of which pages are
followed by a physically contiguous page.
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Ciara Loftus <ciara.loftus@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2019-08-27 05:25:22 +03:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2020-05-20 22:20:53 +03:00
|
|
|
return false;
|
|
|
|
}
|
2018-05-02 14:01:34 +03:00
|
|
|
|
2020-05-20 22:21:02 +03:00
|
|
|
static inline bool xp_aligned_validate_desc(struct xsk_buff_pool *pool,
|
|
|
|
struct xdp_desc *desc)
|
|
|
|
{
|
2021-06-18 10:58:05 +03:00
|
|
|
u64 chunk, chunk_end;
|
2020-05-20 22:21:02 +03:00
|
|
|
|
2021-04-28 12:44:24 +03:00
|
|
|
chunk = xp_aligned_extract_addr(pool, desc->addr);
|
2021-06-18 10:58:05 +03:00
|
|
|
if (likely(desc->len)) {
|
|
|
|
chunk_end = xp_aligned_extract_addr(pool, desc->addr + desc->len - 1);
|
|
|
|
if (chunk != chunk_end)
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2020-05-20 22:21:02 +03:00
|
|
|
if (chunk >= pool->addrs_cnt)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (desc->options)
|
|
|
|
return false;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool xp_unaligned_validate_desc(struct xsk_buff_pool *pool,
|
|
|
|
struct xdp_desc *desc)
|
|
|
|
{
|
|
|
|
u64 addr, base_addr;
|
|
|
|
|
|
|
|
base_addr = xp_unaligned_extract_addr(desc->addr);
|
|
|
|
addr = xp_unaligned_add_offset_to_addr(desc->addr);
|
|
|
|
|
|
|
|
if (desc->len > pool->chunk_size)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (base_addr >= pool->addrs_cnt || addr >= pool->addrs_cnt ||
|
|
|
|
xp_desc_crosses_non_contig_pg(pool, addr, desc->len))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (desc->options)
|
|
|
|
return false;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool xp_validate_desc(struct xsk_buff_pool *pool,
|
|
|
|
struct xdp_desc *desc)
|
|
|
|
{
|
|
|
|
return pool->unaligned ? xp_unaligned_validate_desc(pool, desc) :
|
|
|
|
xp_aligned_validate_desc(pool, desc);
|
|
|
|
}
|
|
|
|
|
2020-05-20 22:20:53 +03:00
|
|
|
static inline bool xskq_cons_is_valid_desc(struct xsk_queue *q,
|
|
|
|
struct xdp_desc *d,
|
2020-08-28 11:26:17 +03:00
|
|
|
struct xsk_buff_pool *pool)
|
2020-05-20 22:20:53 +03:00
|
|
|
{
|
2020-08-28 11:26:17 +03:00
|
|
|
if (!xp_validate_desc(pool, d)) {
|
2018-05-02 14:01:34 +03:00
|
|
|
q->invalid_descs++;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2019-12-19 15:39:27 +03:00
|
|
|
static inline bool xskq_cons_read_desc(struct xsk_queue *q,
|
|
|
|
struct xdp_desc *desc,
|
2020-08-28 11:26:17 +03:00
|
|
|
struct xsk_buff_pool *pool)
|
2018-05-02 14:01:34 +03:00
|
|
|
{
|
2019-12-19 15:39:26 +03:00
|
|
|
while (q->cached_cons != q->cached_prod) {
|
2018-05-02 14:01:34 +03:00
|
|
|
struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
|
2019-12-19 15:39:26 +03:00
|
|
|
u32 idx = q->cached_cons & q->ring_mask;
|
2018-05-02 14:01:34 +03:00
|
|
|
|
2019-12-19 15:39:29 +03:00
|
|
|
*desc = ring->desc[idx];
|
2020-08-28 11:26:17 +03:00
|
|
|
if (xskq_cons_is_valid_desc(q, desc, pool))
|
2019-12-19 15:39:27 +03:00
|
|
|
return true;
|
2018-05-02 14:01:34 +03:00
|
|
|
|
2019-12-19 15:39:26 +03:00
|
|
|
q->cached_cons++;
|
2018-05-02 14:01:34 +03:00
|
|
|
}
|
|
|
|
|
2019-12-19 15:39:27 +03:00
|
|
|
return false;
|
2018-05-02 14:01:34 +03:00
|
|
|
}
|
|
|
|
|
2022-08-30 15:17:05 +03:00
|
|
|
static inline void xskq_cons_release_n(struct xsk_queue *q, u32 cnt)
|
|
|
|
{
|
|
|
|
q->cached_cons += cnt;
|
|
|
|
}
|
|
|
|
|
2022-01-25 19:04:43 +03:00
|
|
|
static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool,
|
|
|
|
u32 max)
|
2020-11-16 14:12:46 +03:00
|
|
|
{
|
|
|
|
u32 cached_cons = q->cached_cons, nb_entries = 0;
|
2022-01-25 19:04:43 +03:00
|
|
|
struct xdp_desc *descs = pool->tx_descs;
|
2020-11-16 14:12:46 +03:00
|
|
|
|
|
|
|
while (cached_cons != q->cached_prod && nb_entries < max) {
|
|
|
|
struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
|
|
|
|
u32 idx = cached_cons & q->ring_mask;
|
|
|
|
|
|
|
|
descs[nb_entries] = ring->desc[idx];
|
|
|
|
if (unlikely(!xskq_cons_is_valid_desc(q, &descs[nb_entries], pool))) {
|
|
|
|
/* Skip the entry */
|
|
|
|
cached_cons++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
nb_entries++;
|
|
|
|
cached_cons++;
|
|
|
|
}
|
|
|
|
|
2022-08-30 15:17:05 +03:00
|
|
|
/* Release valid plus any invalid entries */
|
|
|
|
xskq_cons_release_n(q, cached_cons - q->cached_cons);
|
2020-11-16 14:12:46 +03:00
|
|
|
return nb_entries;
|
|
|
|
}
|
|
|
|
|
2019-12-19 15:39:30 +03:00
|
|
|
/* Functions for consumers */
|
|
|
|
|
|
|
|
static inline void __xskq_cons_release(struct xsk_queue *q)
|
|
|
|
{
|
xsk: Update rings for load-acquire/store-release barriers
Currently, the AF_XDP rings uses general smp_{r,w,}mb() barriers on
the kernel-side. On most modern architectures
load-acquire/store-release barriers perform better, and results in
simpler code for circular ring buffers.
This change updates the XDP socket rings to use
load-acquire/store-release barriers.
It is important to note that changing from the old smp_{r,w,}mb()
barriers, to load-acquire/store-release barriers does not break
compatibility. The old semantics work with the new one, and vice
versa.
As pointed out by "Documentation/memory-barriers.txt" in the "SMP
BARRIER PAIRING" section:
"General barriers pair with each other, though they also pair with
most other types of barriers, albeit without multicopy atomicity.
An acquire barrier pairs with a release barrier, but both may also
pair with other barriers, including of course general barriers."
How different barriers behaves and pairs is outlined in
"tools/memory-model/Documentation/cheatsheet.txt".
In order to make sure that compatibility is not broken, LKMM herd7
based litmus tests can be constructed and verified.
We generalize the XDP socket ring to a one entry ring, and create two
scenarios; One where the ring is full, where only the consumer can
proceed, followed by the producer. One where the ring is empty, where
only the producer can proceed, followed by the consumer. Each scenario
is then expanded to four different tests: general producer/general
consumer, general producer/acqrel consumer, acqrel producer/general
consumer, acqrel producer/acqrel consumer. In total eight tests.
The empty ring test:
C spsc-rb+empty
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{}
// We start at prod==0, cons==0, data==0, i.e. nothing has been
// written to the ring. From here only the producer can start, and
// should write 1. Afterwards, consumer can continue and read 1 to
// data. Can we enter state prod==1, cons==1, but consumer observed
// the incorrect value of 0?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=0 /\ prod=1 /\ cons=1 );
The full ring test:
C spsc-rb+full
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{ prod = 1; }
// We start at prod==1, cons==0, data==1, i.e. producer has
// written 0, so from here only the consumer can start, and should
// consume 0. Afterwards, producer can continue and write 1 to
// data. Can we enter state prod==0, cons==1, but consumer observed
// the write of 1?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=1 /\ prod=0 /\ cons=1 );
where P0 and P1 are:
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_wmb();
WRITE_ONCE(*prod, p ^ 1);
}
}
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_store_release(prod, p ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (READ_ONCE(*prod) != c) {
smp_rmb();
d = READ_ONCE(*data);
smp_mb();
WRITE_ONCE(*cons, c ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (smp_load_acquire(prod) != c) {
d = READ_ONCE(*data);
smp_store_release(cons, c ^ 1);
}
}
The full LKMM litmus tests are found at [1].
On x86-64 systems the l2fwd AF_XDP xdpsock sample performance
increases by 1%. This is mostly due to that the smp_mb() is removed,
which is a relatively expensive operation on these
platforms. Weakly-ordered platforms, such as ARM64 might benefit even
more.
[1] https://github.com/bjoto/litmus-xsk
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://lore.kernel.org/bpf/20210305094113.413544-2-bjorn.topel@gmail.com
2021-03-05 12:41:12 +03:00
|
|
|
smp_store_release(&q->ring->consumer, q->cached_cons); /* D, matchees A */
|
2019-12-19 15:39:30 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void __xskq_cons_peek(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
/* Refresh the local pointer */
|
xsk: Update rings for load-acquire/store-release barriers
Currently, the AF_XDP rings uses general smp_{r,w,}mb() barriers on
the kernel-side. On most modern architectures
load-acquire/store-release barriers perform better, and results in
simpler code for circular ring buffers.
This change updates the XDP socket rings to use
load-acquire/store-release barriers.
It is important to note that changing from the old smp_{r,w,}mb()
barriers, to load-acquire/store-release barriers does not break
compatibility. The old semantics work with the new one, and vice
versa.
As pointed out by "Documentation/memory-barriers.txt" in the "SMP
BARRIER PAIRING" section:
"General barriers pair with each other, though they also pair with
most other types of barriers, albeit without multicopy atomicity.
An acquire barrier pairs with a release barrier, but both may also
pair with other barriers, including of course general barriers."
How different barriers behaves and pairs is outlined in
"tools/memory-model/Documentation/cheatsheet.txt".
In order to make sure that compatibility is not broken, LKMM herd7
based litmus tests can be constructed and verified.
We generalize the XDP socket ring to a one entry ring, and create two
scenarios; One where the ring is full, where only the consumer can
proceed, followed by the producer. One where the ring is empty, where
only the producer can proceed, followed by the consumer. Each scenario
is then expanded to four different tests: general producer/general
consumer, general producer/acqrel consumer, acqrel producer/general
consumer, acqrel producer/acqrel consumer. In total eight tests.
The empty ring test:
C spsc-rb+empty
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{}
// We start at prod==0, cons==0, data==0, i.e. nothing has been
// written to the ring. From here only the producer can start, and
// should write 1. Afterwards, consumer can continue and read 1 to
// data. Can we enter state prod==1, cons==1, but consumer observed
// the incorrect value of 0?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=0 /\ prod=1 /\ cons=1 );
The full ring test:
C spsc-rb+full
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{ prod = 1; }
// We start at prod==1, cons==0, data==1, i.e. producer has
// written 0, so from here only the consumer can start, and should
// consume 0. Afterwards, producer can continue and write 1 to
// data. Can we enter state prod==0, cons==1, but consumer observed
// the write of 1?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=1 /\ prod=0 /\ cons=1 );
where P0 and P1 are:
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_wmb();
WRITE_ONCE(*prod, p ^ 1);
}
}
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_store_release(prod, p ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (READ_ONCE(*prod) != c) {
smp_rmb();
d = READ_ONCE(*data);
smp_mb();
WRITE_ONCE(*cons, c ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (smp_load_acquire(prod) != c) {
d = READ_ONCE(*data);
smp_store_release(cons, c ^ 1);
}
}
The full LKMM litmus tests are found at [1].
On x86-64 systems the l2fwd AF_XDP xdpsock sample performance
increases by 1%. This is mostly due to that the smp_mb() is removed,
which is a relatively expensive operation on these
platforms. Weakly-ordered platforms, such as ARM64 might benefit even
more.
[1] https://github.com/bjoto/litmus-xsk
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://lore.kernel.org/bpf/20210305094113.413544-2-bjorn.topel@gmail.com
2021-03-05 12:41:12 +03:00
|
|
|
q->cached_prod = smp_load_acquire(&q->ring->producer); /* C, matches B */
|
2019-12-19 15:39:30 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_cons_get_entries(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
__xskq_cons_release(q);
|
|
|
|
__xskq_cons_peek(q);
|
|
|
|
}
|
|
|
|
|
2020-11-16 14:12:46 +03:00
|
|
|
static inline u32 xskq_cons_nb_entries(struct xsk_queue *q, u32 max)
|
2019-12-19 15:39:30 +03:00
|
|
|
{
|
|
|
|
u32 entries = q->cached_prod - q->cached_cons;
|
|
|
|
|
2020-11-16 14:12:46 +03:00
|
|
|
if (entries >= max)
|
|
|
|
return max;
|
2019-12-19 15:39:30 +03:00
|
|
|
|
|
|
|
__xskq_cons_peek(q);
|
|
|
|
entries = q->cached_prod - q->cached_cons;
|
|
|
|
|
2020-11-16 14:12:46 +03:00
|
|
|
return entries >= max ? max : entries;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool xskq_cons_has_entries(struct xsk_queue *q, u32 cnt)
|
|
|
|
{
|
2022-04-13 18:30:15 +03:00
|
|
|
return xskq_cons_nb_entries(q, cnt) >= cnt;
|
2019-12-19 15:39:30 +03:00
|
|
|
}
|
|
|
|
|
2020-05-20 22:20:53 +03:00
|
|
|
static inline bool xskq_cons_peek_addr_unchecked(struct xsk_queue *q, u64 *addr)
|
|
|
|
{
|
|
|
|
if (q->cached_prod == q->cached_cons)
|
|
|
|
xskq_cons_get_entries(q);
|
|
|
|
return xskq_cons_read_addr_unchecked(q, addr);
|
|
|
|
}
|
|
|
|
|
2019-12-19 15:39:27 +03:00
|
|
|
static inline bool xskq_cons_peek_desc(struct xsk_queue *q,
|
|
|
|
struct xdp_desc *desc,
|
2020-08-28 11:26:17 +03:00
|
|
|
struct xsk_buff_pool *pool)
|
2018-05-02 14:01:34 +03:00
|
|
|
{
|
2019-12-19 15:39:26 +03:00
|
|
|
if (q->cached_prod == q->cached_cons)
|
|
|
|
xskq_cons_get_entries(q);
|
2020-08-28 11:26:17 +03:00
|
|
|
return xskq_cons_read_desc(q, desc, pool);
|
2018-05-02 14:01:34 +03:00
|
|
|
}
|
|
|
|
|
2020-11-16 14:12:46 +03:00
|
|
|
/* To improve performance in the xskq_cons_release functions, only update local state here.
|
|
|
|
* Reflect this to global state when we get new entries from the ring in
|
|
|
|
* xskq_cons_get_entries() and whenever Rx or Tx processing are completed in the NAPI loop.
|
|
|
|
*/
|
2019-12-19 15:39:30 +03:00
|
|
|
static inline void xskq_cons_release(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
q->cached_cons++;
|
|
|
|
}
|
|
|
|
|
2020-12-01 16:56:58 +03:00
|
|
|
static inline u32 xskq_cons_present_entries(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
/* No barriers needed since data is not accessed */
|
|
|
|
return READ_ONCE(q->ring->producer) - READ_ONCE(q->ring->consumer);
|
|
|
|
}
|
|
|
|
|
2019-12-19 15:39:30 +03:00
|
|
|
/* Functions for producers */
|
|
|
|
|
2020-11-16 14:12:46 +03:00
|
|
|
static inline u32 xskq_prod_nb_free(struct xsk_queue *q, u32 max)
|
2019-12-19 15:39:30 +03:00
|
|
|
{
|
|
|
|
u32 free_entries = q->nentries - (q->cached_prod - q->cached_cons);
|
|
|
|
|
2020-11-16 14:12:46 +03:00
|
|
|
if (free_entries >= max)
|
|
|
|
return max;
|
2019-12-19 15:39:30 +03:00
|
|
|
|
|
|
|
/* Refresh the local tail pointer */
|
|
|
|
q->cached_cons = READ_ONCE(q->ring->consumer);
|
|
|
|
free_entries = q->nentries - (q->cached_prod - q->cached_cons);
|
|
|
|
|
2020-11-16 14:12:46 +03:00
|
|
|
return free_entries >= max ? max : free_entries;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool xskq_prod_is_full(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
return xskq_prod_nb_free(q, 1) ? false : true;
|
2019-12-19 15:39:30 +03:00
|
|
|
}
|
|
|
|
|
2020-12-18 16:45:25 +03:00
|
|
|
static inline void xskq_prod_cancel(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
q->cached_prod--;
|
|
|
|
}
|
|
|
|
|
2019-12-19 15:39:30 +03:00
|
|
|
static inline int xskq_prod_reserve(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
if (xskq_prod_is_full(q))
|
|
|
|
return -ENOSPC;
|
|
|
|
|
|
|
|
/* A, matches D */
|
|
|
|
q->cached_prod++;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int xskq_prod_reserve_addr(struct xsk_queue *q, u64 addr)
|
|
|
|
{
|
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
|
|
|
|
|
|
|
if (xskq_prod_is_full(q))
|
|
|
|
return -ENOSPC;
|
|
|
|
|
|
|
|
/* A, matches D */
|
|
|
|
ring->desc[q->cached_prod++ & q->ring_mask] = addr;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2022-08-30 15:17:05 +03:00
|
|
|
static inline void xskq_prod_write_addr_batch(struct xsk_queue *q, struct xdp_desc *descs,
|
|
|
|
u32 nb_entries)
|
2020-11-16 14:12:46 +03:00
|
|
|
{
|
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
2022-08-30 15:17:05 +03:00
|
|
|
u32 i, cached_prod;
|
2020-11-16 14:12:46 +03:00
|
|
|
|
|
|
|
/* A, matches D */
|
|
|
|
cached_prod = q->cached_prod;
|
|
|
|
for (i = 0; i < nb_entries; i++)
|
|
|
|
ring->desc[cached_prod++ & q->ring_mask] = descs[i].addr;
|
|
|
|
q->cached_prod = cached_prod;
|
|
|
|
}
|
|
|
|
|
2019-12-19 15:39:23 +03:00
|
|
|
static inline int xskq_prod_reserve_desc(struct xsk_queue *q,
|
|
|
|
u64 addr, u32 len)
|
2018-05-02 14:01:27 +03:00
|
|
|
{
|
|
|
|
struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
|
2019-12-19 15:39:23 +03:00
|
|
|
u32 idx;
|
2018-05-02 14:01:27 +03:00
|
|
|
|
2019-12-19 15:39:25 +03:00
|
|
|
if (xskq_prod_is_full(q))
|
2022-04-13 18:30:02 +03:00
|
|
|
return -ENOBUFS;
|
2018-05-02 14:01:27 +03:00
|
|
|
|
2019-04-16 15:58:08 +03:00
|
|
|
/* A, matches D */
|
2019-12-19 15:39:22 +03:00
|
|
|
idx = q->cached_prod++ & q->ring_mask;
|
xsk: new descriptor addressing scheme
Currently, AF_XDP only supports a fixed frame-size memory scheme where
each frame is referenced via an index (idx). A user passes the frame
index to the kernel, and the kernel acts upon the data. Some NICs,
however, do not have a fixed frame-size model, instead they have a
model where a memory window is passed to the hardware and multiple
frames are filled into that window (referred to as the "type-writer"
model).
By changing the descriptor format from the current frame index
addressing scheme, AF_XDP can in the future be extended to support
these kinds of NICs.
In the index-based model, an idx refers to a frame of size
frame_size. Addressing a frame in the UMEM is done by offseting the
UMEM starting address by a global offset, idx * frame_size + offset.
Communicating via the fill- and completion-rings are done by means of
idx.
In this commit, the idx is removed in favor of an address (addr),
which is a relative address ranging over the UMEM. To convert an
idx-based address to the new addr is simply: addr = idx * frame_size +
offset.
We also stop referring to the UMEM "frame" as a frame. Instead it is
simply called a chunk.
To transfer ownership of a chunk to the kernel, the addr of the chunk
is passed in the fill-ring. Note, that the kernel will mask addr to
make it chunk aligned, so there is no need for userspace to do
that. E.g., for a chunk size of 2k, passing an addr of 2048, 2050 or
3000 to the fill-ring will refer to the same chunk.
On the completion-ring, the addr will match that of the Tx descriptor,
passed to the kernel.
Changing the descriptor format to use chunks/addr will allow for
future changes to move to a type-writer based model, where multiple
frames can reside in one chunk. In this model passing one single chunk
into the fill-ring, would potentially result in multiple Rx
descriptors.
This commit changes the uapi of AF_XDP sockets, and updates the
documentation.
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-06-04 14:57:13 +03:00
|
|
|
ring->desc[idx].addr = addr;
|
2018-05-02 14:01:27 +03:00
|
|
|
ring->desc[idx].len = len;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2019-12-19 15:39:30 +03:00
|
|
|
static inline void __xskq_prod_submit(struct xsk_queue *q, u32 idx)
|
2018-05-02 14:01:34 +03:00
|
|
|
{
|
xsk: Update rings for load-acquire/store-release barriers
Currently, the AF_XDP rings uses general smp_{r,w,}mb() barriers on
the kernel-side. On most modern architectures
load-acquire/store-release barriers perform better, and results in
simpler code for circular ring buffers.
This change updates the XDP socket rings to use
load-acquire/store-release barriers.
It is important to note that changing from the old smp_{r,w,}mb()
barriers, to load-acquire/store-release barriers does not break
compatibility. The old semantics work with the new one, and vice
versa.
As pointed out by "Documentation/memory-barriers.txt" in the "SMP
BARRIER PAIRING" section:
"General barriers pair with each other, though they also pair with
most other types of barriers, albeit without multicopy atomicity.
An acquire barrier pairs with a release barrier, but both may also
pair with other barriers, including of course general barriers."
How different barriers behaves and pairs is outlined in
"tools/memory-model/Documentation/cheatsheet.txt".
In order to make sure that compatibility is not broken, LKMM herd7
based litmus tests can be constructed and verified.
We generalize the XDP socket ring to a one entry ring, and create two
scenarios; One where the ring is full, where only the consumer can
proceed, followed by the producer. One where the ring is empty, where
only the producer can proceed, followed by the consumer. Each scenario
is then expanded to four different tests: general producer/general
consumer, general producer/acqrel consumer, acqrel producer/general
consumer, acqrel producer/acqrel consumer. In total eight tests.
The empty ring test:
C spsc-rb+empty
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{}
// We start at prod==0, cons==0, data==0, i.e. nothing has been
// written to the ring. From here only the producer can start, and
// should write 1. Afterwards, consumer can continue and read 1 to
// data. Can we enter state prod==1, cons==1, but consumer observed
// the incorrect value of 0?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=0 /\ prod=1 /\ cons=1 );
The full ring test:
C spsc-rb+full
// Simple one entry ring:
// prod cons allowed action prod cons
// 0 0 => prod => 1 0
// 0 1 => cons => 0 0
// 1 0 => cons => 1 1
// 1 1 => prod => 0 1
{ prod = 1; }
// We start at prod==1, cons==0, data==1, i.e. producer has
// written 0, so from here only the consumer can start, and should
// consume 0. Afterwards, producer can continue and write 1 to
// data. Can we enter state prod==0, cons==1, but consumer observed
// the write of 1?
P0(int *prod, int *cons, int *data)
{
... producer
}
P1(int *prod, int *cons, int *data)
{
... consumer
}
exists( 1:d=1 /\ prod=0 /\ cons=1 );
where P0 and P1 are:
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_wmb();
WRITE_ONCE(*prod, p ^ 1);
}
}
P0(int *prod, int *cons, int *data)
{
int p;
p = READ_ONCE(*prod);
if (READ_ONCE(*cons) == p) {
WRITE_ONCE(*data, 1);
smp_store_release(prod, p ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (READ_ONCE(*prod) != c) {
smp_rmb();
d = READ_ONCE(*data);
smp_mb();
WRITE_ONCE(*cons, c ^ 1);
}
}
P1(int *prod, int *cons, int *data)
{
int c;
int d = -1;
c = READ_ONCE(*cons);
if (smp_load_acquire(prod) != c) {
d = READ_ONCE(*data);
smp_store_release(cons, c ^ 1);
}
}
The full LKMM litmus tests are found at [1].
On x86-64 systems the l2fwd AF_XDP xdpsock sample performance
increases by 1%. This is mostly due to that the smp_mb() is removed,
which is a relatively expensive operation on these
platforms. Weakly-ordered platforms, such as ARM64 might benefit even
more.
[1] https://github.com/bjoto/litmus-xsk
Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://lore.kernel.org/bpf/20210305094113.413544-2-bjorn.topel@gmail.com
2021-03-05 12:41:12 +03:00
|
|
|
smp_store_release(&q->ring->producer, idx); /* B, matches C */
|
2019-12-19 15:39:30 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_prod_submit(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
__xskq_prod_submit(q, q->cached_prod);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_prod_submit_addr(struct xsk_queue *q, u64 addr)
|
|
|
|
{
|
|
|
|
struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
|
|
|
|
u32 idx = q->ring->producer;
|
|
|
|
|
|
|
|
ring->desc[idx++ & q->ring_mask] = addr;
|
|
|
|
|
|
|
|
__xskq_prod_submit(q, idx);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void xskq_prod_submit_n(struct xsk_queue *q, u32 nb_entries)
|
|
|
|
{
|
|
|
|
__xskq_prod_submit(q, q->ring->producer + nb_entries);
|
2018-05-02 14:01:34 +03:00
|
|
|
}
|
|
|
|
|
2019-12-19 15:39:23 +03:00
|
|
|
static inline bool xskq_prod_is_empty(struct xsk_queue *q)
|
2018-05-02 14:01:27 +03:00
|
|
|
{
|
2019-12-19 15:39:21 +03:00
|
|
|
/* No barriers needed since data is not accessed */
|
|
|
|
return READ_ONCE(q->ring->consumer) == READ_ONCE(q->ring->producer);
|
2018-05-02 14:01:27 +03:00
|
|
|
}
|
|
|
|
|
2019-12-19 15:39:30 +03:00
|
|
|
/* For both producers and consumers */
|
|
|
|
|
|
|
|
static inline u64 xskq_nb_invalid_descs(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
return q ? q->invalid_descs : 0;
|
|
|
|
}
|
|
|
|
|
2020-07-08 10:28:33 +03:00
|
|
|
static inline u64 xskq_nb_queue_empty_descs(struct xsk_queue *q)
|
|
|
|
{
|
|
|
|
return q ? q->queue_empty_descs : 0;
|
|
|
|
}
|
|
|
|
|
2018-05-02 14:01:25 +03:00
|
|
|
struct xsk_queue *xskq_create(u32 nentries, bool umem_queue);
|
2018-05-02 14:01:27 +03:00
|
|
|
void xskq_destroy(struct xsk_queue *q_ops);
|
2018-05-02 14:01:24 +03:00
|
|
|
|
|
|
|
#endif /* _LINUX_XSK_QUEUE_H */
|