xsk: Propagate napi_id to XDP socket Rx path

Add napi_id to the xdp_rxq_info structure, and make sure the XDP
socket pick up the napi_id in the Rx path. The napi_id is used to find
the corresponding NAPI structure for socket busy polling.

Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Acked-by: Tariq Toukan <tariqt@nvidia.com>
Link: https://lore.kernel.org/bpf/20201130185205.196029-7-bjorn.topel@gmail.com
This commit is contained in:
Björn Töpel 2020-11-30 19:52:01 +01:00 коммит произвёл Daniel Borkmann
Родитель a0731952d9
Коммит b02e5a0ebb
29 изменённых файлов: 54 добавлений и 36 удалений

Просмотреть файл

@ -416,7 +416,7 @@ static int ena_xdp_register_rxq_info(struct ena_ring *rx_ring)
{ {
int rc; int rc;
rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid); rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0);
if (rc) { if (rc) {
netif_err(rx_ring->adapter, ifup, rx_ring->netdev, netif_err(rx_ring->adapter, ifup, rx_ring->netdev,

Просмотреть файл

@ -2884,7 +2884,7 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp)
if (rc) if (rc)
return rc; return rc;
rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
if (rc < 0) if (rc < 0)
return rc; return rc;

Просмотреть файл

@ -770,7 +770,7 @@ static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs,
rq->caching = 1; rq->caching = 1;
/* Driver have no proper error path for failed XDP RX-queue info reg */ /* Driver have no proper error path for failed XDP RX-queue info reg */
WARN_ON(xdp_rxq_info_reg(&rq->xdp_rxq, nic->netdev, qidx) < 0); WARN_ON(xdp_rxq_info_reg(&rq->xdp_rxq, nic->netdev, qidx, 0) < 0);
/* Send a mailbox msg to PF to config RQ */ /* Send a mailbox msg to PF to config RQ */
mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG; mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG;

Просмотреть файл

@ -3334,7 +3334,7 @@ static int dpaa2_eth_setup_rx_flow(struct dpaa2_eth_priv *priv,
return 0; return 0;
err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev, err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev,
fq->flowid); fq->flowid, 0);
if (err) { if (err) {
dev_err(dev, "xdp_rxq_info_reg failed\n"); dev_err(dev, "xdp_rxq_info_reg failed\n");
return err; return err;

Просмотреть файл

@ -1447,7 +1447,7 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
/* XDP RX-queue info only needed for RX rings exposed to XDP */ /* XDP RX-queue info only needed for RX rings exposed to XDP */
if (rx_ring->vsi->type == I40E_VSI_MAIN) { if (rx_ring->vsi->type == I40E_VSI_MAIN) {
err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
rx_ring->queue_index); rx_ring->queue_index, rx_ring->q_vector->napi.napi_id);
if (err < 0) if (err < 0)
return err; return err;
} }

Просмотреть файл

@ -306,7 +306,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring)
if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) if (!xdp_rxq_info_is_reg(&ring->xdp_rxq))
/* coverity[check_return] */ /* coverity[check_return] */
xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
ring->q_index); ring->q_index, ring->q_vector->napi.napi_id);
ring->xsk_pool = ice_xsk_pool(ring); ring->xsk_pool = ice_xsk_pool(ring);
if (ring->xsk_pool) { if (ring->xsk_pool) {
@ -333,7 +333,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring)
/* coverity[check_return] */ /* coverity[check_return] */
xdp_rxq_info_reg(&ring->xdp_rxq, xdp_rxq_info_reg(&ring->xdp_rxq,
ring->netdev, ring->netdev,
ring->q_index); ring->q_index, ring->q_vector->napi.napi_id);
err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
MEM_TYPE_PAGE_SHARED, MEM_TYPE_PAGE_SHARED,

Просмотреть файл

@ -483,7 +483,7 @@ int ice_setup_rx_ring(struct ice_ring *rx_ring)
if (rx_ring->vsi->type == ICE_VSI_PF && if (rx_ring->vsi->type == ICE_VSI_PF &&
!xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) !xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
rx_ring->q_index)) rx_ring->q_index, rx_ring->q_vector->napi.napi_id))
goto err; goto err;
return 0; return 0;

Просмотреть файл

@ -4352,7 +4352,7 @@ int igb_setup_rx_resources(struct igb_ring *rx_ring)
/* XDP RX-queue info */ /* XDP RX-queue info */
if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
rx_ring->queue_index) < 0) rx_ring->queue_index, 0) < 0)
goto err; goto err;
return 0; return 0;

Просмотреть файл

@ -6577,7 +6577,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter,
/* XDP RX-queue info */ /* XDP RX-queue info */
if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev,
rx_ring->queue_index) < 0) rx_ring->queue_index, rx_ring->q_vector->napi.napi_id) < 0)
goto err; goto err;
rx_ring->xdp_prog = adapter->xdp_prog; rx_ring->xdp_prog = adapter->xdp_prog;

Просмотреть файл

@ -3493,7 +3493,7 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
/* XDP RX-queue info */ /* XDP RX-queue info */
if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev,
rx_ring->queue_index) < 0) rx_ring->queue_index, 0) < 0)
goto err; goto err;
rx_ring->xdp_prog = adapter->xdp_prog; rx_ring->xdp_prog = adapter->xdp_prog;

Просмотреть файл

@ -3227,7 +3227,7 @@ static int mvneta_create_page_pool(struct mvneta_port *pp,
return err; return err;
} }
err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id); err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id, 0);
if (err < 0) if (err < 0)
goto err_free_pp; goto err_free_pp;

Просмотреть файл

@ -2614,11 +2614,11 @@ static int mvpp2_rxq_init(struct mvpp2_port *port,
mvpp2_rxq_status_update(port, rxq->id, 0, rxq->size); mvpp2_rxq_status_update(port, rxq->id, 0, rxq->size);
if (priv->percpu_pools) { if (priv->percpu_pools) {
err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->id); err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->id, 0);
if (err < 0) if (err < 0)
goto err_free_dma; goto err_free_dma;
err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->id); err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->id, 0);
if (err < 0) if (err < 0)
goto err_unregister_rxq_short; goto err_unregister_rxq_short;

Просмотреть файл

@ -283,7 +283,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
ring->log_stride = ffs(ring->stride) - 1; ring->log_stride = ffs(ring->stride) - 1;
ring->buf_size = ring->size * ring->stride + TXBB_SIZE; ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index) < 0) if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index, 0) < 0)
goto err_ring; goto err_ring;
tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS * tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *

Просмотреть файл

@ -434,7 +434,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
rq_xdp_ix = rq->ix; rq_xdp_ix = rq->ix;
if (xsk) if (xsk)
rq_xdp_ix += params->num_channels * MLX5E_RQ_GROUP_XSK; rq_xdp_ix += params->num_channels * MLX5E_RQ_GROUP_XSK;
err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix); err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, 0);
if (err < 0) if (err < 0)
goto err_rq_xdp_prog; goto err_rq_xdp_prog;

Просмотреть файл

@ -2533,7 +2533,7 @@ nfp_net_rx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring)
if (dp->netdev) { if (dp->netdev) {
err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, dp->netdev, err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, dp->netdev,
rx_ring->idx); rx_ring->idx, rx_ring->r_vec->napi.napi_id);
if (err < 0) if (err < 0)
return err; return err;
} }

Просмотреть файл

@ -1762,7 +1762,7 @@ static void qede_init_fp(struct qede_dev *edev)
/* Driver have no error path from here */ /* Driver have no error path from here */
WARN_ON(xdp_rxq_info_reg(&fp->rxq->xdp_rxq, edev->ndev, WARN_ON(xdp_rxq_info_reg(&fp->rxq->xdp_rxq, edev->ndev,
fp->rxq->rxq_id) < 0); fp->rxq->rxq_id, 0) < 0);
if (xdp_rxq_info_reg_mem_model(&fp->rxq->xdp_rxq, if (xdp_rxq_info_reg_mem_model(&fp->rxq->xdp_rxq,
MEM_TYPE_PAGE_ORDER0, MEM_TYPE_PAGE_ORDER0,

Просмотреть файл

@ -262,7 +262,7 @@ void efx_init_rx_queue(struct efx_rx_queue *rx_queue)
/* Initialise XDP queue information */ /* Initialise XDP queue information */
rc = xdp_rxq_info_reg(&rx_queue->xdp_rxq_info, efx->net_dev, rc = xdp_rxq_info_reg(&rx_queue->xdp_rxq_info, efx->net_dev,
rx_queue->core_index); rx_queue->core_index, 0);
if (rc) { if (rc) {
netif_err(efx, rx_err, efx->net_dev, netif_err(efx, rx_err, efx->net_dev,

Просмотреть файл

@ -1304,7 +1304,7 @@ static int netsec_setup_rx_dring(struct netsec_priv *priv)
goto err_out; goto err_out;
} }
err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0); err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0, priv->napi.napi_id);
if (err) if (err)
goto err_out; goto err_out;

Просмотреть файл

@ -1186,7 +1186,7 @@ static int cpsw_ndev_create_xdp_rxq(struct cpsw_priv *priv, int ch)
pool = cpsw->page_pool[ch]; pool = cpsw->page_pool[ch];
rxq = &priv->xdp_rxq[ch]; rxq = &priv->xdp_rxq[ch];
ret = xdp_rxq_info_reg(rxq, priv->ndev, ch); ret = xdp_rxq_info_reg(rxq, priv->ndev, ch, 0);
if (ret) if (ret)
return ret; return ret;

Просмотреть файл

@ -1499,7 +1499,7 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device,
u64_stats_init(&nvchan->tx_stats.syncp); u64_stats_init(&nvchan->tx_stats.syncp);
u64_stats_init(&nvchan->rx_stats.syncp); u64_stats_init(&nvchan->rx_stats.syncp);
ret = xdp_rxq_info_reg(&nvchan->xdp_rxq, ndev, i); ret = xdp_rxq_info_reg(&nvchan->xdp_rxq, ndev, i, 0);
if (ret) { if (ret) {
netdev_err(ndev, "xdp_rxq_info_reg fail: %d\n", ret); netdev_err(ndev, "xdp_rxq_info_reg fail: %d\n", ret);

Просмотреть файл

@ -780,7 +780,7 @@ static int tun_attach(struct tun_struct *tun, struct file *file,
} else { } else {
/* Setup XDP RX-queue info, for new tfile getting attached */ /* Setup XDP RX-queue info, for new tfile getting attached */
err = xdp_rxq_info_reg(&tfile->xdp_rxq, err = xdp_rxq_info_reg(&tfile->xdp_rxq,
tun->dev, tfile->queue_index); tun->dev, tfile->queue_index, 0);
if (err < 0) if (err < 0)
goto out; goto out;
err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq, err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq,

Просмотреть файл

@ -884,7 +884,6 @@ static int veth_napi_add(struct net_device *dev)
for (i = 0; i < dev->real_num_rx_queues; i++) { for (i = 0; i < dev->real_num_rx_queues; i++) {
struct veth_rq *rq = &priv->rq[i]; struct veth_rq *rq = &priv->rq[i];
netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT);
napi_enable(&rq->xdp_napi); napi_enable(&rq->xdp_napi);
} }
@ -926,7 +925,8 @@ static int veth_enable_xdp(struct net_device *dev)
for (i = 0; i < dev->real_num_rx_queues; i++) { for (i = 0; i < dev->real_num_rx_queues; i++) {
struct veth_rq *rq = &priv->rq[i]; struct veth_rq *rq = &priv->rq[i];
err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i); netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT);
err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id);
if (err < 0) if (err < 0)
goto err_rxq_reg; goto err_rxq_reg;
@ -952,8 +952,12 @@ static int veth_enable_xdp(struct net_device *dev)
err_reg_mem: err_reg_mem:
xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq);
err_rxq_reg: err_rxq_reg:
for (i--; i >= 0; i--) for (i--; i >= 0; i--) {
xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); struct veth_rq *rq = &priv->rq[i];
xdp_rxq_info_unreg(&rq->xdp_rxq);
netif_napi_del(&rq->xdp_napi);
}
return err; return err;
} }

Просмотреть файл

@ -1485,7 +1485,7 @@ static int virtnet_open(struct net_device *dev)
if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
schedule_delayed_work(&vi->refill, 0); schedule_delayed_work(&vi->refill, 0);
err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i); err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i, vi->rq[i].napi.napi_id);
if (err < 0) if (err < 0)
return err; return err;

Просмотреть файл

@ -2014,7 +2014,7 @@ static int xennet_create_page_pool(struct netfront_queue *queue)
} }
err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev, err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev,
queue->id); queue->id, 0);
if (err) { if (err) {
netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n"); netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n");
goto err_free_pp; goto err_free_pp;

Просмотреть файл

@ -135,14 +135,25 @@ static inline void sk_mark_napi_id(struct sock *sk, const struct sk_buff *skb)
sk_rx_queue_set(sk, skb); sk_rx_queue_set(sk, skb);
} }
static inline void __sk_mark_napi_id_once_xdp(struct sock *sk, unsigned int napi_id)
{
#ifdef CONFIG_NET_RX_BUSY_POLL
if (!READ_ONCE(sk->sk_napi_id))
WRITE_ONCE(sk->sk_napi_id, napi_id);
#endif
}
/* variant used for unconnected sockets */ /* variant used for unconnected sockets */
static inline void sk_mark_napi_id_once(struct sock *sk, static inline void sk_mark_napi_id_once(struct sock *sk,
const struct sk_buff *skb) const struct sk_buff *skb)
{ {
#ifdef CONFIG_NET_RX_BUSY_POLL __sk_mark_napi_id_once_xdp(sk, skb->napi_id);
if (!READ_ONCE(sk->sk_napi_id)) }
WRITE_ONCE(sk->sk_napi_id, skb->napi_id);
#endif static inline void sk_mark_napi_id_once_xdp(struct sock *sk,
const struct xdp_buff *xdp)
{
__sk_mark_napi_id_once_xdp(sk, xdp->rxq->napi_id);
} }
#endif /* _LINUX_NET_BUSY_POLL_H */ #endif /* _LINUX_NET_BUSY_POLL_H */

Просмотреть файл

@ -59,6 +59,7 @@ struct xdp_rxq_info {
u32 queue_index; u32 queue_index;
u32 reg_state; u32 reg_state;
struct xdp_mem_info mem; struct xdp_mem_info mem;
unsigned int napi_id;
} ____cacheline_aligned; /* perf critical, avoid false-sharing */ } ____cacheline_aligned; /* perf critical, avoid false-sharing */
struct xdp_txq_info { struct xdp_txq_info {
@ -226,7 +227,7 @@ static inline void xdp_release_frame(struct xdp_frame *xdpf)
} }
int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
struct net_device *dev, u32 queue_index); struct net_device *dev, u32 queue_index, unsigned int napi_id);
void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq);
void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq);
bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq);

Просмотреть файл

@ -9810,7 +9810,7 @@ static int netif_alloc_rx_queues(struct net_device *dev)
rx[i].dev = dev; rx[i].dev = dev;
/* XDP RX-queue setup */ /* XDP RX-queue setup */
err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i); err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
if (err < 0) if (err < 0)
goto err_rxq_info; goto err_rxq_info;
} }

Просмотреть файл

@ -158,7 +158,7 @@ static void xdp_rxq_info_init(struct xdp_rxq_info *xdp_rxq)
/* Returns 0 on success, negative on failure */ /* Returns 0 on success, negative on failure */
int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
struct net_device *dev, u32 queue_index) struct net_device *dev, u32 queue_index, unsigned int napi_id)
{ {
if (xdp_rxq->reg_state == REG_STATE_UNUSED) { if (xdp_rxq->reg_state == REG_STATE_UNUSED) {
WARN(1, "Driver promised not to register this"); WARN(1, "Driver promised not to register this");
@ -179,6 +179,7 @@ int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
xdp_rxq_info_init(xdp_rxq); xdp_rxq_info_init(xdp_rxq);
xdp_rxq->dev = dev; xdp_rxq->dev = dev;
xdp_rxq->queue_index = queue_index; xdp_rxq->queue_index = queue_index;
xdp_rxq->napi_id = napi_id;
xdp_rxq->reg_state = REG_STATE_REGISTERED; xdp_rxq->reg_state = REG_STATE_REGISTERED;
return 0; return 0;

Просмотреть файл

@ -233,6 +233,7 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp,
if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)
return -EINVAL; return -EINVAL;
sk_mark_napi_id_once_xdp(&xs->sk, xdp);
len = xdp->data_end - xdp->data; len = xdp->data_end - xdp->data;
return xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ? return xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ?