microsoft
/
WSL2-Linux-Kernel
зеркало из https://github.com/microsoft/WSL2-Linux-Kernel.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

net/mlx5e: RX, Enhance legacy Receive Queue memory scheme 

Enhance the memory scheme of the legacy RQ, such that
only order-0 pages are used.

Whenever possible, prefer using a linear SKB, and build it
wrapping the WQE buffer.

Otherwise (for example, jumbo frames on x86), use non-linear SKB,
with as many frags as needed. In this case, multiple WQE
scatter entries are used, up to a maximum of 4 frags and 10KB of MTU.

This implied to remove support of HW LRO in legacy RQ, as it would
require large number of page allocations and scatter entries per WQE
on archs with PAGE_SIZE = 4KB, yielding bad performance.

In earlier patches, we guaranteed that all completions are in-order,
and that we use a cyclic WQ.
This creates an oppurtunity for a performance optimization:
The mapping between a "struct mlx5e_dma_info", and the
WQEs (struct mlx5e_wqe_frag_info) pointing to it, is constant
across different cycles of a WQ. This allows initializing
the mapping in the time of RQ creation, and not handle it
in datapath.

A struct mlx5e_dma_info that is shared between different WQEs
is allocated by the first WQE, and freed by the last one.
This implies an important requirement: WQEs that share the same
struct mlx5e_dma_info must be posted within the same NAPI.
Otherwise, upon completion, struct mlx5e_wqe_frag_info would mistakenly
point to the new struct mlx5e_dma_info, not the one that was posted
(and the HW wrote to).
This bulking requirement is actually good also for performance reasons,
hence we extend the bulk beyong the minimal requirement above.

With this memory scheme, the RQs memory footprint is reduce by a
factor of 2 on x86, and by a factor of 32 on PowerPC.
Same factors apply for the number of pages in a GRO session.

Performance tests:
ConnectX-4, single core, single RX ring, default MTU.

x86:
CPU: Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz

Packet rate (early drop in TC): no degradation
TCP streams: ~5% improvement

PowerPC:
CPU: POWER8 (raw), altivec supported

Packet rate (early drop in TC): 20% gain
TCP streams: 25% gain

Signed-off-by: Tariq Toukan <tariqt@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>
This commit is contained in:
Tariq Toukan 2018-05-02 18:23:58 +03:00 коммит произвёл Saeed Mahameed
Родитель 99cbfa93a6
Коммит 069d11465a
3 изменённых файлов: 365 добавлений и 130 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

44
drivers/net/ethernet/mellanox/mlx5/core/en.h
Просмотреть файл

@ -101,11 +101,15 @@ struct page_pool;
(MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW + \
(MLX5_MPWRQ_LOG_WQE_SZ - MLX5E_ORDER2_MAX_PACKET_MTU))
#define MLX5E_MIN_SKB_FRAG_SZ (MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM))
#define MLX5E_LOG_MAX_RX_WQE_BULK \
(ilog2(PAGE_SIZE / roundup_pow_of_two(MLX5E_MIN_SKB_FRAG_SZ)))
#define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE 0x6
#define MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE 0xa
#define MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE 0xd
#define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE 0x1
#define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE (1 + MLX5E_LOG_MAX_RX_WQE_BULK)
#define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa
#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE min_t(u8, 0xd, \
MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW)
@ -462,8 +466,9 @@ struct mlx5e_dma_info {
};
struct mlx5e_wqe_frag_info {
struct mlx5e_dma_info di;
struct mlx5e_dma_info *di;
u32 offset;
bool last_in_page;
};
struct mlx5e_umr_dma_info {
@ -476,6 +481,8 @@ struct mlx5e_mpw_info {
DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
};
#define MLX5E_MAX_RX_FRAGS 4
/* a single cache unit is capable to serve one napi call (for non-striding rq)
* or a MPWQE (for striding rq).
*/
@ -493,6 +500,9 @@ typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*);
typedef struct sk_buff *
(*mlx5e_fp_skb_from_cqe_mpwrq)(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
u16 cqe_bcnt, u32 head_offset, u32 page_idx);
typedef struct sk_buff *
(*mlx5e_fp_skb_from_cqe)(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt);
typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq);
typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);
@ -500,16 +510,27 @@ enum mlx5e_rq_flag {
MLX5E_RQ_FLAG_XDP_XMIT = BIT(0),
};
struct mlx5e_rq_frag_info {
int frag_size;
int frag_stride;
};
struct mlx5e_rq_frags_info {
struct mlx5e_rq_frag_info arr[MLX5E_MAX_RX_FRAGS];
u8 num_frags;
u8 log_num_frags;
u8 wqe_bulk;
};
struct mlx5e_rq {
/* data path */
union {
struct {
struct mlx5_wq_cyc wq;
struct mlx5e_wqe_frag_info *frag_info;
u32 frag_sz; /* max possible skb frag_sz */
union {
bool page_reuse;
};
struct mlx5_wq_cyc wq;
struct mlx5e_wqe_frag_info *frags;
struct mlx5e_dma_info *di;
struct mlx5e_rq_frags_info info;
mlx5e_fp_skb_from_cqe skb_from_cqe;
} wqe;
struct {
struct mlx5_wq_ll wq;
@ -523,7 +544,6 @@ struct mlx5e_rq {
};
struct {
u16 headroom;
u8 page_order;
u8 map_dir; /* dma map direction */
} buff;
@ -879,6 +899,12 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
struct sk_buff *
mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
u16 cqe_bcnt, u32 head_offset, u32 page_idx);
struct sk_buff *
mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt);
struct sk_buff *
mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt);
void mlx5e_update_stats(struct mlx5e_priv *priv);

238
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
Просмотреть файл

@ -51,6 +51,7 @@
struct mlx5e_rq_param {
u32 rqc[MLX5_ST_SZ_DW(rqc)];
struct mlx5_wq_param wq;
struct mlx5e_rq_frags_info frags_info;
};
struct mlx5e_sq_param {
@ -93,7 +94,7 @@ bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
return true;
}
static u32 mlx5e_mpwqe_get_linear_frag_sz(struct mlx5e_params *params)
static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)
{
if (!params->xdp_prog) {
u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
@ -107,19 +108,27 @@ static u32 mlx5e_mpwqe_get_linear_frag_sz(struct mlx5e_params *params)
static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)
{
u32 linear_frag_sz = mlx5e_mpwqe_get_linear_frag_sz(params);
u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params);
return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
}
static bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
struct mlx5e_params *params)
{
u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
return !params->lro_en && frag_sz <= PAGE_SIZE;
}
static bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
struct mlx5e_params *params)
{
u32 frag_sz = mlx5e_mpwqe_get_linear_frag_sz(params);
u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
s8 signed_log_num_strides_param;
u8 log_num_strides;
if (params->lro_en || frag_sz > PAGE_SIZE)
if (!mlx5e_rx_is_linear_skb(mdev, params))
return false;
if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
@ -145,7 +154,7 @@ static u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
struct mlx5e_params *params)
{
if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
return order_base_2(mlx5e_mpwqe_get_linear_frag_sz(params));
return order_base_2(mlx5e_rx_get_linear_frag_sz(params));
return MLX5E_MPWQE_STRIDE_SZ(mdev,
MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
@ -163,16 +172,15 @@ static u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
{
u16 linear_rq_headroom = params->xdp_prog ?
XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
bool is_linear_skb;
linear_rq_headroom += NET_IP_ALIGN;
if (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC)
return linear_rq_headroom;
is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?
mlx5e_rx_is_linear_skb(mdev, params) :
mlx5e_rx_mpwqe_is_linear_skb(mdev, params);
if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
return linear_rq_headroom;
return 0;
return is_linear_skb ? linear_rq_headroom : 0;
}
void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
@ -400,6 +408,61 @@ static inline u64 mlx5e_get_mpwqe_offset(struct mlx5e_rq *rq, u16 wqe_ix)
return (wqe_ix << MLX5E_LOG_ALIGNED_MPWQE_PPW) << PAGE_SHIFT;
}
static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
{
struct mlx5e_wqe_frag_info next_frag, *prev;
int i;
next_frag.di = &rq->wqe.di[0];
next_frag.offset = 0;
prev = NULL;
for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
struct mlx5e_wqe_frag_info *frag =
&rq->wqe.frags[i << rq->wqe.info.log_num_frags];
int f;
for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
next_frag.di++;
next_frag.offset = 0;
if (prev)
prev->last_in_page = true;
}
*frag = next_frag;
/* prepare next */
next_frag.offset += frag_info[f].frag_stride;
prev = frag;
}
}
if (prev)
prev->last_in_page = true;
}
static int mlx5e_init_di_list(struct mlx5e_rq *rq,
struct mlx5e_params *params,
int wq_sz, int cpu)
{
int len = wq_sz << rq->wqe.info.log_num_frags;
rq->wqe.di = kvzalloc_node(len * sizeof(*rq->wqe.di),
GFP_KERNEL, cpu_to_node(cpu));
if (!rq->wqe.di)
return -ENOMEM;
mlx5e_init_frags_partition(rq);
return 0;
}
static void mlx5e_free_di_list(struct mlx5e_rq *rq)
{
kvfree(rq->wqe.di);
}
static int mlx5e_alloc_rq(struct mlx5e_channel *c,
struct mlx5e_params *params,
struct mlx5e_rq_param *rqp,
@ -409,8 +472,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
struct mlx5_core_dev *mdev = c->mdev;
void *rqc = rqp->rqc;
void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
u32 byte_count, pool_size;
int npages;
u32 pool_size;
int wq_sz;
int err;
int i;
@ -480,8 +542,6 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params);
rq->mpwqe.num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params));
byte_count = rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
err = mlx5e_create_rq_umr_mkey(mdev, rq);
if (err)
goto err_rq_wq_destroy;
@ -489,7 +549,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
err = mlx5e_rq_alloc_mpwqe_info(rq, c);
if (err)
goto err_destroy_umr_mkey;
goto err_free;
break;
default: /* MLX5_WQ_TYPE_CYCLIC */
err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
@ -501,13 +561,17 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
rq->wqe.frag_info =
kzalloc_node(wq_sz * sizeof(*rq->wqe.frag_info),
GFP_KERNEL, cpu_to_node(c->cpu));
if (!rq->wqe.frag_info) {
err = -ENOMEM;
goto err_rq_wq_destroy;
}
rq->wqe.info = rqp->frags_info;
rq->wqe.frags =
kvzalloc_node((wq_sz << rq->wqe.info.log_num_frags) *
sizeof(*rq->wqe.frags),
GFP_KERNEL, cpu_to_node(c->cpu));
if (!rq->wqe.frags)
goto err_free;
err = mlx5e_init_di_list(rq, params, wq_sz, c->cpu);
if (err)
goto err_free;
rq->post_wqes = mlx5e_post_rx_wqes;
rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
@ -518,30 +582,19 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
#endif
rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe;
if (!rq->handle_rx_cqe) {
kfree(rq->wqe.frag_info);
err = -EINVAL;
netdev_err(c->netdev, "RX handler of RQ is not set, err %d\n", err);
goto err_rq_wq_destroy;
goto err_free;
}
byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu);
#ifdef CONFIG_MLX5_EN_IPSEC
if (MLX5_IPSEC_DEV(mdev))
byte_count += MLX5E_METADATA_ETHER_LEN;
#endif
rq->wqe.page_reuse = !params->xdp_prog;
/* calc the required page order */
rq->wqe.frag_sz = MLX5_SKB_FRAG_SZ(rq->buff.headroom + byte_count);
npages = DIV_ROUND_UP(rq->wqe.frag_sz, PAGE_SIZE);
rq->buff.page_order = order_base_2(npages);
byte_count |= MLX5_HW_START_PADDING;
rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(mdev, params) ?
mlx5e_skb_from_cqe_linear :
mlx5e_skb_from_cqe_nonlinear;
rq->mkey_be = c->mkey_be;
}
/* Create a page_pool and register it with rxq */
pp_params.order = rq->buff.page_order;
pp_params.order = 0;
pp_params.flags = 0; /* No-internal DMA mapping in page_pool */
pp_params.pool_size = pool_size;
pp_params.nid = cpu_to_node(c->cpu);
@ -555,21 +608,21 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
*/
rq->page_pool = page_pool_create(&pp_params);
if (IS_ERR(rq->page_pool)) {
if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
kfree(rq->wqe.frag_info);
err = PTR_ERR(rq->page_pool);
rq->page_pool = NULL;
goto err_rq_wq_destroy;
goto err_free;
}
err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
MEM_TYPE_PAGE_POOL, rq->page_pool);
if (err)
goto err_rq_wq_destroy;
goto err_free;
for (i = 0; i < wq_sz; i++) {
if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
struct mlx5e_rx_wqe_ll *wqe =
mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i);
u32 byte_count =
rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
u64 dma_offset = mlx5e_get_mpwqe_offset(rq, i);
wqe->data[0].addr = cpu_to_be64(dma_offset + rq->buff.headroom);
@ -578,9 +631,21 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
} else {
struct mlx5e_rx_wqe_cyc *wqe =
mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i);
int f;
wqe->data[0].byte_count = cpu_to_be32(byte_count);
wqe->data[0].lkey = rq->mkey_be;
for (f = 0; f < rq->wqe.info.num_frags; f++) {
u32 frag_size = rq->wqe.info.arr[f].frag_size |
MLX5_HW_START_PADDING;
wqe->data[f].byte_count = cpu_to_be32(frag_size);
wqe->data[f].lkey = rq->mkey_be;
}
/* check if num_frags is not a pow of two */
if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {
wqe->data[f].byte_count = 0;
wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
wqe->data[f].addr = 0;
}
}
}
@ -600,8 +665,16 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
return 0;
err_destroy_umr_mkey:
mlx5_core_destroy_mkey(mdev, &rq->umr_mkey);
err_free:
switch (rq->wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
kfree(rq->mpwqe.info);
mlx5_core_destroy_mkey(mdev, &rq->umr_mkey);
break;
default: /* MLX5_WQ_TYPE_CYCLIC */
kvfree(rq->wqe.frags);
mlx5e_free_di_list(rq);
}
err_rq_wq_destroy:
if (rq->xdp_prog)
@ -631,7 +704,8 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey);
break;
default: /* MLX5_WQ_TYPE_CYCLIC */
kfree(rq->wqe.frag_info);
kvfree(rq->wqe.frags);
mlx5e_free_di_list(rq);
}
for (i = rq->page_cache.head; i != rq->page_cache.tail;
@ -823,17 +897,8 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
rq->dealloc_wqe(rq, wqe_ix);
mlx5_wq_cyc_pop(wq);
}
/* Clean outstanding pages on handled WQEs that decided to do page-reuse,
* but yet to be re-posted.
*/
if (rq->wqe.page_reuse) {
int wq_sz = mlx5_wq_cyc_get_size(wq);
for (wqe_ix = 0; wqe_ix < wq_sz; wqe_ix++)
rq->dealloc_wqe(rq, wqe_ix);
}
}
}
static int mlx5e_open_rq(struct mlx5e_channel *c,
@ -1954,6 +2019,61 @@ static void mlx5e_close_channel(struct mlx5e_channel *c)
kfree(c);
}
#define DEFAULT_FRAG_SIZE (2048)
static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_rq_frags_info *info)
{
u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu);
int frag_size_max = DEFAULT_FRAG_SIZE;
u32 buf_size = 0;
int i;
#ifdef CONFIG_MLX5_EN_IPSEC
if (MLX5_IPSEC_DEV(mdev))
byte_count += MLX5E_METADATA_ETHER_LEN;
#endif
if (mlx5e_rx_is_linear_skb(mdev, params)) {
int frag_stride;
frag_stride = mlx5e_rx_get_linear_frag_sz(params);
frag_stride = roundup_pow_of_two(frag_stride);
info->arr[0].frag_size = byte_count;
info->arr[0].frag_stride = frag_stride;
info->num_frags = 1;
info->wqe_bulk = PAGE_SIZE / frag_stride;
goto out;
}
if (byte_count > PAGE_SIZE +
(MLX5E_MAX_RX_FRAGS - 1) * frag_size_max)
frag_size_max = PAGE_SIZE;
i = 0;
while (buf_size < byte_count) {
int frag_size = byte_count - buf_size;
if (i < MLX5E_MAX_RX_FRAGS - 1)
frag_size = min(frag_size, frag_size_max);
info->arr[i].frag_size = frag_size;
info->arr[i].frag_stride = roundup_pow_of_two(frag_size);
buf_size += frag_size;
i++;
}
info->num_frags = i;
/* number of different wqes sharing a page */
info->wqe_bulk = 1 + (info->num_frags % 2);
out:
info->wqe_bulk = max_t(u8, info->wqe_bulk, 8);
info->log_num_frags = order_base_2(info->num_frags);
}
static inline u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs)
{
int sz = sizeof(struct mlx5_wqe_data_seg) * ndsegs;
@ -1990,6 +2110,8 @@ static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
break;
default: /* MLX5_WQ_TYPE_CYCLIC */
MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
mlx5e_build_rq_frags_info(mdev, params, &param->frags_info);
ndsegs = param->frags_info.num_frags;
}
MLX5_SET(wq, wq, wq_type, params->rq_wq_type);

213
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
Просмотреть файл

@ -164,8 +164,6 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
return mlx5e_decompress_cqes_cont(rq, cq, 1, budget_rem) - 1;
}
#define RQ_PAGE_SIZE(rq) ((1 << rq->buff.page_order) << PAGE_SHIFT)
static inline bool mlx5e_page_is_reserved(struct page *page)
{
return page_is_pfmemalloc(page) || page_to_nid(page) != numa_mem_id();
@ -214,7 +212,7 @@ static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq,
stats->cache_reuse++;
dma_sync_single_for_device(rq->pdev, dma_info->addr,
RQ_PAGE_SIZE(rq),
PAGE_SIZE,
DMA_FROM_DEVICE);
return true;
}
@ -230,7 +228,7 @@ static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq,
return -ENOMEM;
dma_info->addr = dma_map_page(rq->pdev, dma_info->page, 0,
RQ_PAGE_SIZE(rq), rq->buff.map_dir);
PAGE_SIZE, rq->buff.map_dir);
if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
put_page(dma_info->page);
dma_info->page = NULL;
@ -243,8 +241,7 @@ static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq,
static void mlx5e_page_dma_unmap(struct mlx5e_rq *rq,
struct mlx5e_dma_info *dma_info)
{
dma_unmap_page(rq->pdev, dma_info->addr, RQ_PAGE_SIZE(rq),
rq->buff.map_dir);
dma_unmap_page(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir);
}
void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,
@ -262,58 +259,96 @@ void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,
}
}
static inline bool mlx5e_page_reuse(struct mlx5e_rq *rq,
struct mlx5e_wqe_frag_info *wi)
static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq,
struct mlx5e_wqe_frag_info *frag)
{
return rq->wqe.page_reuse && wi->di.page &&
(wi->offset + rq->wqe.frag_sz <= RQ_PAGE_SIZE(rq)) &&
!mlx5e_page_is_reserved(wi->di.page);
int err = 0;
if (!frag->offset)
/* On first frag (offset == 0), replenish page (dma_info actually).
* Other frags that point to the same dma_info (with a different
* offset) should just use the new one without replenishing again
* by themselves.
*/
err = mlx5e_page_alloc_mapped(rq, frag->di);
return err;
}
static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq,
struct mlx5e_wqe_frag_info *frag)
{
if (frag->last_in_page)
mlx5e_page_release(rq, frag->di, true);
}
static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix)
{
return &rq->wqe.frag_info[ix];
return &rq->wqe.frags[ix << rq->wqe.info.log_num_frags];
}
static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe, u16 ix)
static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe,
u16 ix)
{
struct mlx5e_wqe_frag_info *wi = &rq->wqe.frag_info[ix];
struct mlx5e_wqe_frag_info *frag = get_frag(rq, ix);
int err;
int i;
/* check if page exists, hence can be reused */
if (!wi->di.page) {
if (unlikely(mlx5e_page_alloc_mapped(rq, &wi->di)))
return -ENOMEM;
wi->offset = 0;
for (i = 0; i < rq->wqe.info.num_frags; i++, frag++) {
err = mlx5e_get_rx_frag(rq, frag);
if (unlikely(err))
goto free_frags;
wqe->data[i].addr = cpu_to_be64(frag->di->addr +
frag->offset + rq->buff.headroom);
}
wqe->data[0].addr = cpu_to_be64(wi->di.addr + wi->offset + rq->buff.headroom);
return 0;
free_frags:
while (--i >= 0)
mlx5e_put_rx_frag(rq, --frag);
return err;
}
static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,
struct mlx5e_wqe_frag_info *wi)
{
mlx5e_page_release(rq, &wi->di, true);
wi->di.page = NULL;
}
int i;
static inline void mlx5e_free_rx_wqe_reuse(struct mlx5e_rq *rq,
struct mlx5e_wqe_frag_info *wi)
{
if (mlx5e_page_reuse(rq, wi)) {
rq->stats->page_reuse++;
return;
}
mlx5e_free_rx_wqe(rq, wi);
for (i = 0; i < rq->wqe.info.num_frags; i++, wi++)
mlx5e_put_rx_frag(rq, wi);
}
void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
{
struct mlx5e_wqe_frag_info *wi = &rq->wqe.frag_info[ix];
struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix);
if (wi->di.page)
mlx5e_free_rx_wqe(rq, wi);
mlx5e_free_rx_wqe(rq, wi);
}
static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk)
{
struct mlx5_wq_cyc *wq = &rq->wqe.wq;
int err;
int i;
for (i = 0; i < wqe_bulk; i++) {
struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(wq, ix + i);
err = mlx5e_alloc_rx_wqe(rq, wqe, ix + i);
if (unlikely(err))
goto free_wqes;
}
return 0;
free_wqes:
while (--i >= 0)
mlx5e_dealloc_rx_wqe(rq, ix + i);
return err;
}
static inline void
@ -476,26 +511,28 @@ void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
{
struct mlx5_wq_cyc *wq = &rq->wqe.wq;
u8 wqe_bulk;
int err;
if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
return false;
if (mlx5_wq_cyc_is_full(wq))
wqe_bulk = rq->wqe.info.wqe_bulk;
if (mlx5_wq_cyc_missing(wq) < wqe_bulk)
return false;
do {
u16 head = mlx5_wq_cyc_get_head(wq);
struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(wq, head);
err = mlx5e_alloc_rx_wqe(rq, wqe, head);
err = mlx5e_alloc_rx_wqes(rq, head, wqe_bulk);
if (unlikely(err)) {
rq->stats->buff_alloc_err++;
break;
}
mlx5_wq_cyc_push(wq);
} while (!mlx5_wq_cyc_is_full(wq));
mlx5_wq_cyc_push_n(wq, wqe_bulk);
} while (mlx5_wq_cyc_missing(wq) >= wqe_bulk);
/* ensure wqes are visible to device before updating doorbell record */
dma_wmb();
@ -949,11 +986,11 @@ struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
return skb;
}
static inline
struct sk_buff *skb_from_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
struct sk_buff *
mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
{
struct mlx5e_dma_info *di = &wi->di;
struct mlx5e_dma_info *di = wi->di;
u16 rx_headroom = rq->buff.headroom;
struct sk_buff *skb;
void *va, *data;
@ -968,7 +1005,6 @@ struct sk_buff *skb_from_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
frag_size, DMA_FROM_DEVICE);
prefetchw(va); /* xdp_frame data area */
prefetch(data);
wi->offset += frag_size;
if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
rq->stats->wqe_err++;
@ -991,6 +1027,56 @@ struct sk_buff *skb_from_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
return skb;
}
struct sk_buff *
mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
{
struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
struct mlx5e_wqe_frag_info *head_wi = wi;
u16 headlen = min_t(u32, MLX5E_RX_MAX_HEAD, cqe_bcnt);
u16 frag_headlen = headlen;
u16 byte_cnt = cqe_bcnt - headlen;
struct sk_buff *skb;
if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
rq->stats->wqe_err++;
return NULL;
}
/* XDP is not supported in this configuration, as incoming packets
* might spread among multiple pages.
*/
skb = napi_alloc_skb(rq->cq.napi,
ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long)));
if (unlikely(!skb)) {
rq->stats->buff_alloc_err++;
return NULL;
}
prefetchw(skb->data);
while (byte_cnt) {
u16 frag_consumed_bytes =
min_t(u16, frag_info->frag_size - frag_headlen, byte_cnt);
mlx5e_add_skb_frag(rq, skb, wi->di, wi->offset + frag_headlen,
frag_consumed_bytes, frag_info->frag_stride);
byte_cnt -= frag_consumed_bytes;
frag_headlen = 0;
frag_info++;
wi++;
}
/* copy header */
mlx5e_copy_skb_header(rq->pdev, skb, head_wi->di, head_wi->offset,
0, headlen);
/* skb linear part was allocated with headlen and aligned to long */
skb->tail += headlen;
skb->len += headlen;
return skb;
}
void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
{
struct mlx5_wq_cyc *wq = &rq->wqe.wq;
@ -1003,23 +1089,23 @@ void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
wi = get_frag(rq, ci);
cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
skb = skb_from_cqe(rq, cqe, wi, cqe_bcnt);
skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt);
if (!skb) {
/* probably for XDP */
if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
wi->di.page = NULL;
/* do not return page to cache, it will be returned on XDP_TX completion */
/* do not return page to cache,
* it will be returned on XDP_TX completion.
*/
goto wq_cyc_pop;
}
/* probably an XDP_DROP, save the page-reuse checks */
mlx5e_free_rx_wqe(rq, wi);
goto wq_cyc_pop;
goto free_wqe;
}
mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
napi_gro_receive(rq->cq.napi, skb);
mlx5e_free_rx_wqe_reuse(rq, wi);
free_wqe:
mlx5e_free_rx_wqe(rq, wi);
wq_cyc_pop:
mlx5_wq_cyc_pop(wq);
}
@ -1041,16 +1127,16 @@ void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
wi = get_frag(rq, ci);
cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
skb = skb_from_cqe(rq, cqe, wi, cqe_bcnt);
skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt);
if (!skb) {
/* probably for XDP */
if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
wi->di.page = NULL;
/* do not return page to cache, it will be returned on XDP_TX completion */
/* do not return page to cache,
* it will be returned on XDP_TX completion.
*/
goto wq_cyc_pop;
}
/* probably an XDP_DROP, save the page-reuse checks */
mlx5e_free_rx_wqe(rq, wi);
goto wq_cyc_pop;
goto free_wqe;
}
mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
@ -1060,7 +1146,8 @@ void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
napi_gro_receive(rq->cq.napi, skb);
mlx5e_free_rx_wqe_reuse(rq, wi);
free_wqe:
mlx5e_free_rx_wqe(rq, wi);
wq_cyc_pop:
mlx5_wq_cyc_pop(wq);
}
@ -1409,7 +1496,7 @@ void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
wi = get_frag(rq, ci);
cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
skb = skb_from_cqe(rq, cqe, wi, cqe_bcnt);
skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt);
if (!skb)
goto wq_free_wqe;
@ -1421,7 +1508,7 @@ void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
napi_gro_receive(rq->cq.napi, skb);
wq_free_wqe:
mlx5e_free_rx_wqe_reuse(rq, wi);
mlx5e_free_rx_wqe(rq, wi);
mlx5_wq_cyc_pop(wq);
}
@ -1441,7 +1528,7 @@ void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
wi = get_frag(rq, ci);
cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
skb = skb_from_cqe(rq, cqe, wi, cqe_bcnt);
skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt);
if (unlikely(!skb)) {
/* a DROP, save the page-reuse checks */
mlx5e_free_rx_wqe(rq, wi);
@ -1456,7 +1543,7 @@ void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
napi_gro_receive(rq->cq.napi, skb);
mlx5e_free_rx_wqe_reuse(rq, wi);
mlx5e_free_rx_wqe(rq, wi);
wq_cyc_pop:
mlx5_wq_cyc_pop(wq);
}