microsoft
/
WSL2-Linux-Kernel
зеркало из https://github.com/microsoft/WSL2-Linux-Kernel.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge branch 'mlx5-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux into net-next 

Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
This commit is contained in:
Saeed Mahameed 2021-10-27 12:45:37 -07:00
Родитель f25c0515c5 60dd57c747
Коммит 573bce9e67
34 изменённых файлов: 293 добавлений и 249 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

26
drivers/infiniband/hw/mlx5/cmd.c
Просмотреть файл

@ -206,3 +206,29 @@ out:
kfree(in); kfree(in);
return err; return err;
} }
int mlx5_cmd_uar_alloc(struct mlx5_core_dev *dev, u32 *uarn, u16 uid)
{
u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {};
u32 in[MLX5_ST_SZ_DW(alloc_uar_in)] = {};
int err;
MLX5_SET(alloc_uar_in, in, opcode, MLX5_CMD_OP_ALLOC_UAR);
MLX5_SET(alloc_uar_in, in, uid, uid);
err = mlx5_cmd_exec_inout(dev, alloc_uar, in, out);
if (err)
return err;
*uarn = MLX5_GET(alloc_uar_out, out, uar);
return 0;
}
int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid)
{
u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)] = {};
MLX5_SET(dealloc_uar_in, in, opcode, MLX5_CMD_OP_DEALLOC_UAR);
MLX5_SET(dealloc_uar_in, in, uar, uarn);
MLX5_SET(dealloc_uar_in, in, uid, uid);
return mlx5_cmd_exec_in(dev, dealloc_uar, in);
}

2
drivers/infiniband/hw/mlx5/cmd.h
Просмотреть файл

@ -57,4 +57,6 @@ int mlx5_cmd_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn, u16 uid);
int mlx5_cmd_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn, u16 uid); int mlx5_cmd_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn, u16 uid);
int mlx5_cmd_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb, int mlx5_cmd_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb,
u16 opmod, u8 port); u16 opmod, u8 port);
int mlx5_cmd_uar_alloc(struct mlx5_core_dev *dev, u32 *uarn, u16 uid);
int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid);
#endif /* MLX5_IB_CMD_H */ #endif /* MLX5_IB_CMD_H */

13
drivers/infiniband/hw/mlx5/devx.c
Просмотреть файл

@ -1292,21 +1292,16 @@ static int devx_handle_mkey_indirect(struct devx_obj *obj,
struct mlx5_ib_dev *dev, struct mlx5_ib_dev *dev,
void *in, void *out) void *in, void *out)
{ {
struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr; struct mlx5_ib_mkey *mkey = &obj->mkey;
struct mlx5_core_mkey *mkey;
void *mkc; void *mkc;
u8 key; u8 key;
mkey = &devx_mr->mmkey;
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
key = MLX5_GET(mkc, mkc, mkey_7_0); key = MLX5_GET(mkc, mkc, mkey_7_0);
mkey->key = mlx5_idx_to_mkey( mkey->key = mlx5_idx_to_mkey(
MLX5_GET(create_mkey_out, out, mkey_index)) | key; MLX5_GET(create_mkey_out, out, mkey_index)) | key;
mkey->type = MLX5_MKEY_INDIRECT_DEVX; mkey->type = MLX5_MKEY_INDIRECT_DEVX;
mkey->iova = MLX5_GET64(mkc, mkc, start_addr); mkey->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
mkey->size = MLX5_GET64(mkc, mkc, len);
mkey->pd = MLX5_GET(mkc, mkc, pd);
devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
init_waitqueue_head(&mkey->wait); init_waitqueue_head(&mkey->wait);
return mlx5r_store_odp_mkey(dev, mkey); return mlx5r_store_odp_mkey(dev, mkey);
@ -1384,13 +1379,13 @@ static int devx_obj_cleanup(struct ib_uobject *uobject,
dev = mlx5_udata_to_mdev(&attrs->driver_udata); dev = mlx5_udata_to_mdev(&attrs->driver_udata);
if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY && if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY &&
xa_erase(&obj->ib_dev->odp_mkeys, xa_erase(&obj->ib_dev->odp_mkeys,
mlx5_base_mkey(obj->devx_mr.mmkey.key))) mlx5_base_mkey(obj->mkey.key)))
/* /*
* The pagefault_single_data_segment() does commands against * The pagefault_single_data_segment() does commands against
* the mmkey, we must wait for that to stop before freeing the * the mmkey, we must wait for that to stop before freeing the
* mkey, as another allocation could get the same mkey #. * mkey, as another allocation could get the same mkey #.
*/ */
mlx5r_deref_wait_odp_mkey(&obj->devx_mr.mmkey); mlx5r_deref_wait_odp_mkey(&obj->mkey);
if (obj->flags & DEVX_OBJ_FLAGS_DCT) if (obj->flags & DEVX_OBJ_FLAGS_DCT)
ret = mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct); ret = mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);

2
drivers/infiniband/hw/mlx5/devx.h
Просмотреть файл

@ -16,7 +16,7 @@ struct devx_obj {
u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW]; u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
u32 flags; u32 flags;
union { union {
struct mlx5_ib_devx_mr devx_mr; struct mlx5_ib_mkey mkey;
struct mlx5_core_dct core_dct; struct mlx5_core_dct core_dct;
struct mlx5_core_cq core_cq; struct mlx5_core_cq core_cq;
u32 flow_counter_bulk_size; u32 flow_counter_bulk_size;

55
drivers/infiniband/hw/mlx5/main.c
Просмотреть файл

@ -1643,7 +1643,8 @@ static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *conte
bfregi = &context->bfregi; bfregi = &context->bfregi;
for (i = 0; i < bfregi->num_static_sys_pages; i++) { for (i = 0; i < bfregi->num_static_sys_pages; i++) {
err = mlx5_cmd_alloc_uar(dev->mdev, &bfregi->sys_pages[i]); err = mlx5_cmd_uar_alloc(dev->mdev, &bfregi->sys_pages[i],
context->devx_uid);
if (err) if (err)
goto error; goto error;
@ -1657,7 +1658,8 @@ static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *conte
error: error:
for (--i; i >= 0; i--) for (--i; i >= 0; i--)
if (mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i])) if (mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
context->devx_uid))
mlx5_ib_warn(dev, "failed to free uar %d\n", i); mlx5_ib_warn(dev, "failed to free uar %d\n", i);
return err; return err;
@ -1673,7 +1675,8 @@ static void deallocate_uars(struct mlx5_ib_dev *dev,
for (i = 0; i < bfregi->num_sys_pages; i++) for (i = 0; i < bfregi->num_sys_pages; i++)
if (i < bfregi->num_static_sys_pages || if (i < bfregi->num_static_sys_pages ||
bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX) bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX)
mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]); mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
context->devx_uid);
} }
int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
@ -1891,6 +1894,13 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
if (req.num_low_latency_bfregs > req.total_num_bfregs - 1) if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
return -EINVAL; return -EINVAL;
if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
err = mlx5_ib_devx_create(dev, true);
if (err < 0)
goto out_ctx;
context->devx_uid = err;
}
lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR; lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR; lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
bfregi = &context->bfregi; bfregi = &context->bfregi;
@ -1903,7 +1913,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
/* updates req->total_num_bfregs */ /* updates req->total_num_bfregs */
err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi); err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
if (err) if (err)
goto out_ctx; goto out_devx;
mutex_init(&bfregi->lock); mutex_init(&bfregi->lock);
bfregi->lib_uar_4k = lib_uar_4k; bfregi->lib_uar_4k = lib_uar_4k;
@ -1911,7 +1921,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
GFP_KERNEL); GFP_KERNEL);
if (!bfregi->count) { if (!bfregi->count) {
err = -ENOMEM; err = -ENOMEM;
goto out_ctx; goto out_devx;
} }
bfregi->sys_pages = kcalloc(bfregi->num_sys_pages, bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
@ -1927,17 +1937,10 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
goto out_sys_pages; goto out_sys_pages;
uar_done: uar_done:
if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
err = mlx5_ib_devx_create(dev, true);
if (err < 0)
goto out_uars;
context->devx_uid = err;
}
err = mlx5_ib_alloc_transport_domain(dev, &context->tdn, err = mlx5_ib_alloc_transport_domain(dev, &context->tdn,
context->devx_uid); context->devx_uid);
if (err) if (err)
goto out_devx; goto out_uars;
INIT_LIST_HEAD(&context->db_page_list); INIT_LIST_HEAD(&context->db_page_list);
mutex_init(&context->db_page_mutex); mutex_init(&context->db_page_mutex);
@ -1972,9 +1975,6 @@ uar_done:
out_mdev: out_mdev:
mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid); mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
out_devx:
if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
mlx5_ib_devx_destroy(dev, context->devx_uid);
out_uars: out_uars:
deallocate_uars(dev, context); deallocate_uars(dev, context);
@ -1985,6 +1985,10 @@ out_sys_pages:
out_count: out_count:
kfree(bfregi->count); kfree(bfregi->count);
out_devx:
if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
mlx5_ib_devx_destroy(dev, context->devx_uid);
out_ctx: out_ctx:
return err; return err;
} }
@ -2021,12 +2025,12 @@ static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
bfregi = &context->bfregi; bfregi = &context->bfregi;
mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid); mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
if (context->devx_uid)
mlx5_ib_devx_destroy(dev, context->devx_uid);
deallocate_uars(dev, context); deallocate_uars(dev, context);
kfree(bfregi->sys_pages); kfree(bfregi->sys_pages);
kfree(bfregi->count); kfree(bfregi->count);
if (context->devx_uid)
mlx5_ib_devx_destroy(dev, context->devx_uid);
} }
static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev, static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
@ -2119,6 +2123,7 @@ static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
struct mlx5_user_mmap_entry *mentry = to_mmmap(entry); struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device); struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
struct mlx5_var_table *var_table = &dev->var_table; struct mlx5_var_table *var_table = &dev->var_table;
struct mlx5_ib_ucontext *context = to_mucontext(entry->ucontext);
switch (mentry->mmap_flag) { switch (mentry->mmap_flag) {
case MLX5_IB_MMAP_TYPE_MEMIC: case MLX5_IB_MMAP_TYPE_MEMIC:
@ -2133,7 +2138,8 @@ static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
break; break;
case MLX5_IB_MMAP_TYPE_UAR_WC: case MLX5_IB_MMAP_TYPE_UAR_WC:
case MLX5_IB_MMAP_TYPE_UAR_NC: case MLX5_IB_MMAP_TYPE_UAR_NC:
mlx5_cmd_free_uar(dev->mdev, mentry->page_idx); mlx5_cmd_uar_dealloc(dev->mdev, mentry->page_idx,
context->devx_uid);
kfree(mentry); kfree(mentry);
break; break;
default: default:
@ -2211,7 +2217,8 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
bfregi->count[bfreg_dyn_idx]++; bfregi->count[bfreg_dyn_idx]++;
mutex_unlock(&bfregi->lock); mutex_unlock(&bfregi->lock);
err = mlx5_cmd_alloc_uar(dev->mdev, &uar_index); err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index,
context->devx_uid);
if (err) { if (err) {
mlx5_ib_warn(dev, "UAR alloc failed\n"); mlx5_ib_warn(dev, "UAR alloc failed\n");
goto free_bfreg; goto free_bfreg;
@ -2240,7 +2247,7 @@ err:
if (!dyn_uar) if (!dyn_uar)
return err; return err;
mlx5_cmd_free_uar(dev->mdev, idx); mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid);
free_bfreg: free_bfreg:
mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx); mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
@ -3489,7 +3496,7 @@ alloc_uar_entry(struct mlx5_ib_ucontext *c,
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
dev = to_mdev(c->ibucontext.device); dev = to_mdev(c->ibucontext.device);
err = mlx5_cmd_alloc_uar(dev->mdev, &uar_index); err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index, c->devx_uid);
if (err) if (err)
goto end; goto end;
@ -3507,7 +3514,7 @@ alloc_uar_entry(struct mlx5_ib_ucontext *c,
return entry; return entry;
err_insert: err_insert:
mlx5_cmd_free_uar(dev->mdev, uar_index); mlx5_cmd_uar_dealloc(dev->mdev, uar_index, c->devx_uid);
end: end:
kfree(entry); kfree(entry);
return ERR_PTR(err); return ERR_PTR(err);

31
drivers/infiniband/hw/mlx5/mlx5_ib.h
Просмотреть файл

@ -619,6 +619,20 @@ struct mlx5_user_mmap_entry {
u32 page_idx; u32 page_idx;
}; };
enum mlx5_mkey_type {
MLX5_MKEY_MR = 1,
MLX5_MKEY_MW,
MLX5_MKEY_INDIRECT_DEVX,
};
struct mlx5_ib_mkey {
u32 key;
enum mlx5_mkey_type type;
unsigned int ndescs;
struct wait_queue_head wait;
refcount_t usecount;
};
#define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE) #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
#define MLX5_IB_DM_MEMIC_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE |\ #define MLX5_IB_DM_MEMIC_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE |\
@ -637,7 +651,7 @@ struct mlx5_user_mmap_entry {
struct mlx5_ib_mr { struct mlx5_ib_mr {
struct ib_mr ibmr; struct ib_mr ibmr;
struct mlx5_core_mkey mmkey; struct mlx5_ib_mkey mmkey;
/* User MR data */ /* User MR data */
struct mlx5_cache_ent *cache_ent; struct mlx5_cache_ent *cache_ent;
@ -659,7 +673,6 @@ struct mlx5_ib_mr {
void *descs_alloc; void *descs_alloc;
dma_addr_t desc_map; dma_addr_t desc_map;
int max_descs; int max_descs;
int ndescs;
int desc_size; int desc_size;
int access_mode; int access_mode;
@ -713,13 +726,7 @@ static inline bool is_dmabuf_mr(struct mlx5_ib_mr *mr)
struct mlx5_ib_mw { struct mlx5_ib_mw {
struct ib_mw ibmw; struct ib_mw ibmw;
struct mlx5_core_mkey mmkey; struct mlx5_ib_mkey mmkey;
int ndescs;
};
struct mlx5_ib_devx_mr {
struct mlx5_core_mkey mmkey;
int ndescs;
}; };
struct mlx5_ib_umr_context { struct mlx5_ib_umr_context {
@ -1579,7 +1586,7 @@ static inline bool mlx5_ib_can_reconfig_with_umr(struct mlx5_ib_dev *dev,
} }
static inline int mlx5r_store_odp_mkey(struct mlx5_ib_dev *dev, static inline int mlx5r_store_odp_mkey(struct mlx5_ib_dev *dev,
struct mlx5_core_mkey *mmkey) struct mlx5_ib_mkey *mmkey)
{ {
refcount_set(&mmkey->usecount, 1); refcount_set(&mmkey->usecount, 1);
@ -1588,14 +1595,14 @@ static inline int mlx5r_store_odp_mkey(struct mlx5_ib_dev *dev,
} }
/* deref an mkey that can participate in ODP flow */ /* deref an mkey that can participate in ODP flow */
static inline void mlx5r_deref_odp_mkey(struct mlx5_core_mkey *mmkey) static inline void mlx5r_deref_odp_mkey(struct mlx5_ib_mkey *mmkey)
{ {
if (refcount_dec_and_test(&mmkey->usecount)) if (refcount_dec_and_test(&mmkey->usecount))
wake_up(&mmkey->wait); wake_up(&mmkey->wait);
} }
/* deref an mkey that can participate in ODP flow and wait for relese */ /* deref an mkey that can participate in ODP flow and wait for relese */
static inline void mlx5r_deref_wait_odp_mkey(struct mlx5_core_mkey *mmkey) static inline void mlx5r_deref_wait_odp_mkey(struct mlx5_ib_mkey *mmkey)
{ {
mlx5r_deref_odp_mkey(mmkey); mlx5r_deref_odp_mkey(mmkey);
wait_event(mmkey->wait, refcount_read(&mmkey->usecount) == 0); wait_event(mmkey->wait, refcount_read(&mmkey->usecount) == 0);

83
drivers/infiniband/hw/mlx5/mr.c
Просмотреть файл

@ -88,9 +88,8 @@ static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,
MLX5_SET64(mkc, mkc, start_addr, start_addr); MLX5_SET64(mkc, mkc, start_addr, start_addr);
} }
static void static void assign_mkey_variant(struct mlx5_ib_dev *dev,
assign_mkey_variant(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey, struct mlx5_ib_mkey *mkey, u32 *in)
u32 *in)
{ {
u8 key = atomic_inc_return(&dev->mkey_var); u8 key = atomic_inc_return(&dev->mkey_var);
void *mkc; void *mkc;
@ -100,17 +99,22 @@ assign_mkey_variant(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey,
mkey->key = key; mkey->key = key;
} }
static int static int mlx5_ib_create_mkey(struct mlx5_ib_dev *dev,
mlx5_ib_create_mkey(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey, struct mlx5_ib_mkey *mkey, u32 *in, int inlen)
u32 *in, int inlen)
{ {
int ret;
assign_mkey_variant(dev, mkey, in); assign_mkey_variant(dev, mkey, in);
return mlx5_core_create_mkey(dev->mdev, mkey, in, inlen); ret = mlx5_core_create_mkey(dev->mdev, &mkey->key, in, inlen);
if (!ret)
init_waitqueue_head(&mkey->wait);
return ret;
} }
static int static int
mlx5_ib_create_mkey_cb(struct mlx5_ib_dev *dev, mlx5_ib_create_mkey_cb(struct mlx5_ib_dev *dev,
struct mlx5_core_mkey *mkey, struct mlx5_ib_mkey *mkey,
struct mlx5_async_ctx *async_ctx, struct mlx5_async_ctx *async_ctx,
u32 *in, int inlen, u32 *out, int outlen, u32 *in, int inlen, u32 *out, int outlen,
struct mlx5_async_work *context) struct mlx5_async_work *context)
@ -133,7 +137,7 @@ static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
{ {
WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key))); WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)));
return mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); return mlx5_core_destroy_mkey(dev->mdev, mr->mmkey.key);
} }
static void create_mkey_callback(int status, struct mlx5_async_work *context) static void create_mkey_callback(int status, struct mlx5_async_work *context)
@ -260,10 +264,11 @@ static struct mlx5_ib_mr *create_cache_mr(struct mlx5_cache_ent *ent)
goto free_in; goto free_in;
} }
err = mlx5_core_create_mkey(ent->dev->mdev, &mr->mmkey, in, inlen); err = mlx5_core_create_mkey(ent->dev->mdev, &mr->mmkey.key, in, inlen);
if (err) if (err)
goto free_mr; goto free_mr;
init_waitqueue_head(&mr->mmkey.wait);
mr->mmkey.type = MLX5_MKEY_MR; mr->mmkey.type = MLX5_MKEY_MR;
WRITE_ONCE(ent->dev->cache.last_add, jiffies); WRITE_ONCE(ent->dev->cache.last_add, jiffies);
spin_lock_irq(&ent->lock); spin_lock_irq(&ent->lock);
@ -290,7 +295,7 @@ static void remove_cache_mr_locked(struct mlx5_cache_ent *ent)
ent->available_mrs--; ent->available_mrs--;
ent->total_mrs--; ent->total_mrs--;
spin_unlock_irq(&ent->lock); spin_unlock_irq(&ent->lock);
mlx5_core_destroy_mkey(ent->dev->mdev, &mr->mmkey); mlx5_core_destroy_mkey(ent->dev->mdev, mr->mmkey.key);
kfree(mr); kfree(mr);
spin_lock_irq(&ent->lock); spin_lock_irq(&ent->lock);
} }
@ -658,7 +663,7 @@ static void clean_keys(struct mlx5_ib_dev *dev, int c)
ent->available_mrs--; ent->available_mrs--;
ent->total_mrs--; ent->total_mrs--;
spin_unlock_irq(&ent->lock); spin_unlock_irq(&ent->lock);
mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); mlx5_core_destroy_mkey(dev->mdev, mr->mmkey.key);
} }
list_for_each_entry_safe(mr, tmp_mr, &del_list, list) { list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
@ -911,12 +916,13 @@ static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev,
} }
static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
u64 length, int access_flags) u64 length, int access_flags, u64 iova)
{ {
mr->ibmr.lkey = mr->mmkey.key; mr->ibmr.lkey = mr->mmkey.key;
mr->ibmr.rkey = mr->mmkey.key; mr->ibmr.rkey = mr->mmkey.key;
mr->ibmr.length = length; mr->ibmr.length = length;
mr->ibmr.device = &dev->ib_dev; mr->ibmr.device = &dev->ib_dev;
mr->ibmr.iova = iova;
mr->access_flags = access_flags; mr->access_flags = access_flags;
} }
@ -974,11 +980,8 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,
mr->ibmr.pd = pd; mr->ibmr.pd = pd;
mr->umem = umem; mr->umem = umem;
mr->mmkey.iova = iova;
mr->mmkey.size = umem->length;
mr->mmkey.pd = to_mpd(pd)->pdn;
mr->page_shift = order_base_2(page_size); mr->page_shift = order_base_2(page_size);
set_mr_fields(dev, mr, umem->length, access_flags); set_mr_fields(dev, mr, umem->length, access_flags, iova);
return mr; return mr;
} }
@ -1087,8 +1090,8 @@ static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr,
wr->wr.opcode = MLX5_IB_WR_UMR; wr->wr.opcode = MLX5_IB_WR_UMR;
wr->pd = mr->ibmr.pd; wr->pd = mr->ibmr.pd;
wr->mkey = mr->mmkey.key; wr->mkey = mr->mmkey.key;
wr->length = mr->mmkey.size; wr->length = mr->ibmr.length;
wr->virt_addr = mr->mmkey.iova; wr->virt_addr = mr->ibmr.iova;
wr->access_flags = mr->access_flags; wr->access_flags = mr->access_flags;
wr->page_shift = mr->page_shift; wr->page_shift = mr->page_shift;
wr->xlt_size = sg->length; wr->xlt_size = sg->length;
@ -1341,7 +1344,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
mr->mmkey.type = MLX5_MKEY_MR; mr->mmkey.type = MLX5_MKEY_MR;
mr->desc_size = sizeof(struct mlx5_mtt); mr->desc_size = sizeof(struct mlx5_mtt);
mr->umem = umem; mr->umem = umem;
set_mr_fields(dev, mr, umem->length, access_flags); set_mr_fields(dev, mr, umem->length, access_flags, iova);
kvfree(in); kvfree(in);
mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key); mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key);
@ -1388,7 +1391,7 @@ static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr,
kfree(in); kfree(in);
set_mr_fields(dev, mr, length, acc); set_mr_fields(dev, mr, length, acc, start_addr);
return &mr->ibmr; return &mr->ibmr;
@ -1709,7 +1712,6 @@ static int umr_rereg_pd_access(struct mlx5_ib_mr *mr, struct ib_pd *pd,
return err; return err;
mr->access_flags = access_flags; mr->access_flags = access_flags;
mr->mmkey.pd = to_mpd(pd)->pdn;
return 0; return 0;
} }
@ -1754,7 +1756,6 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd,
if (flags & IB_MR_REREG_PD) { if (flags & IB_MR_REREG_PD) {
mr->ibmr.pd = pd; mr->ibmr.pd = pd;
mr->mmkey.pd = to_mpd(pd)->pdn;
upd_flags |= MLX5_IB_UPD_XLT_PD; upd_flags |= MLX5_IB_UPD_XLT_PD;
} }
if (flags & IB_MR_REREG_ACCESS) { if (flags & IB_MR_REREG_ACCESS) {
@ -1763,8 +1764,8 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd,
} }
mr->ibmr.length = new_umem->length; mr->ibmr.length = new_umem->length;
mr->mmkey.iova = iova; mr->ibmr.iova = iova;
mr->mmkey.size = new_umem->length; mr->ibmr.length = new_umem->length;
mr->page_shift = order_base_2(page_size); mr->page_shift = order_base_2(page_size);
mr->umem = new_umem; mr->umem = new_umem;
err = mlx5_ib_update_mr_pas(mr, upd_flags); err = mlx5_ib_update_mr_pas(mr, upd_flags);
@ -1834,7 +1835,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
mr->umem = NULL; mr->umem = NULL;
atomic_sub(ib_umem_num_pages(umem), &dev->mdev->priv.reg_pages); atomic_sub(ib_umem_num_pages(umem), &dev->mdev->priv.reg_pages);
return create_real_mr(new_pd, umem, mr->mmkey.iova, return create_real_mr(new_pd, umem, mr->ibmr.iova,
new_access_flags); new_access_flags);
} }
@ -2263,9 +2264,9 @@ int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
struct mlx5_ib_dev *dev = to_mdev(ibmw->device); struct mlx5_ib_dev *dev = to_mdev(ibmw->device);
int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
struct mlx5_ib_mw *mw = to_mmw(ibmw); struct mlx5_ib_mw *mw = to_mmw(ibmw);
unsigned int ndescs;
u32 *in = NULL; u32 *in = NULL;
void *mkc; void *mkc;
int ndescs;
int err; int err;
struct mlx5_ib_alloc_mw req = {}; struct mlx5_ib_alloc_mw req = {};
struct { struct {
@ -2310,7 +2311,7 @@ int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
mw->mmkey.type = MLX5_MKEY_MW; mw->mmkey.type = MLX5_MKEY_MW;
ibmw->rkey = mw->mmkey.key; ibmw->rkey = mw->mmkey.key;
mw->ndescs = ndescs; mw->mmkey.ndescs = ndescs;
resp.response_length = resp.response_length =
min(offsetofend(typeof(resp), response_length), udata->outlen); min(offsetofend(typeof(resp), response_length), udata->outlen);
@ -2330,7 +2331,7 @@ int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
return 0; return 0;
free_mkey: free_mkey:
mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey); mlx5_core_destroy_mkey(dev->mdev, mw->mmkey.key);
free: free:
kfree(in); kfree(in);
return err; return err;
@ -2349,7 +2350,7 @@ int mlx5_ib_dealloc_mw(struct ib_mw *mw)
*/ */
mlx5r_deref_wait_odp_mkey(&mmw->mmkey); mlx5r_deref_wait_odp_mkey(&mmw->mmkey);
return mlx5_core_destroy_mkey(dev->mdev, &mmw->mmkey); return mlx5_core_destroy_mkey(dev->mdev, mmw->mmkey.key);
} }
int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
@ -2406,7 +2407,7 @@ mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
mr->meta_length = 0; mr->meta_length = 0;
if (data_sg_nents == 1) { if (data_sg_nents == 1) {
n++; n++;
mr->ndescs = 1; mr->mmkey.ndescs = 1;
if (data_sg_offset) if (data_sg_offset)
sg_offset = *data_sg_offset; sg_offset = *data_sg_offset;
mr->data_length = sg_dma_len(data_sg) - sg_offset; mr->data_length = sg_dma_len(data_sg) - sg_offset;
@ -2459,7 +2460,7 @@ mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
if (sg_offset_p) if (sg_offset_p)
*sg_offset_p = sg_offset; *sg_offset_p = sg_offset;
mr->ndescs = i; mr->mmkey.ndescs = i;
mr->data_length = mr->ibmr.length; mr->data_length = mr->ibmr.length;
if (meta_sg_nents) { if (meta_sg_nents) {
@ -2492,11 +2493,11 @@ static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
struct mlx5_ib_mr *mr = to_mmr(ibmr); struct mlx5_ib_mr *mr = to_mmr(ibmr);
__be64 *descs; __be64 *descs;
if (unlikely(mr->ndescs == mr->max_descs)) if (unlikely(mr->mmkey.ndescs == mr->max_descs))
return -ENOMEM; return -ENOMEM;
descs = mr->descs; descs = mr->descs;
descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); descs[mr->mmkey.ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
return 0; return 0;
} }
@ -2506,11 +2507,11 @@ static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr)
struct mlx5_ib_mr *mr = to_mmr(ibmr); struct mlx5_ib_mr *mr = to_mmr(ibmr);
__be64 *descs; __be64 *descs;
if (unlikely(mr->ndescs + mr->meta_ndescs == mr->max_descs)) if (unlikely(mr->mmkey.ndescs + mr->meta_ndescs == mr->max_descs))
return -ENOMEM; return -ENOMEM;
descs = mr->descs; descs = mr->descs;
descs[mr->ndescs + mr->meta_ndescs++] = descs[mr->mmkey.ndescs + mr->meta_ndescs++] =
cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
return 0; return 0;
@ -2526,7 +2527,7 @@ mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
struct mlx5_ib_mr *pi_mr = mr->mtt_mr; struct mlx5_ib_mr *pi_mr = mr->mtt_mr;
int n; int n;
pi_mr->ndescs = 0; pi_mr->mmkey.ndescs = 0;
pi_mr->meta_ndescs = 0; pi_mr->meta_ndescs = 0;
pi_mr->meta_length = 0; pi_mr->meta_length = 0;
@ -2560,7 +2561,7 @@ mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
* metadata offset at the first metadata page * metadata offset at the first metadata page
*/ */
pi_mr->pi_iova = (iova & page_mask) + pi_mr->pi_iova = (iova & page_mask) +
pi_mr->ndescs * ibmr->page_size + pi_mr->mmkey.ndescs * ibmr->page_size +
(pi_mr->ibmr.iova & ~page_mask); (pi_mr->ibmr.iova & ~page_mask);
/* /*
* In order to use one MTT MR for data and metadata, we register * In order to use one MTT MR for data and metadata, we register
@ -2591,7 +2592,7 @@ mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
struct mlx5_ib_mr *pi_mr = mr->klm_mr; struct mlx5_ib_mr *pi_mr = mr->klm_mr;
int n; int n;
pi_mr->ndescs = 0; pi_mr->mmkey.ndescs = 0;
pi_mr->meta_ndescs = 0; pi_mr->meta_ndescs = 0;
pi_mr->meta_length = 0; pi_mr->meta_length = 0;
@ -2626,7 +2627,7 @@ int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY); WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY);
mr->ndescs = 0; mr->mmkey.ndescs = 0;
mr->data_length = 0; mr->data_length = 0;
mr->data_iova = 0; mr->data_iova = 0;
mr->meta_ndescs = 0; mr->meta_ndescs = 0;
@ -2682,7 +2683,7 @@ int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
struct mlx5_ib_mr *mr = to_mmr(ibmr); struct mlx5_ib_mr *mr = to_mmr(ibmr);
int n; int n;
mr->ndescs = 0; mr->mmkey.ndescs = 0;
ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map, ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map,
mr->desc_size * mr->max_descs, mr->desc_size * mr->max_descs,

38
drivers/infiniband/hw/mlx5/odp.c
Просмотреть файл

@ -430,7 +430,7 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr,
mr->umem = &odp->umem; mr->umem = &odp->umem;
mr->ibmr.lkey = mr->mmkey.key; mr->ibmr.lkey = mr->mmkey.key;
mr->ibmr.rkey = mr->mmkey.key; mr->ibmr.rkey = mr->mmkey.key;
mr->mmkey.iova = idx * MLX5_IMR_MTT_SIZE; mr->ibmr.iova = idx * MLX5_IMR_MTT_SIZE;
mr->parent = imr; mr->parent = imr;
odp->private = mr; odp->private = mr;
@ -500,7 +500,7 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,
} }
imr->ibmr.pd = &pd->ibpd; imr->ibmr.pd = &pd->ibpd;
imr->mmkey.iova = 0; imr->ibmr.iova = 0;
imr->umem = &umem_odp->umem; imr->umem = &umem_odp->umem;
imr->ibmr.lkey = imr->mmkey.key; imr->ibmr.lkey = imr->mmkey.key;
imr->ibmr.rkey = imr->mmkey.key; imr->ibmr.rkey = imr->mmkey.key;
@ -738,7 +738,7 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt,
{ {
struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
if (unlikely(io_virt < mr->mmkey.iova)) if (unlikely(io_virt < mr->ibmr.iova))
return -EFAULT; return -EFAULT;
if (mr->umem->is_dmabuf) if (mr->umem->is_dmabuf)
@ -747,7 +747,7 @@ static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt,
if (!odp->is_implicit_odp) { if (!odp->is_implicit_odp) {
u64 user_va; u64 user_va;
if (check_add_overflow(io_virt - mr->mmkey.iova, if (check_add_overflow(io_virt - mr->ibmr.iova,
(u64)odp->umem.address, &user_va)) (u64)odp->umem.address, &user_va))
return -EFAULT; return -EFAULT;
if (unlikely(user_va >= ib_umem_end(odp) || if (unlikely(user_va >= ib_umem_end(odp) ||
@ -788,7 +788,7 @@ struct pf_frame {
int depth; int depth;
}; };
static bool mkey_is_eq(struct mlx5_core_mkey *mmkey, u32 key) static bool mkey_is_eq(struct mlx5_ib_mkey *mmkey, u32 key)
{ {
if (!mmkey) if (!mmkey)
return false; return false;
@ -797,21 +797,6 @@ static bool mkey_is_eq(struct mlx5_core_mkey *mmkey, u32 key)
return mmkey->key == key; return mmkey->key == key;
} }
static int get_indirect_num_descs(struct mlx5_core_mkey *mmkey)
{
struct mlx5_ib_mw *mw;
struct mlx5_ib_devx_mr *devx_mr;
if (mmkey->type == MLX5_MKEY_MW) {
mw = container_of(mmkey, struct mlx5_ib_mw, mmkey);
return mw->ndescs;
}
devx_mr = container_of(mmkey, struct mlx5_ib_devx_mr,
mmkey);
return devx_mr->ndescs;
}
/* /*
* Handle a single data segment in a page-fault WQE or RDMA region. * Handle a single data segment in a page-fault WQE or RDMA region.
* *
@ -831,12 +816,11 @@ static int pagefault_single_data_segment(struct mlx5_ib_dev *dev,
{ {
int npages = 0, ret, i, outlen, cur_outlen = 0, depth = 0; int npages = 0, ret, i, outlen, cur_outlen = 0, depth = 0;
struct pf_frame *head = NULL, *frame; struct pf_frame *head = NULL, *frame;
struct mlx5_core_mkey *mmkey; struct mlx5_ib_mkey *mmkey;
struct mlx5_ib_mr *mr; struct mlx5_ib_mr *mr;
struct mlx5_klm *pklm; struct mlx5_klm *pklm;
u32 *out = NULL; u32 *out = NULL;
size_t offset; size_t offset;
int ndescs;
io_virt += *bytes_committed; io_virt += *bytes_committed;
bcnt -= *bytes_committed; bcnt -= *bytes_committed;
@ -885,8 +869,6 @@ next_mr:
case MLX5_MKEY_MW: case MLX5_MKEY_MW:
case MLX5_MKEY_INDIRECT_DEVX: case MLX5_MKEY_INDIRECT_DEVX:
ndescs = get_indirect_num_descs(mmkey);
if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) { if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) {
mlx5_ib_dbg(dev, "indirection level exceeded\n"); mlx5_ib_dbg(dev, "indirection level exceeded\n");
ret = -EFAULT; ret = -EFAULT;
@ -894,7 +876,7 @@ next_mr:
} }
outlen = MLX5_ST_SZ_BYTES(query_mkey_out) + outlen = MLX5_ST_SZ_BYTES(query_mkey_out) +
sizeof(*pklm) * (ndescs - 2); sizeof(*pklm) * (mmkey->ndescs - 2);
if (outlen > cur_outlen) { if (outlen > cur_outlen) {
kfree(out); kfree(out);
@ -909,14 +891,14 @@ next_mr:
pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out, pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out,
bsf0_klm0_pas_mtt0_1); bsf0_klm0_pas_mtt0_1);
ret = mlx5_core_query_mkey(dev->mdev, mmkey, out, outlen); ret = mlx5_core_query_mkey(dev->mdev, mmkey->key, out, outlen);
if (ret) if (ret)
goto end; goto end;
offset = io_virt - MLX5_GET64(query_mkey_out, out, offset = io_virt - MLX5_GET64(query_mkey_out, out,
memory_key_mkey_entry.start_addr); memory_key_mkey_entry.start_addr);
for (i = 0; bcnt && i < ndescs; i++, pklm++) { for (i = 0; bcnt && i < mmkey->ndescs; i++, pklm++) {
if (offset >= be32_to_cpu(pklm->bcount)) { if (offset >= be32_to_cpu(pklm->bcount)) {
offset -= be32_to_cpu(pklm->bcount); offset -= be32_to_cpu(pklm->bcount);
continue; continue;
@ -1704,8 +1686,8 @@ get_prefetchable_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
u32 lkey) u32 lkey)
{ {
struct mlx5_ib_dev *dev = to_mdev(pd->device); struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_core_mkey *mmkey;
struct mlx5_ib_mr *mr = NULL; struct mlx5_ib_mr *mr = NULL;
struct mlx5_ib_mkey *mmkey;
xa_lock(&dev->odp_mkeys); xa_lock(&dev->odp_mkeys);
mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(lkey)); mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(lkey));

10
drivers/infiniband/hw/mlx5/wr.c
Просмотреть файл

@ -217,7 +217,7 @@ static __be64 sig_mkey_mask(void)
static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr, static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
struct mlx5_ib_mr *mr, u8 flags, bool atomic) struct mlx5_ib_mr *mr, u8 flags, bool atomic)
{ {
int size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size; int size = (mr->mmkey.ndescs + mr->meta_ndescs) * mr->desc_size;
memset(umr, 0, sizeof(*umr)); memset(umr, 0, sizeof(*umr));
@ -374,7 +374,7 @@ static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
struct mlx5_ib_mr *mr, struct mlx5_ib_mr *mr,
u32 key, int access) u32 key, int access)
{ {
int ndescs = ALIGN(mr->ndescs + mr->meta_ndescs, 8) >> 1; int ndescs = ALIGN(mr->mmkey.ndescs + mr->meta_ndescs, 8) >> 1;
memset(seg, 0, sizeof(*seg)); memset(seg, 0, sizeof(*seg));
@ -439,7 +439,7 @@ static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
struct mlx5_ib_mr *mr, struct mlx5_ib_mr *mr,
struct mlx5_ib_pd *pd) struct mlx5_ib_pd *pd)
{ {
int bcount = mr->desc_size * (mr->ndescs + mr->meta_ndescs); int bcount = mr->desc_size * (mr->mmkey.ndescs + mr->meta_ndescs);
dseg->addr = cpu_to_be64(mr->desc_map); dseg->addr = cpu_to_be64(mr->desc_map);
dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64)); dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
@ -861,7 +861,7 @@ static int set_reg_wr(struct mlx5_ib_qp *qp,
struct mlx5_ib_mr *mr = to_mmr(wr->mr); struct mlx5_ib_mr *mr = to_mmr(wr->mr);
struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd); struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device); struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
int mr_list_size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size; int mr_list_size = (mr->mmkey.ndescs + mr->meta_ndescs) * mr->desc_size;
bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD; bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC; bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC;
u8 flags = 0; u8 flags = 0;
@ -1111,7 +1111,7 @@ static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
memset(&pa_pi_mr, 0, sizeof(struct mlx5_ib_mr)); memset(&pa_pi_mr, 0, sizeof(struct mlx5_ib_mr));
/* No UMR, use local_dma_lkey */ /* No UMR, use local_dma_lkey */
pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey; pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey;
pa_pi_mr.ndescs = mr->ndescs; pa_pi_mr.mmkey.ndescs = mr->mmkey.ndescs;
pa_pi_mr.data_length = mr->data_length; pa_pi_mr.data_length = mr->data_length;
pa_pi_mr.data_iova = mr->data_iova; pa_pi_mr.data_iova = mr->data_iova;
if (mr->meta_ndescs) { if (mr->meta_ndescs) {

6
drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c
Просмотреть файл

@ -745,7 +745,7 @@ static int mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer *tracer)
MLX5_SET(mtrc_conf, in, trace_mode, TRACE_TO_MEMORY); MLX5_SET(mtrc_conf, in, trace_mode, TRACE_TO_MEMORY);
MLX5_SET(mtrc_conf, in, log_trace_buffer_size, MLX5_SET(mtrc_conf, in, log_trace_buffer_size,
ilog2(TRACER_BUFFER_PAGE_NUM)); ilog2(TRACER_BUFFER_PAGE_NUM));
MLX5_SET(mtrc_conf, in, trace_mkey, tracer->buff.mkey.key); MLX5_SET(mtrc_conf, in, trace_mkey, tracer->buff.mkey);
err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
MLX5_REG_MTRC_CONF, 0, 1); MLX5_REG_MTRC_CONF, 0, 1);
@ -1028,7 +1028,7 @@ int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer)
err_notifier_unregister: err_notifier_unregister:
mlx5_eq_notifier_unregister(dev, &tracer->nb); mlx5_eq_notifier_unregister(dev, &tracer->nb);
mlx5_core_destroy_mkey(dev, &tracer->buff.mkey); mlx5_core_destroy_mkey(dev, tracer->buff.mkey);
err_dealloc_pd: err_dealloc_pd:
mlx5_core_dealloc_pd(dev, tracer->buff.pdn); mlx5_core_dealloc_pd(dev, tracer->buff.pdn);
err_cancel_work: err_cancel_work:
@ -1051,7 +1051,7 @@ void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer)
if (tracer->owner) if (tracer->owner)
mlx5_fw_tracer_ownership_release(tracer); mlx5_fw_tracer_ownership_release(tracer);
mlx5_core_destroy_mkey(tracer->dev, &tracer->buff.mkey); mlx5_core_destroy_mkey(tracer->dev, tracer->buff.mkey);
mlx5_core_dealloc_pd(tracer->dev, tracer->buff.pdn); mlx5_core_dealloc_pd(tracer->dev, tracer->buff.pdn);
} }

2
drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.h
Просмотреть файл

@ -89,7 +89,7 @@ struct mlx5_fw_tracer {
void *log_buf; void *log_buf;
dma_addr_t dma; dma_addr_t dma;
u32 size; u32 size;
struct mlx5_core_mkey mkey; u32 mkey;
u32 consumer_index; u32 consumer_index;
} buff; } buff;

10
drivers/net/ethernet/mellanox/mlx5/core/diag/rsc_dump.c
Просмотреть файл

@ -30,7 +30,7 @@ static const char *const mlx5_rsc_sgmt_name[] = {
struct mlx5_rsc_dump { struct mlx5_rsc_dump {
u32 pdn; u32 pdn;
struct mlx5_core_mkey mkey; u32 mkey;
u16 fw_segment_type[MLX5_SGMT_TYPE_NUM]; u16 fw_segment_type[MLX5_SGMT_TYPE_NUM];
}; };
@ -89,7 +89,7 @@ static int mlx5_rsc_dump_trigger(struct mlx5_core_dev *dev, struct mlx5_rsc_dump
return -ENOMEM; return -ENOMEM;
in_seq_num = MLX5_GET(resource_dump, cmd->cmd, seq_num); in_seq_num = MLX5_GET(resource_dump, cmd->cmd, seq_num);
MLX5_SET(resource_dump, cmd->cmd, mkey, rsc_dump->mkey.key); MLX5_SET(resource_dump, cmd->cmd, mkey, rsc_dump->mkey);
MLX5_SET64(resource_dump, cmd->cmd, address, dma); MLX5_SET64(resource_dump, cmd->cmd, address, dma);
err = mlx5_core_access_reg(dev, cmd->cmd, sizeof(cmd->cmd), cmd->cmd, err = mlx5_core_access_reg(dev, cmd->cmd, sizeof(cmd->cmd), cmd->cmd,
@ -202,7 +202,7 @@ free_page:
} }
static int mlx5_rsc_dump_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, static int mlx5_rsc_dump_create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
struct mlx5_core_mkey *mkey) u32 *mkey)
{ {
int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
void *mkc; void *mkc;
@ -276,7 +276,7 @@ int mlx5_rsc_dump_init(struct mlx5_core_dev *dev)
return err; return err;
destroy_mkey: destroy_mkey:
mlx5_core_destroy_mkey(dev, &rsc_dump->mkey); mlx5_core_destroy_mkey(dev, rsc_dump->mkey);
free_pd: free_pd:
mlx5_core_dealloc_pd(dev, rsc_dump->pdn); mlx5_core_dealloc_pd(dev, rsc_dump->pdn);
return err; return err;
@ -287,6 +287,6 @@ void mlx5_rsc_dump_cleanup(struct mlx5_core_dev *dev)
if (IS_ERR_OR_NULL(dev->rsc_dump)) if (IS_ERR_OR_NULL(dev->rsc_dump))
return; return;
mlx5_core_destroy_mkey(dev, &dev->rsc_dump->mkey); mlx5_core_destroy_mkey(dev, dev->rsc_dump->mkey);
mlx5_core_dealloc_pd(dev, dev->rsc_dump->pdn); mlx5_core_dealloc_pd(dev, dev->rsc_dump->pdn);
} }

4
drivers/net/ethernet/mellanox/mlx5/core/en.h
Просмотреть файл

@ -643,7 +643,7 @@ struct mlx5e_rq_frags_info {
}; };
struct mlx5e_shampo_hd { struct mlx5e_shampo_hd {
struct mlx5_core_mkey mkey; u32 mkey;
struct mlx5e_dma_info *info; struct mlx5e_dma_info *info;
struct page *last_page; struct page *last_page;
u16 hd_per_wq; u16 hd_per_wq;
@ -731,7 +731,7 @@ struct mlx5e_rq {
u8 wq_type; u8 wq_type;
u32 rqn; u32 rqn;
struct mlx5_core_dev *mdev; struct mlx5_core_dev *mdev;
struct mlx5_core_mkey umr_mkey; u32 umr_mkey;
struct mlx5e_dma_info wqe_overflow; struct mlx5e_dma_info wqe_overflow;
/* XDP read-mostly */ /* XDP read-mostly */

2
drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
Просмотреть файл

@ -682,7 +682,7 @@ int mlx5e_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params,
c->tstamp = &priv->tstamp; c->tstamp = &priv->tstamp;
c->pdev = mlx5_core_dma_dev(priv->mdev); c->pdev = mlx5_core_dma_dev(priv->mdev);
c->netdev = priv->netdev; c->netdev = priv->netdev;
c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey.key); c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey);
c->num_tc = mlx5e_get_dcb_num_tc(params); c->num_tc = mlx5e_get_dcb_num_tc(params);
c->stats = &priv->ptp_stats.ch; c->stats = &priv->ptp_stats.ch;
c->lag_port = lag_port; c->lag_port = lag_port;

2
drivers/net/ethernet/mellanox/mlx5/core/en/trap.c
Просмотреть файл

@ -137,7 +137,7 @@ static struct mlx5e_trap *mlx5e_open_trap(struct mlx5e_priv *priv)
t->tstamp = &priv->tstamp; t->tstamp = &priv->tstamp;
t->pdev = mlx5_core_dma_dev(priv->mdev); t->pdev = mlx5_core_dma_dev(priv->mdev);
t->netdev = priv->netdev; t->netdev = priv->netdev;
t->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey.key); t->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey);
t->stats = &priv->trap_stats.ch; t->stats = &priv->trap_stats.ch;
netif_napi_add(netdev, &t->napi, mlx5e_trap_napi_poll, 64); netif_napi_add(netdev, &t->napi, mlx5e_trap_napi_poll, 64);

6
drivers/net/ethernet/mellanox/mlx5/core/en_common.c
Просмотреть файл

@ -47,7 +47,7 @@ void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc)
} }
static int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, static int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
struct mlx5_core_mkey *mkey) u32 *mkey)
{ {
int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
void *mkc; void *mkc;
@ -108,7 +108,7 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev)
return 0; return 0;
err_destroy_mkey: err_destroy_mkey:
mlx5_core_destroy_mkey(mdev, &res->mkey); mlx5_core_destroy_mkey(mdev, res->mkey);
err_dealloc_transport_domain: err_dealloc_transport_domain:
mlx5_core_dealloc_transport_domain(mdev, res->td.tdn); mlx5_core_dealloc_transport_domain(mdev, res->td.tdn);
err_dealloc_pd: err_dealloc_pd:
@ -121,7 +121,7 @@ void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev)
struct mlx5e_hw_objs *res = &mdev->mlx5e_res.hw_objs; struct mlx5e_hw_objs *res = &mdev->mlx5e_res.hw_objs;
mlx5_free_bfreg(mdev, &res->bfreg); mlx5_free_bfreg(mdev, &res->bfreg);
mlx5_core_destroy_mkey(mdev, &res->mkey); mlx5_core_destroy_mkey(mdev, res->mkey);
mlx5_core_dealloc_transport_domain(mdev, res->td.tdn); mlx5_core_dealloc_transport_domain(mdev, res->td.tdn);
mlx5_core_dealloc_pd(mdev, res->pdn); mlx5_core_dealloc_pd(mdev, res->pdn);
memset(res, 0, sizeof(*res)); memset(res, 0, sizeof(*res));

23
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
Просмотреть файл

@ -273,8 +273,7 @@ static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
} }
static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev, static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev,
u64 npages, u8 page_shift, u64 npages, u8 page_shift, u32 *umr_mkey,
struct mlx5_core_mkey *umr_mkey,
dma_addr_t filler_addr) dma_addr_t filler_addr)
{ {
struct mlx5_mtt *mtt; struct mlx5_mtt *mtt;
@ -325,7 +324,7 @@ static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev,
static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev, static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev,
u64 nentries, u64 nentries,
struct mlx5_core_mkey *umr_mkey) u32 *umr_mkey)
{ {
int inlen; int inlen;
void *mkc; void *mkc;
@ -519,7 +518,7 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
goto err_hw_gro_data; goto err_hw_gro_data;
} }
rq->mpwqe.shampo->key = rq->mpwqe.shampo->key =
cpu_to_be32(rq->mpwqe.shampo->mkey.key); cpu_to_be32(rq->mpwqe.shampo->mkey);
rq->mpwqe.shampo->hd_per_wqe = rq->mpwqe.shampo->hd_per_wqe =
mlx5e_shampo_hd_per_wqe(mdev, params, rqp); mlx5e_shampo_hd_per_wqe(mdev, params, rqp);
wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz)); wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));
@ -530,7 +529,7 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
err_hw_gro_data: err_hw_gro_data:
mlx5e_rq_shampo_hd_info_free(rq); mlx5e_rq_shampo_hd_info_free(rq);
err_shampo_info: err_shampo_info:
mlx5_core_destroy_mkey(mdev, &rq->mpwqe.shampo->mkey); mlx5_core_destroy_mkey(mdev, rq->mpwqe.shampo->mkey);
err_shampo_hd: err_shampo_hd:
mlx5e_rq_shampo_hd_free(rq); mlx5e_rq_shampo_hd_free(rq);
out: out:
@ -544,7 +543,7 @@ static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq)
kvfree(rq->hw_gro_data); kvfree(rq->hw_gro_data);
mlx5e_rq_shampo_hd_info_free(rq); mlx5e_rq_shampo_hd_info_free(rq);
mlx5_core_destroy_mkey(rq->mdev, &rq->mpwqe.shampo->mkey); mlx5_core_destroy_mkey(rq->mdev, rq->mpwqe.shampo->mkey);
mlx5e_rq_shampo_hd_free(rq); mlx5e_rq_shampo_hd_free(rq);
} }
@ -600,7 +599,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
err = mlx5e_create_rq_umr_mkey(mdev, rq); err = mlx5e_create_rq_umr_mkey(mdev, rq);
if (err) if (err)
goto err_rq_drop_page; goto err_rq_drop_page;
rq->mkey_be = cpu_to_be32(rq->umr_mkey.key); rq->mkey_be = cpu_to_be32(rq->umr_mkey);
err = mlx5e_rq_alloc_mpwqe_info(rq, node); err = mlx5e_rq_alloc_mpwqe_info(rq, node);
if (err) if (err)
@ -637,7 +636,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
if (err) if (err)
goto err_rq_frags; goto err_rq_frags;
rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey.key); rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
} }
if (xsk) { if (xsk) {
@ -728,7 +727,7 @@ err_free_by_rq_type:
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
kvfree(rq->mpwqe.info); kvfree(rq->mpwqe.info);
err_rq_mkey: err_rq_mkey:
mlx5_core_destroy_mkey(mdev, &rq->umr_mkey); mlx5_core_destroy_mkey(mdev, rq->umr_mkey);
err_rq_drop_page: err_rq_drop_page:
mlx5e_free_mpwqe_rq_drop_page(rq); mlx5e_free_mpwqe_rq_drop_page(rq);
break; break;
@ -761,7 +760,7 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
switch (rq->wq_type) { switch (rq->wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
kvfree(rq->mpwqe.info); kvfree(rq->mpwqe.info);
mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey); mlx5_core_destroy_mkey(rq->mdev, rq->umr_mkey);
mlx5e_free_mpwqe_rq_drop_page(rq); mlx5e_free_mpwqe_rq_drop_page(rq);
mlx5e_rq_free_shampo(rq); mlx5e_rq_free_shampo(rq);
break; break;
@ -820,7 +819,7 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
MLX5_SET(wq, wq, log_headers_buffer_entry_num, MLX5_SET(wq, wq, log_headers_buffer_entry_num,
order_base_2(rq->mpwqe.shampo->hd_per_wq)); order_base_2(rq->mpwqe.shampo->hd_per_wq));
MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey.key); MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey);
} }
mlx5_fill_page_frag_array(&rq->wq_ctrl.buf, mlx5_fill_page_frag_array(&rq->wq_ctrl.buf,
@ -2205,7 +2204,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
c->cpu = cpu; c->cpu = cpu;
c->pdev = mlx5_core_dma_dev(priv->mdev); c->pdev = mlx5_core_dma_dev(priv->mdev);
c->netdev = priv->netdev; c->netdev = priv->netdev;
c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey.key); c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey);
c->num_tc = mlx5e_get_dcb_num_tc(params); c->num_tc = mlx5e_get_dcb_num_tc(params);
c->xdp = !!params->xdp_prog; c->xdp = !!params->xdp_prog;
c->stats = &priv->channel_stats[ix].ch; c->stats = &priv->channel_stats[ix].ch;

2
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
Просмотреть файл

@ -544,7 +544,7 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq,
{ {
struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
u16 entries, pi, i, header_offset, err, wqe_bbs, new_entries; u16 entries, pi, i, header_offset, err, wqe_bbs, new_entries;
u32 lkey = rq->mdev->mlx5e_res.hw_objs.mkey.key; u32 lkey = rq->mdev->mlx5e_res.hw_objs.mkey;
struct page *page = shampo->last_page; struct page *page = shampo->last_page;
u64 addr = shampo->last_addr; u64 addr = shampo->last_addr;
struct mlx5e_dma_info *dma_info; struct mlx5e_dma_info *dma_info;

10
drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
Просмотреть файл

@ -115,7 +115,7 @@ static int mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn *conn,
ix = conn->qp.rq.pc & (conn->qp.rq.size - 1); ix = conn->qp.rq.pc & (conn->qp.rq.size - 1);
data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix); data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix);
data->byte_count = cpu_to_be32(buf->sg[0].size); data->byte_count = cpu_to_be32(buf->sg[0].size);
data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key); data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey);
data->addr = cpu_to_be64(buf->sg[0].dma_addr); data->addr = cpu_to_be64(buf->sg[0].dma_addr);
conn->qp.rq.pc++; conn->qp.rq.pc++;
@ -155,7 +155,7 @@ static void mlx5_fpga_conn_post_send(struct mlx5_fpga_conn *conn,
if (!buf->sg[sgi].data) if (!buf->sg[sgi].data)
break; break;
data->byte_count = cpu_to_be32(buf->sg[sgi].size); data->byte_count = cpu_to_be32(buf->sg[sgi].size);
data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key); data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey);
data->addr = cpu_to_be64(buf->sg[sgi].dma_addr); data->addr = cpu_to_be64(buf->sg[sgi].dma_addr);
data++; data++;
size++; size++;
@ -221,7 +221,7 @@ static int mlx5_fpga_conn_post_recv_buf(struct mlx5_fpga_conn *conn)
} }
static int mlx5_fpga_conn_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, static int mlx5_fpga_conn_create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
struct mlx5_core_mkey *mkey) u32 *mkey)
{ {
int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
void *mkc; void *mkc;
@ -978,7 +978,7 @@ int mlx5_fpga_conn_device_init(struct mlx5_fpga_device *fdev)
mlx5_fpga_err(fdev, "create mkey failed, %d\n", err); mlx5_fpga_err(fdev, "create mkey failed, %d\n", err);
goto err_dealloc_pd; goto err_dealloc_pd;
} }
mlx5_fpga_dbg(fdev, "Created mkey 0x%x\n", fdev->conn_res.mkey.key); mlx5_fpga_dbg(fdev, "Created mkey 0x%x\n", fdev->conn_res.mkey);
return 0; return 0;
@ -994,7 +994,7 @@ out:
void mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device *fdev) void mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device *fdev)
{ {
mlx5_core_destroy_mkey(fdev->mdev, &fdev->conn_res.mkey); mlx5_core_destroy_mkey(fdev->mdev, fdev->conn_res.mkey);
mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn); mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn);
mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar); mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar);
mlx5_nic_vport_disable_roce(fdev->mdev); mlx5_nic_vport_disable_roce(fdev->mdev);

2
drivers/net/ethernet/mellanox/mlx5/core/fpga/core.h
Просмотреть файл

@ -54,7 +54,7 @@ struct mlx5_fpga_device {
/* QP Connection resources */ /* QP Connection resources */
struct { struct {
u32 pdn; u32 pdn;
struct mlx5_core_mkey mkey; u32 mkey;
struct mlx5_uars_page *uar; struct mlx5_uars_page *uar;
} conn_res; } conn_res;

54
drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
Просмотреть файл

@ -99,6 +99,9 @@
#define LEFTOVERS_NUM_LEVELS 1 #define LEFTOVERS_NUM_LEVELS 1
#define LEFTOVERS_NUM_PRIOS 1 #define LEFTOVERS_NUM_PRIOS 1
#define RDMA_RX_COUNTERS_PRIO_NUM_LEVELS 1
#define RDMA_TX_COUNTERS_PRIO_NUM_LEVELS 1
#define BY_PASS_PRIO_NUM_LEVELS 1 #define BY_PASS_PRIO_NUM_LEVELS 1
#define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\ #define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\
LEFTOVERS_NUM_PRIOS) LEFTOVERS_NUM_PRIOS)
@ -206,34 +209,63 @@ static struct init_tree_node egress_root_fs = {
} }
}; };
#define RDMA_RX_BYPASS_PRIO 0 enum {
#define RDMA_RX_KERNEL_PRIO 1 RDMA_RX_COUNTERS_PRIO,
RDMA_RX_BYPASS_PRIO,
RDMA_RX_KERNEL_PRIO,
};
#define RDMA_RX_BYPASS_MIN_LEVEL MLX5_BY_PASS_NUM_REGULAR_PRIOS
#define RDMA_RX_KERNEL_MIN_LEVEL (RDMA_RX_BYPASS_MIN_LEVEL + 1)
#define RDMA_RX_COUNTERS_MIN_LEVEL (RDMA_RX_KERNEL_MIN_LEVEL + 2)
static struct init_tree_node rdma_rx_root_fs = { static struct init_tree_node rdma_rx_root_fs = {
.type = FS_TYPE_NAMESPACE, .type = FS_TYPE_NAMESPACE,
.ar_size = 2, .ar_size = 3,
.children = (struct init_tree_node[]) { .children = (struct init_tree_node[]) {
[RDMA_RX_COUNTERS_PRIO] =
ADD_PRIO(0, RDMA_RX_COUNTERS_MIN_LEVEL, 0,
FS_CHAINING_CAPS,
ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
ADD_MULTIPLE_PRIO(MLX5_RDMA_RX_NUM_COUNTERS_PRIOS,
RDMA_RX_COUNTERS_PRIO_NUM_LEVELS))),
[RDMA_RX_BYPASS_PRIO] = [RDMA_RX_BYPASS_PRIO] =
ADD_PRIO(0, MLX5_BY_PASS_NUM_REGULAR_PRIOS, 0, ADD_PRIO(0, RDMA_RX_BYPASS_MIN_LEVEL, 0,
FS_CHAINING_CAPS, FS_CHAINING_CAPS,
ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_REGULAR_PRIOS, ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_REGULAR_PRIOS,
BY_PASS_PRIO_NUM_LEVELS))), BY_PASS_PRIO_NUM_LEVELS))),
[RDMA_RX_KERNEL_PRIO] = [RDMA_RX_KERNEL_PRIO] =
ADD_PRIO(0, MLX5_BY_PASS_NUM_REGULAR_PRIOS + 1, 0, ADD_PRIO(0, RDMA_RX_KERNEL_MIN_LEVEL, 0,
FS_CHAINING_CAPS, FS_CHAINING_CAPS,
ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN, ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN,
ADD_MULTIPLE_PRIO(1, 1))), ADD_MULTIPLE_PRIO(1, 1))),
} }
}; };
enum {
RDMA_TX_COUNTERS_PRIO,
RDMA_TX_BYPASS_PRIO,
};
#define RDMA_TX_BYPASS_MIN_LEVEL MLX5_BY_PASS_NUM_PRIOS
#define RDMA_TX_COUNTERS_MIN_LEVEL (RDMA_TX_BYPASS_MIN_LEVEL + 1)
static struct init_tree_node rdma_tx_root_fs = { static struct init_tree_node rdma_tx_root_fs = {
.type = FS_TYPE_NAMESPACE, .type = FS_TYPE_NAMESPACE,
.ar_size = 1, .ar_size = 2,
.children = (struct init_tree_node[]) { .children = (struct init_tree_node[]) {
ADD_PRIO(0, MLX5_BY_PASS_NUM_PRIOS, 0, [RDMA_TX_COUNTERS_PRIO] =
ADD_PRIO(0, RDMA_TX_COUNTERS_MIN_LEVEL, 0,
FS_CHAINING_CAPS,
ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
ADD_MULTIPLE_PRIO(MLX5_RDMA_TX_NUM_COUNTERS_PRIOS,
RDMA_TX_COUNTERS_PRIO_NUM_LEVELS))),
[RDMA_TX_BYPASS_PRIO] =
ADD_PRIO(0, RDMA_TX_BYPASS_MIN_LEVEL, 0,
FS_CHAINING_CAPS_RDMA_TX, FS_CHAINING_CAPS_RDMA_TX,
ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS, ADD_MULTIPLE_PRIO(RDMA_TX_BYPASS_MIN_LEVEL,
BY_PASS_PRIO_NUM_LEVELS))), BY_PASS_PRIO_NUM_LEVELS))),
} }
}; };
@ -2219,6 +2251,12 @@ struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
prio = RDMA_RX_KERNEL_PRIO; prio = RDMA_RX_KERNEL_PRIO;
} else if (type == MLX5_FLOW_NAMESPACE_RDMA_TX) { } else if (type == MLX5_FLOW_NAMESPACE_RDMA_TX) {
root_ns = steering->rdma_tx_root_ns; root_ns = steering->rdma_tx_root_ns;
} else if (type == MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS) {
root_ns = steering->rdma_rx_root_ns;
prio = RDMA_RX_COUNTERS_PRIO;
} else if (type == MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS) {
root_ns = steering->rdma_tx_root_ns;
prio = RDMA_TX_COUNTERS_PRIO;
} else { /* Must be NIC RX */ } else { /* Must be NIC RX */
root_ns = steering->root_ns; root_ns = steering->root_ns;
prio = type; prio = type;

27
drivers/net/ethernet/mellanox/mlx5/core/mr.c
Просмотреть файл

@ -35,13 +35,11 @@
#include <linux/mlx5/driver.h> #include <linux/mlx5/driver.h>
#include "mlx5_core.h" #include "mlx5_core.h"
int mlx5_core_create_mkey(struct mlx5_core_dev *dev, int mlx5_core_create_mkey(struct mlx5_core_dev *dev, u32 *mkey, u32 *in,
struct mlx5_core_mkey *mkey, int inlen)
u32 *in, int inlen)
{ {
u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {}; u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {};
u32 mkey_index; u32 mkey_index;
void *mkc;
int err; int err;
MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
@ -50,38 +48,33 @@ int mlx5_core_create_mkey(struct mlx5_core_dev *dev,
if (err) if (err)
return err; return err;
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index); mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index);
mkey->iova = MLX5_GET64(mkc, mkc, start_addr); *mkey = MLX5_GET(create_mkey_in, in, memory_key_mkey_entry.mkey_7_0) |
mkey->size = MLX5_GET64(mkc, mkc, len); mlx5_idx_to_mkey(mkey_index);
mkey->key = (u32)mlx5_mkey_variant(mkey->key) | mlx5_idx_to_mkey(mkey_index);
mkey->pd = MLX5_GET(mkc, mkc, pd);
init_waitqueue_head(&mkey->wait);
mlx5_core_dbg(dev, "out 0x%x, mkey 0x%x\n", mkey_index, mkey->key); mlx5_core_dbg(dev, "out 0x%x, mkey 0x%x\n", mkey_index, *mkey);
return 0; return 0;
} }
EXPORT_SYMBOL(mlx5_core_create_mkey); EXPORT_SYMBOL(mlx5_core_create_mkey);
int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, u32 mkey)
struct mlx5_core_mkey *mkey)
{ {
u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {}; u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {};
MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY); MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key)); MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey));
return mlx5_cmd_exec_in(dev, destroy_mkey, in); return mlx5_cmd_exec_in(dev, destroy_mkey, in);
} }
EXPORT_SYMBOL(mlx5_core_destroy_mkey); EXPORT_SYMBOL(mlx5_core_destroy_mkey);
int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mkey, int mlx5_core_query_mkey(struct mlx5_core_dev *dev, u32 mkey, u32 *out,
u32 *out, int outlen) int outlen)
{ {
u32 in[MLX5_ST_SZ_DW(query_mkey_in)] = {}; u32 in[MLX5_ST_SZ_DW(query_mkey_in)] = {};
memset(out, 0, outlen); memset(out, 0, outlen);
MLX5_SET(query_mkey_in, in, opcode, MLX5_CMD_OP_QUERY_MKEY); MLX5_SET(query_mkey_in, in, opcode, MLX5_CMD_OP_QUERY_MKEY);
MLX5_SET(query_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key)); MLX5_SET(query_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey));
return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen); return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
} }
EXPORT_SYMBOL(mlx5_core_query_mkey); EXPORT_SYMBOL(mlx5_core_query_mkey);

10
drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c
Просмотреть файл

@ -24,7 +24,7 @@ struct mlx5dr_icm_dm {
}; };
struct mlx5dr_icm_mr { struct mlx5dr_icm_mr {
struct mlx5_core_mkey mkey; u32 mkey;
struct mlx5dr_icm_dm dm; struct mlx5dr_icm_dm dm;
struct mlx5dr_domain *dmn; struct mlx5dr_domain *dmn;
size_t length; size_t length;
@ -33,7 +33,7 @@ struct mlx5dr_icm_mr {
static int dr_icm_create_dm_mkey(struct mlx5_core_dev *mdev, static int dr_icm_create_dm_mkey(struct mlx5_core_dev *mdev,
u32 pd, u64 length, u64 start_addr, int mode, u32 pd, u64 length, u64 start_addr, int mode,
struct mlx5_core_mkey *mkey) u32 *mkey)
{ {
u32 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); u32 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {}; u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {};
@ -116,7 +116,7 @@ dr_icm_pool_mr_create(struct mlx5dr_icm_pool *pool)
return icm_mr; return icm_mr;
free_mkey: free_mkey:
mlx5_core_destroy_mkey(mdev, &icm_mr->mkey); mlx5_core_destroy_mkey(mdev, icm_mr->mkey);
free_dm: free_dm:
mlx5_dm_sw_icm_dealloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0, mlx5_dm_sw_icm_dealloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0,
icm_mr->dm.addr, icm_mr->dm.obj_id); icm_mr->dm.addr, icm_mr->dm.obj_id);
@ -130,7 +130,7 @@ static void dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr *icm_mr)
struct mlx5_core_dev *mdev = icm_mr->dmn->mdev; struct mlx5_core_dev *mdev = icm_mr->dmn->mdev;
struct mlx5dr_icm_dm *dm = &icm_mr->dm; struct mlx5dr_icm_dm *dm = &icm_mr->dm;
mlx5_core_destroy_mkey(mdev, &icm_mr->mkey); mlx5_core_destroy_mkey(mdev, icm_mr->mkey);
mlx5_dm_sw_icm_dealloc(mdev, dm->type, dm->length, 0, mlx5_dm_sw_icm_dealloc(mdev, dm->type, dm->length, 0,
dm->addr, dm->obj_id); dm->addr, dm->obj_id);
kvfree(icm_mr); kvfree(icm_mr);
@ -252,7 +252,7 @@ dr_icm_chunk_create(struct mlx5dr_icm_pool *pool,
offset = mlx5dr_icm_pool_dm_type_to_entry_size(pool->icm_type) * seg; offset = mlx5dr_icm_pool_dm_type_to_entry_size(pool->icm_type) * seg;
chunk->rkey = buddy_mem_pool->icm_mr->mkey.key; chunk->rkey = buddy_mem_pool->icm_mr->mkey;
chunk->mr_addr = offset; chunk->mr_addr = offset;
chunk->icm_addr = chunk->icm_addr =
(uintptr_t)buddy_mem_pool->icm_mr->icm_start_addr + offset; (uintptr_t)buddy_mem_pool->icm_mr->icm_start_addr + offset;

11
drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c
Просмотреть файл

@ -350,7 +350,7 @@ static void dr_fill_data_segs(struct mlx5dr_send_ring *send_ring,
send_info->read.length = send_info->write.length; send_info->read.length = send_info->write.length;
/* Read into the same write area */ /* Read into the same write area */
send_info->read.addr = (uintptr_t)send_info->write.addr; send_info->read.addr = (uintptr_t)send_info->write.addr;
send_info->read.lkey = send_ring->mr->mkey.key; send_info->read.lkey = send_ring->mr->mkey;
if (send_ring->pending_wqe % send_ring->signal_th == 0) if (send_ring->pending_wqe % send_ring->signal_th == 0)
send_info->read.send_flags = IB_SEND_SIGNALED; send_info->read.send_flags = IB_SEND_SIGNALED;
@ -388,7 +388,7 @@ static int dr_postsend_icm_data(struct mlx5dr_domain *dmn,
(void *)(uintptr_t)send_info->write.addr, (void *)(uintptr_t)send_info->write.addr,
send_info->write.length); send_info->write.length);
send_info->write.addr = (uintptr_t)send_ring->mr->dma_addr + buff_offset; send_info->write.addr = (uintptr_t)send_ring->mr->dma_addr + buff_offset;
send_info->write.lkey = send_ring->mr->mkey.key; send_info->write.lkey = send_ring->mr->mkey;
} }
send_ring->tx_head++; send_ring->tx_head++;
@ -848,8 +848,7 @@ static void dr_destroy_cq(struct mlx5_core_dev *mdev, struct mlx5dr_cq *cq)
kfree(cq); kfree(cq);
} }
static int static int dr_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey)
dr_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, struct mlx5_core_mkey *mkey)
{ {
u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {}; u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {};
void *mkc; void *mkc;
@ -908,7 +907,7 @@ static struct mlx5dr_mr *dr_reg_mr(struct mlx5_core_dev *mdev,
static void dr_dereg_mr(struct mlx5_core_dev *mdev, struct mlx5dr_mr *mr) static void dr_dereg_mr(struct mlx5_core_dev *mdev, struct mlx5dr_mr *mr)
{ {
mlx5_core_destroy_mkey(mdev, &mr->mkey); mlx5_core_destroy_mkey(mdev, mr->mkey);
dma_unmap_single(mlx5_core_dma_dev(mdev), mr->dma_addr, mr->size, dma_unmap_single(mlx5_core_dma_dev(mdev), mr->dma_addr, mr->size,
DMA_BIDIRECTIONAL); DMA_BIDIRECTIONAL);
kfree(mr); kfree(mr);
@ -1039,7 +1038,7 @@ int mlx5dr_send_ring_force_drain(struct mlx5dr_domain *dmn)
send_info.write.lkey = 0; send_info.write.lkey = 0;
/* Using the sync_mr in order to write/read */ /* Using the sync_mr in order to write/read */
send_info.remote_addr = (uintptr_t)send_ring->sync_mr->addr; send_info.remote_addr = (uintptr_t)send_ring->sync_mr->addr;
send_info.rkey = send_ring->sync_mr->mkey.key; send_info.rkey = send_ring->sync_mr->mkey;
for (i = 0; i < num_of_sends_req; i++) { for (i = 0; i < num_of_sends_req; i++) {
ret = dr_postsend_icm_data(dmn, &send_info); ret = dr_postsend_icm_data(dmn, &send_info);

2
drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h
Просмотреть файл

@ -1264,7 +1264,7 @@ struct mlx5dr_cq {
struct mlx5dr_mr { struct mlx5dr_mr {
struct mlx5_core_dev *mdev; struct mlx5_core_dev *mdev;
struct mlx5_core_mkey mkey; u32 mkey;
dma_addr_t dma_addr; dma_addr_t dma_addr;
void *addr; void *addr;
size_t size; size_t size;

16
drivers/net/ethernet/mellanox/mlx5/core/uar.c
Просмотреть файл

@ -36,7 +36,7 @@
#include <linux/mlx5/driver.h> #include <linux/mlx5/driver.h>
#include "mlx5_core.h" #include "mlx5_core.h"
int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn) static int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn)
{ {
u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {}; u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {};
u32 in[MLX5_ST_SZ_DW(alloc_uar_in)] = {}; u32 in[MLX5_ST_SZ_DW(alloc_uar_in)] = {};
@ -44,13 +44,14 @@ int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn)
MLX5_SET(alloc_uar_in, in, opcode, MLX5_CMD_OP_ALLOC_UAR); MLX5_SET(alloc_uar_in, in, opcode, MLX5_CMD_OP_ALLOC_UAR);
err = mlx5_cmd_exec_inout(dev, alloc_uar, in, out); err = mlx5_cmd_exec_inout(dev, alloc_uar, in, out);
if (!err) if (err)
*uarn = MLX5_GET(alloc_uar_out, out, uar); return err;
return err;
}
EXPORT_SYMBOL(mlx5_cmd_alloc_uar);
int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn) *uarn = MLX5_GET(alloc_uar_out, out, uar);
return 0;
}
static int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn)
{ {
u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)] = {}; u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)] = {};
@ -58,7 +59,6 @@ int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn)
MLX5_SET(dealloc_uar_in, in, uar, uarn); MLX5_SET(dealloc_uar_in, in, uar, uarn);
return mlx5_cmd_exec_in(dev, dealloc_uar, in); return mlx5_cmd_exec_in(dev, dealloc_uar, in);
} }
EXPORT_SYMBOL(mlx5_cmd_free_uar);
static int uars_per_sys_page(struct mlx5_core_dev *mdev) static int uars_per_sys_page(struct mlx5_core_dev *mdev)
{ {

8
drivers/vdpa/mlx5/core/mlx5_vdpa.h
Просмотреть файл

@ -15,7 +15,7 @@ struct mlx5_vdpa_direct_mr {
u64 start; u64 start;
u64 end; u64 end;
u32 perm; u32 perm;
struct mlx5_core_mkey mr; u32 mr;
struct sg_table sg_head; struct sg_table sg_head;
int log_size; int log_size;
int nsg; int nsg;
@ -25,7 +25,7 @@ struct mlx5_vdpa_direct_mr {
}; };
struct mlx5_vdpa_mr { struct mlx5_vdpa_mr {
struct mlx5_core_mkey mkey; u32 mkey;
/* list of direct MRs descendants of this indirect mr */ /* list of direct MRs descendants of this indirect mr */
struct list_head head; struct list_head head;
@ -99,9 +99,9 @@ int mlx5_vdpa_alloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 *tdn);
void mlx5_vdpa_dealloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 tdn); void mlx5_vdpa_dealloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 tdn);
int mlx5_vdpa_alloc_resources(struct mlx5_vdpa_dev *mvdev); int mlx5_vdpa_alloc_resources(struct mlx5_vdpa_dev *mvdev);
void mlx5_vdpa_free_resources(struct mlx5_vdpa_dev *mvdev); void mlx5_vdpa_free_resources(struct mlx5_vdpa_dev *mvdev);
int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev *mvdev, struct mlx5_core_mkey *mkey, u32 *in, int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev *mvdev, u32 *mkey, u32 *in,
int inlen); int inlen);
int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, struct mlx5_core_mkey *mkey); int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, u32 mkey);
int mlx5_vdpa_handle_set_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb, int mlx5_vdpa_handle_set_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb,
bool *change_map); bool *change_map);
int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb); int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb);

8
drivers/vdpa/mlx5/core/mr.c
Просмотреть файл

@ -88,7 +88,7 @@ static int create_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct
static void destroy_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr) static void destroy_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
{ {
mlx5_vdpa_destroy_mkey(mvdev, &mr->mr); mlx5_vdpa_destroy_mkey(mvdev, mr->mr);
} }
static u64 map_start(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr) static u64 map_start(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)
@ -162,7 +162,7 @@ again:
} }
if (preve == dmr->start) { if (preve == dmr->start) {
klm->key = cpu_to_be32(dmr->mr.key); klm->key = cpu_to_be32(dmr->mr);
klm->bcount = cpu_to_be32(klm_bcount(dmr->end - dmr->start)); klm->bcount = cpu_to_be32(klm_bcount(dmr->end - dmr->start));
preve = dmr->end; preve = dmr->end;
} else { } else {
@ -217,7 +217,7 @@ static int create_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr
static void destroy_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey) static void destroy_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey)
{ {
mlx5_vdpa_destroy_mkey(mvdev, &mkey->mkey); mlx5_vdpa_destroy_mkey(mvdev, mkey->mkey);
} }
static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr, static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr,
@ -449,7 +449,7 @@ static int create_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
static void destroy_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr) static void destroy_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
{ {
mlx5_vdpa_destroy_mkey(mvdev, &mr->mkey); mlx5_vdpa_destroy_mkey(mvdev, mr->mkey);
} }
static int dup_iotlb(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *src) static int dup_iotlb(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *src)

13
drivers/vdpa/mlx5/core/resources.c
Просмотреть файл

@ -198,12 +198,11 @@ void mlx5_vdpa_dealloc_transport_domain(struct mlx5_vdpa_dev *mvdev, u32 tdn)
mlx5_cmd_exec_in(mvdev->mdev, dealloc_transport_domain, in); mlx5_cmd_exec_in(mvdev->mdev, dealloc_transport_domain, in);
} }
int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev *mvdev, struct mlx5_core_mkey *mkey, u32 *in, int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev *mvdev, u32 *mkey, u32 *in,
int inlen) int inlen)
{ {
u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {}; u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {};
u32 mkey_index; u32 mkey_index;
void *mkc;
int err; int err;
MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
@ -213,22 +212,18 @@ int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev *mvdev, struct mlx5_core_mkey *mk
if (err) if (err)
return err; return err;
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index); mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index);
mkey->iova = MLX5_GET64(mkc, mkc, start_addr); *mkey |= mlx5_idx_to_mkey(mkey_index);
mkey->size = MLX5_GET64(mkc, mkc, len);
mkey->key |= mlx5_idx_to_mkey(mkey_index);
mkey->pd = MLX5_GET(mkc, mkc, pd);
return 0; return 0;
} }
int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, struct mlx5_core_mkey *mkey) int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, u32 mkey)
{ {
u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {}; u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {};
MLX5_SET(destroy_mkey_in, in, uid, mvdev->res.uid); MLX5_SET(destroy_mkey_in, in, uid, mvdev->res.uid);
MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY); MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key)); MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey));
return mlx5_cmd_exec_in(mvdev->mdev, destroy_mkey, in); return mlx5_cmd_exec_in(mvdev->mdev, destroy_mkey, in);
} }

2
drivers/vdpa/mlx5/net/mlx5_vnet.c
Просмотреть файл

@ -865,7 +865,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr);
MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr);
MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr);
MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey.key); MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey);
MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id); MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id);
MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size); MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size);
MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id); MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id);

2
include/linux/mlx5/device.h
Просмотреть файл

@ -1493,6 +1493,8 @@ static inline u16 mlx5_to_sw_pkey_sz(int pkey_sz)
return MLX5_MIN_PKEY_TABLE_SIZE << pkey_sz; return MLX5_MIN_PKEY_TABLE_SIZE << pkey_sz;
} }
#define MLX5_RDMA_RX_NUM_COUNTERS_PRIOS 2
#define MLX5_RDMA_TX_NUM_COUNTERS_PRIOS 1
#define MLX5_BY_PASS_NUM_REGULAR_PRIOS 16 #define MLX5_BY_PASS_NUM_REGULAR_PRIOS 16
#define MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS 16 #define MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS 16
#define MLX5_BY_PASS_NUM_MULTICAST_PRIOS 1 #define MLX5_BY_PASS_NUM_MULTICAST_PRIOS 1

32
include/linux/mlx5/driver.h
Просмотреть файл

@ -357,22 +357,6 @@ struct mlx5_core_sig_ctx {
u32 sigerr_count; u32 sigerr_count;
}; };
enum {
MLX5_MKEY_MR = 1,
MLX5_MKEY_MW,
MLX5_MKEY_INDIRECT_DEVX,
};
struct mlx5_core_mkey {
u64 iova;
u64 size;
u32 key;
u32 pd;
u32 type;
struct wait_queue_head wait;
refcount_t usecount;
};
#define MLX5_24BIT_MASK ((1 << 24) - 1) #define MLX5_24BIT_MASK ((1 << 24) - 1)
enum mlx5_res_type { enum mlx5_res_type {
@ -654,7 +638,7 @@ struct mlx5e_resources {
struct mlx5e_hw_objs { struct mlx5e_hw_objs {
u32 pdn; u32 pdn;
struct mlx5_td td; struct mlx5_td td;
struct mlx5_core_mkey mkey; u32 mkey;
struct mlx5_sq_bfreg bfreg; struct mlx5_sq_bfreg bfreg;
} hw_objs; } hw_objs;
struct devlink_port dl_port; struct devlink_port dl_port;
@ -1007,8 +991,6 @@ void mlx5_cmd_mbox_status(void *out, u8 *status, u32 *syndrome);
bool mlx5_cmd_is_down(struct mlx5_core_dev *dev); bool mlx5_cmd_is_down(struct mlx5_core_dev *dev);
int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type); int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type);
int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn);
int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn);
void mlx5_health_flush(struct mlx5_core_dev *dev); void mlx5_health_flush(struct mlx5_core_dev *dev);
void mlx5_health_cleanup(struct mlx5_core_dev *dev); void mlx5_health_cleanup(struct mlx5_core_dev *dev);
int mlx5_health_init(struct mlx5_core_dev *dev); int mlx5_health_init(struct mlx5_core_dev *dev);
@ -1026,13 +1008,11 @@ struct mlx5_cmd_mailbox *mlx5_alloc_cmd_mailbox_chain(struct mlx5_core_dev *dev,
gfp_t flags, int npages); gfp_t flags, int npages);
void mlx5_free_cmd_mailbox_chain(struct mlx5_core_dev *dev, void mlx5_free_cmd_mailbox_chain(struct mlx5_core_dev *dev,
struct mlx5_cmd_mailbox *head); struct mlx5_cmd_mailbox *head);
int mlx5_core_create_mkey(struct mlx5_core_dev *dev, int mlx5_core_create_mkey(struct mlx5_core_dev *dev, u32 *mkey, u32 *in,
struct mlx5_core_mkey *mkey, int inlen);
u32 *in, int inlen); int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, u32 mkey);
int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, int mlx5_core_query_mkey(struct mlx5_core_dev *dev, u32 mkey, u32 *out,
struct mlx5_core_mkey *mkey); int outlen);
int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mkey,
u32 *out, int outlen);
int mlx5_core_alloc_pd(struct mlx5_core_dev *dev, u32 *pdn); int mlx5_core_alloc_pd(struct mlx5_core_dev *dev, u32 *pdn);
int mlx5_core_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn); int mlx5_core_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn);
int mlx5_pagealloc_init(struct mlx5_core_dev *dev); int mlx5_pagealloc_init(struct mlx5_core_dev *dev);

2
include/linux/mlx5/fs.h
Просмотреть файл

@ -84,6 +84,8 @@ enum mlx5_flow_namespace_type {
MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL, MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL,
MLX5_FLOW_NAMESPACE_RDMA_TX, MLX5_FLOW_NAMESPACE_RDMA_TX,
MLX5_FLOW_NAMESPACE_PORT_SEL, MLX5_FLOW_NAMESPACE_PORT_SEL,
MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS,
MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS,
}; };
enum { enum {

26
include/linux/mlx5/mlx5_ifc.h
Просмотреть файл

@ -343,7 +343,7 @@ struct mlx5_ifc_flow_table_fields_supported_bits {
u8 outer_geneve_oam[0x1]; u8 outer_geneve_oam[0x1];
u8 outer_geneve_protocol_type[0x1]; u8 outer_geneve_protocol_type[0x1];
u8 outer_geneve_opt_len[0x1]; u8 outer_geneve_opt_len[0x1];
u8 reserved_at_1e[0x1]; u8 source_vhca_port[0x1];
u8 source_eswitch_port[0x1]; u8 source_eswitch_port[0x1];
u8 inner_dmac[0x1]; u8 inner_dmac[0x1];
@ -394,6 +394,14 @@ struct mlx5_ifc_flow_table_fields_supported_bits {
u8 metadata_reg_c_0[0x1]; u8 metadata_reg_c_0[0x1];
}; };
struct mlx5_ifc_flow_table_fields_supported_2_bits {
u8 reserved_at_0[0xe];
u8 bth_opcode[0x1];
u8 reserved_at_f[0x11];
u8 reserved_at_20[0x60];
};
struct mlx5_ifc_flow_table_prop_layout_bits { struct mlx5_ifc_flow_table_prop_layout_bits {
u8 ft_support[0x1]; u8 ft_support[0x1];
u8 reserved_at_1[0x1]; u8 reserved_at_1[0x1];
@ -540,7 +548,7 @@ struct mlx5_ifc_fte_match_set_misc_bits {
union mlx5_ifc_gre_key_bits gre_key; union mlx5_ifc_gre_key_bits gre_key;
u8 vxlan_vni[0x18]; u8 vxlan_vni[0x18];
u8 reserved_at_b8[0x8]; u8 bth_opcode[0x8];
u8 geneve_vni[0x18]; u8 geneve_vni[0x18];
u8 reserved_at_d8[0x7]; u8 reserved_at_d8[0x7];
@ -757,7 +765,15 @@ struct mlx5_ifc_flow_table_nic_cap_bits {
struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_nic_transmit_sniffer; struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_nic_transmit_sniffer;
u8 reserved_at_e00[0x1200]; u8 reserved_at_e00[0x700];
struct mlx5_ifc_flow_table_fields_supported_2_bits ft_field_support_2_nic_receive_rdma;
u8 reserved_at_1580[0x280];
struct mlx5_ifc_flow_table_fields_supported_2_bits ft_field_support_2_nic_transmit_rdma;
u8 reserved_at_1880[0x780];
u8 sw_steering_nic_rx_action_drop_icm_address[0x40]; u8 sw_steering_nic_rx_action_drop_icm_address[0x40];
@ -7907,7 +7923,7 @@ struct mlx5_ifc_dealloc_uar_out_bits {
struct mlx5_ifc_dealloc_uar_in_bits { struct mlx5_ifc_dealloc_uar_in_bits {
u8 opcode[0x10]; u8 opcode[0x10];
u8 reserved_at_10[0x10]; u8 uid[0x10];
u8 reserved_at_20[0x10]; u8 reserved_at_20[0x10];
u8 op_mod[0x10]; u8 op_mod[0x10];
@ -8764,7 +8780,7 @@ struct mlx5_ifc_alloc_uar_out_bits {
struct mlx5_ifc_alloc_uar_in_bits { struct mlx5_ifc_alloc_uar_in_bits {
u8 opcode[0x10]; u8 opcode[0x10];
u8 reserved_at_10[0x10]; u8 uid[0x10];
u8 reserved_at_20[0x10]; u8 reserved_at_20[0x10];
u8 op_mod[0x10]; u8 op_mod[0x10];