IB/rdmavt: Use per-CPU reference count for MRs
Having per-CPU reference count for each MR prevents cache-line bouncing across the system. Thus, it prevents bottlenecks. Use per-CPU reference counts per MR. The per-CPU reference count for FMRs is used in atomic mode to allow accurate testing of the busy state. Other MR types run in per-CPU mode MR until they're freed. Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com> Signed-off-by: Sebastian Sanchez <sebastian.sanchez@intel.com> Signed-off-by: Mike Marciniszyn <mike.marciniszyn@intel.com> Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com> Signed-off-by: Doug Ledford <dledford@redhat.com>
This commit is contained in:
Sebastian Sanchez
Doug Ledford
Родитель
f3e862cb68
Коммит
338adfdddf
|
|
@ -120,10 +120,19 @@ static void rvt_deinit_mregion(struct rvt_mregion *mr)
|
|||
mr->mapsz = 0;
|
||||
while (i)
|
||||
kfree(mr->map[--i]);
|
||||
percpu_ref_exit(&mr->refcount);
|
||||
}
|
||||
|
||||
static void __rvt_mregion_complete(struct percpu_ref *ref)
|
||||
{
|
||||
struct rvt_mregion *mr = container_of(ref, struct rvt_mregion,
|
||||
refcount);
|
||||
|
||||
complete(&mr->comp);
|
||||
}
|
||||
|
||||
static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
|
||||
int count)
|
||||
int count, unsigned int percpu_flags)
|
||||
{
|
||||
int m, i = 0;
|
||||
struct rvt_dev_info *dev = ib_to_rvt(pd->device);
|
||||
|
|
@ -133,19 +142,23 @@ static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
|
|||
for (; i < m; i++) {
|
||||
mr->map[i] = kzalloc_node(sizeof(*mr->map[0]), GFP_KERNEL,
|
||||
dev->dparms.node);
|
||||
if (!mr->map[i]) {
|
||||
rvt_deinit_mregion(mr);
|
||||
return -ENOMEM;
|
||||
}
|
||||
if (!mr->map[i])
|
||||
goto bail;
|
||||
mr->mapsz++;
|
||||
}
|
||||
init_completion(&mr->comp);
|
||||
/* count returning the ptr to user */
|
||||
atomic_set(&mr->refcount, 1);
|
||||
if (percpu_ref_init(&mr->refcount, &__rvt_mregion_complete,
|
||||
percpu_flags, GFP_KERNEL))
|
||||
goto bail;
|
||||
|
||||
atomic_set(&mr->lkey_invalid, 0);
|
||||
mr->pd = pd;
|
||||
mr->max_segs = count;
|
||||
return 0;
|
||||
bail:
|
||||
rvt_deinit_mregion(mr);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -180,8 +193,7 @@ static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region)
|
|||
if (!tmr) {
|
||||
rcu_assign_pointer(dev->dma_mr, mr);
|
||||
mr->lkey_published = 1;
|
||||
} else {
|
||||
rvt_put_mr(mr);
|
||||
rvt_get_mr(mr);
|
||||
}
|
||||
goto success;
|
||||
}
|
||||
|
|
@ -239,11 +251,14 @@ static void rvt_free_lkey(struct rvt_mregion *mr)
|
|||
int freed = 0;
|
||||
|
||||
spin_lock_irqsave(&rkt->lock, flags);
|
||||
if (!mr->lkey_published)
|
||||
goto out;
|
||||
if (lkey == 0) {
|
||||
RCU_INIT_POINTER(dev->dma_mr, NULL);
|
||||
if (!lkey) {
|
||||
if (mr->lkey_published) {
|
||||
RCU_INIT_POINTER(dev->dma_mr, NULL);
|
||||
rvt_put_mr(mr);
|
||||
}
|
||||
} else {
|
||||
if (!mr->lkey_published)
|
||||
goto out;
|
||||
r = lkey >> (32 - dev->dparms.lkey_table_size);
|
||||
RCU_INIT_POINTER(rkt->table[r], NULL);
|
||||
}
|
||||
|
|
@ -253,7 +268,7 @@ out:
|
|||
spin_unlock_irqrestore(&rkt->lock, flags);
|
||||
if (freed) {
|
||||
synchronize_rcu();
|
||||
rvt_put_mr(mr);
|
||||
percpu_ref_kill(&mr->refcount);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -269,7 +284,7 @@ static struct rvt_mr *__rvt_alloc_mr(int count, struct ib_pd *pd)
|
|||
if (!mr)
|
||||
goto bail;
|
||||
|
||||
rval = rvt_init_mregion(&mr->mr, pd, count);
|
||||
rval = rvt_init_mregion(&mr->mr, pd, count, 0);
|
||||
if (rval)
|
||||
goto bail;
|
||||
/*
|
||||
|
|
@ -294,8 +309,8 @@ bail:
|
|||
|
||||
static void __rvt_free_mr(struct rvt_mr *mr)
|
||||
{
|
||||
rvt_deinit_mregion(&mr->mr);
|
||||
rvt_free_lkey(&mr->mr);
|
||||
rvt_deinit_mregion(&mr->mr);
|
||||
kfree(mr);
|
||||
}
|
||||
|
||||
|
|
@ -323,7 +338,7 @@ struct ib_mr *rvt_get_dma_mr(struct ib_pd *pd, int acc)
|
|||
goto bail;
|
||||
}
|
||||
|
||||
rval = rvt_init_mregion(&mr->mr, pd, 0);
|
||||
rval = rvt_init_mregion(&mr->mr, pd, 0, 0);
|
||||
if (rval) {
|
||||
ret = ERR_PTR(rval);
|
||||
goto bail;
|
||||
|
|
@ -445,8 +460,8 @@ int rvt_dereg_mr(struct ib_mr *ibmr)
|
|||
timeout = wait_for_completion_timeout(&mr->mr.comp, 5 * HZ);
|
||||
if (!timeout) {
|
||||
rvt_pr_err(rdi,
|
||||
"rvt_dereg_mr timeout mr %p pd %p refcount %u\n",
|
||||
mr, mr->mr.pd, atomic_read(&mr->mr.refcount));
|
||||
"rvt_dereg_mr timeout mr %p pd %p\n",
|
||||
mr, mr->mr.pd);
|
||||
rvt_get_mr(&mr->mr);
|
||||
ret = -EBUSY;
|
||||
goto out;
|
||||
|
|
@ -623,7 +638,8 @@ struct ib_fmr *rvt_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
|
|||
if (!fmr)
|
||||
goto bail;
|
||||
|
||||
rval = rvt_init_mregion(&fmr->mr, pd, fmr_attr->max_pages);
|
||||
rval = rvt_init_mregion(&fmr->mr, pd, fmr_attr->max_pages,
|
||||
PERCPU_REF_INIT_ATOMIC);
|
||||
if (rval)
|
||||
goto bail;
|
||||
|
||||
|
|
@ -674,11 +690,12 @@ int rvt_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
|
|||
struct rvt_fmr *fmr = to_ifmr(ibfmr);
|
||||
struct rvt_lkey_table *rkt;
|
||||
unsigned long flags;
|
||||
int m, n, i;
|
||||
int m, n;
|
||||
unsigned long i;
|
||||
u32 ps;
|
||||
struct rvt_dev_info *rdi = ib_to_rvt(ibfmr->device);
|
||||
|
||||
i = atomic_read(&fmr->mr.refcount);
|
||||
i = atomic_long_read(&fmr->mr.refcount.count);
|
||||
if (i > 2)
|
||||
return -EBUSY;
|
||||
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@
|
|||
* For Memory Regions. This stuff should probably be moved into rdmavt/mr.h once
|
||||
* drivers no longer need access to the MR directly.
|
||||
*/
|
||||
#include <linux/percpu-refcount.h>
|
||||
|
||||
/*
|
||||
* A segment is a linear region of low physical memory.
|
||||
|
|
@ -79,11 +80,11 @@ struct rvt_mregion {
|
|||
int access_flags;
|
||||
u32 max_segs; /* number of rvt_segs in all the arrays */
|
||||
u32 mapsz; /* size of the map array */
|
||||
atomic_t lkey_invalid; /* true if current lkey is invalid */
|
||||
u8 page_shift; /* 0 - non unform/non powerof2 sizes */
|
||||
u8 lkey_published; /* in global table */
|
||||
atomic_t lkey_invalid; /* true if current lkey is invalid */
|
||||
struct percpu_ref refcount;
|
||||
struct completion comp; /* complete when refcount goes to zero */
|
||||
atomic_t refcount;
|
||||
struct rvt_segarray *map[0]; /* the segments */
|
||||
};
|
||||
|
||||
|
|
@ -123,13 +124,12 @@ struct rvt_sge_state {
|
|||
|
||||
static inline void rvt_put_mr(struct rvt_mregion *mr)
|
||||
{
|
||||
if (unlikely(atomic_dec_and_test(&mr->refcount)))
|
||||
complete(&mr->comp);
|
||||
percpu_ref_put(&mr->refcount);
|
||||
}
|
||||
|
||||
static inline void rvt_get_mr(struct rvt_mregion *mr)
|
||||
{
|
||||
atomic_inc(&mr->refcount);
|
||||
percpu_ref_get(&mr->refcount);
|
||||
}
|
||||
|
||||
static inline void rvt_put_ss(struct rvt_sge_state *ss)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче