refscale: Add tests using SLAB_TYPESAFE_BY_RCU
This commit adds three read-side-only tests of three use cases featuring SLAB_TYPESAFE_BY_RCU: One using per-object reference counting, one using per-object locking, and one using per-object sequence locking. [ paulmck: Apply feedback from kernel test robot. ] Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
Paul E. McKenney
Родитель
3c6496c86e
Коммит
a6889becb0
|
|
@ -76,6 +76,8 @@ torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");
|
|||
// Wait until there are multiple CPUs before starting test.
|
||||
torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
|
||||
"Holdoff time before test start (s)");
|
||||
// Number of typesafe_lookup structures, that is, the degree of concurrency.
|
||||
torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures.");
|
||||
// Number of loops per experiment, all readers execute operations concurrently.
|
||||
torture_param(long, loops, 10000, "Number of loops per experiment.");
|
||||
// Number of readers, with -1 defaulting to about 75% of the CPUs.
|
||||
|
|
@ -526,6 +528,237 @@ static struct ref_scale_ops clock_ops = {
|
|||
.name = "clock"
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Methods leveraging SLAB_TYPESAFE_BY_RCU.
|
||||
//
|
||||
|
||||
// Item to look up in a typesafe manner. Array of pointers to these.
|
||||
struct refscale_typesafe {
|
||||
atomic_t rts_refctr; // Used by all flavors
|
||||
spinlock_t rts_lock;
|
||||
seqlock_t rts_seqlock;
|
||||
unsigned int a;
|
||||
unsigned int b;
|
||||
};
|
||||
|
||||
static struct kmem_cache *typesafe_kmem_cachep;
|
||||
static struct refscale_typesafe **rtsarray;
|
||||
static long rtsarray_size;
|
||||
static DEFINE_TORTURE_RANDOM_PERCPU(refscale_rand);
|
||||
static bool (*rts_acquire)(struct refscale_typesafe *rtsp, unsigned int *start);
|
||||
static bool (*rts_release)(struct refscale_typesafe *rtsp, unsigned int start);
|
||||
|
||||
// Conditionally acquire an explicit in-structure reference count.
|
||||
static bool typesafe_ref_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
|
||||
{
|
||||
return atomic_inc_not_zero(&rtsp->rts_refctr);
|
||||
}
|
||||
|
||||
// Unconditionally release an explicit in-structure reference count.
|
||||
static bool typesafe_ref_release(struct refscale_typesafe *rtsp, unsigned int start)
|
||||
{
|
||||
if (!atomic_dec_return(&rtsp->rts_refctr)) {
|
||||
WRITE_ONCE(rtsp->a, rtsp->a + 1);
|
||||
kmem_cache_free(typesafe_kmem_cachep, rtsp);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Unconditionally acquire an explicit in-structure spinlock.
|
||||
static bool typesafe_lock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
|
||||
{
|
||||
spin_lock(&rtsp->rts_lock);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Unconditionally release an explicit in-structure spinlock.
|
||||
static bool typesafe_lock_release(struct refscale_typesafe *rtsp, unsigned int start)
|
||||
{
|
||||
spin_unlock(&rtsp->rts_lock);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Unconditionally acquire an explicit in-structure sequence lock.
|
||||
static bool typesafe_seqlock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
|
||||
{
|
||||
*start = read_seqbegin(&rtsp->rts_seqlock);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Conditionally release an explicit in-structure sequence lock. Return
|
||||
// true if this release was successful, that is, if no retry is required.
|
||||
static bool typesafe_seqlock_release(struct refscale_typesafe *rtsp, unsigned int start)
|
||||
{
|
||||
return !read_seqretry(&rtsp->rts_seqlock, start);
|
||||
}
|
||||
|
||||
// Do a read-side critical section with the specified delay in
|
||||
// microseconds and nanoseconds inserted so as to increase probability
|
||||
// of failure.
|
||||
static void typesafe_delay_section(const int nloops, const int udl, const int ndl)
|
||||
{
|
||||
unsigned int a;
|
||||
unsigned int b;
|
||||
int i;
|
||||
long idx;
|
||||
struct refscale_typesafe *rtsp;
|
||||
unsigned int start;
|
||||
|
||||
for (i = nloops; i >= 0; i--) {
|
||||
preempt_disable();
|
||||
idx = torture_random(this_cpu_ptr(&refscale_rand)) % rtsarray_size;
|
||||
preempt_enable();
|
||||
retry:
|
||||
rcu_read_lock();
|
||||
rtsp = rcu_dereference(rtsarray[idx]);
|
||||
a = READ_ONCE(rtsp->a);
|
||||
if (!rts_acquire(rtsp, &start)) {
|
||||
rcu_read_unlock();
|
||||
goto retry;
|
||||
}
|
||||
if (a != READ_ONCE(rtsp->a)) {
|
||||
(void)rts_release(rtsp, start);
|
||||
rcu_read_unlock();
|
||||
goto retry;
|
||||
}
|
||||
un_delay(udl, ndl);
|
||||
// Remember, seqlock read-side release can fail.
|
||||
if (!rts_release(rtsp, start)) {
|
||||
rcu_read_unlock();
|
||||
goto retry;
|
||||
}
|
||||
b = READ_ONCE(rtsp->a);
|
||||
WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b);
|
||||
b = rtsp->b;
|
||||
rcu_read_unlock();
|
||||
WARN_ON_ONCE(a * a != b);
|
||||
}
|
||||
}
|
||||
|
||||
// Because the acquisition and release methods are expensive, there
|
||||
// is no point in optimizing away the un_delay() function's two checks.
|
||||
// Thus simply define typesafe_read_section() as a simple wrapper around
|
||||
// typesafe_delay_section().
|
||||
static void typesafe_read_section(const int nloops)
|
||||
{
|
||||
typesafe_delay_section(nloops, 0, 0);
|
||||
}
|
||||
|
||||
// Allocate and initialize one refscale_typesafe structure.
|
||||
static struct refscale_typesafe *typesafe_alloc_one(void)
|
||||
{
|
||||
struct refscale_typesafe *rtsp;
|
||||
|
||||
rtsp = kmem_cache_alloc(typesafe_kmem_cachep, GFP_KERNEL);
|
||||
if (!rtsp)
|
||||
return NULL;
|
||||
atomic_set(&rtsp->rts_refctr, 1);
|
||||
WRITE_ONCE(rtsp->a, rtsp->a + 1);
|
||||
WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a);
|
||||
return rtsp;
|
||||
}
|
||||
|
||||
// Slab-allocator constructor for refscale_typesafe structures created
|
||||
// out of a new slab of system memory.
|
||||
static void refscale_typesafe_ctor(void *rtsp_in)
|
||||
{
|
||||
struct refscale_typesafe *rtsp = rtsp_in;
|
||||
|
||||
spin_lock_init(&rtsp->rts_lock);
|
||||
seqlock_init(&rtsp->rts_seqlock);
|
||||
preempt_disable();
|
||||
rtsp->a = torture_random(this_cpu_ptr(&refscale_rand));
|
||||
preempt_enable();
|
||||
}
|
||||
|
||||
static struct ref_scale_ops typesafe_ref_ops;
|
||||
static struct ref_scale_ops typesafe_lock_ops;
|
||||
static struct ref_scale_ops typesafe_seqlock_ops;
|
||||
|
||||
// Initialize for a typesafe test.
|
||||
static bool typesafe_init(void)
|
||||
{
|
||||
long idx;
|
||||
long si = lookup_instances;
|
||||
|
||||
typesafe_kmem_cachep = kmem_cache_create("refscale_typesafe",
|
||||
sizeof(struct refscale_typesafe), sizeof(void *),
|
||||
SLAB_TYPESAFE_BY_RCU, refscale_typesafe_ctor);
|
||||
if (!typesafe_kmem_cachep)
|
||||
return false;
|
||||
if (si < 0)
|
||||
si = -si * nr_cpu_ids;
|
||||
else if (si == 0)
|
||||
si = nr_cpu_ids;
|
||||
rtsarray_size = si;
|
||||
rtsarray = kcalloc(si, sizeof(*rtsarray), GFP_KERNEL);
|
||||
if (!rtsarray)
|
||||
return false;
|
||||
for (idx = 0; idx < rtsarray_size; idx++) {
|
||||
rtsarray[idx] = typesafe_alloc_one();
|
||||
if (!rtsarray[idx])
|
||||
return false;
|
||||
}
|
||||
if (cur_ops == &typesafe_ref_ops) {
|
||||
rts_acquire = typesafe_ref_acquire;
|
||||
rts_release = typesafe_ref_release;
|
||||
} else if (cur_ops == &typesafe_lock_ops) {
|
||||
rts_acquire = typesafe_lock_acquire;
|
||||
rts_release = typesafe_lock_release;
|
||||
} else if (cur_ops == &typesafe_seqlock_ops) {
|
||||
rts_acquire = typesafe_seqlock_acquire;
|
||||
rts_release = typesafe_seqlock_release;
|
||||
} else {
|
||||
WARN_ON_ONCE(1);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Clean up after a typesafe test.
|
||||
static void typesafe_cleanup(void)
|
||||
{
|
||||
long idx;
|
||||
|
||||
if (rtsarray) {
|
||||
for (idx = 0; idx < rtsarray_size; idx++)
|
||||
kmem_cache_free(typesafe_kmem_cachep, rtsarray[idx]);
|
||||
kfree(rtsarray);
|
||||
rtsarray = NULL;
|
||||
rtsarray_size = 0;
|
||||
}
|
||||
kmem_cache_destroy(typesafe_kmem_cachep);
|
||||
typesafe_kmem_cachep = NULL;
|
||||
rts_acquire = NULL;
|
||||
rts_release = NULL;
|
||||
}
|
||||
|
||||
// The typesafe_init() function distinguishes these structures by address.
|
||||
static struct ref_scale_ops typesafe_ref_ops = {
|
||||
.init = typesafe_init,
|
||||
.cleanup = typesafe_cleanup,
|
||||
.readsection = typesafe_read_section,
|
||||
.delaysection = typesafe_delay_section,
|
||||
.name = "typesafe_ref"
|
||||
};
|
||||
|
||||
static struct ref_scale_ops typesafe_lock_ops = {
|
||||
.init = typesafe_init,
|
||||
.cleanup = typesafe_cleanup,
|
||||
.readsection = typesafe_read_section,
|
||||
.delaysection = typesafe_delay_section,
|
||||
.name = "typesafe_lock"
|
||||
};
|
||||
|
||||
static struct ref_scale_ops typesafe_seqlock_ops = {
|
||||
.init = typesafe_init,
|
||||
.cleanup = typesafe_cleanup,
|
||||
.readsection = typesafe_read_section,
|
||||
.delaysection = typesafe_delay_section,
|
||||
.name = "typesafe_seqlock"
|
||||
};
|
||||
|
||||
static void rcu_scale_one_reader(void)
|
||||
{
|
||||
if (readdelay <= 0)
|
||||
|
|
@ -815,6 +1048,7 @@ ref_scale_init(void)
|
|||
static struct ref_scale_ops *scale_ops[] = {
|
||||
&rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops,
|
||||
&rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops,
|
||||
&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
|
||||
};
|
||||
|
||||
if (!torture_init_begin(scale_type, verbose))
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче