locking/mutexes: Refactor optimistic spinning code
When we fail to acquire the mutex in the fastpath, we end up calling __mutex_lock_common(). A *lot* goes on in this function. Move out the optimistic spinning code into mutex_optimistic_spin() and simplify the former a bit. Furthermore, this is similar to what we have in rwsems. No logical changes. Signed-off-by: Davidlohr Bueso <davidlohr@hp.com> Acked-by: Jason Low <jason.low2@hp.com> Signed-off-by: Peter Zijlstra <peterz@infradead.org> Cc: aswin@hp.com Cc: mingo@kernel.org Cc: Linus Torvalds <torvalds@linux-foundation.org> Link: http://lkml.kernel.org/r/1406752916-3341-4-git-send-email-davidlohr@hp.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Davidlohr Bueso
Ingo Molnar
Родитель
aa9fc0c19b
Коммит
76916515d9
|
|
@ -106,6 +106,92 @@ void __sched mutex_lock(struct mutex *lock)
|
||||||
EXPORT_SYMBOL(mutex_lock);
|
EXPORT_SYMBOL(mutex_lock);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
|
||||||
|
struct ww_acquire_ctx *ww_ctx)
|
||||||
|
{
|
||||||
|
#ifdef CONFIG_DEBUG_MUTEXES
|
||||||
|
/*
|
||||||
|
* If this WARN_ON triggers, you used ww_mutex_lock to acquire,
|
||||||
|
* but released with a normal mutex_unlock in this call.
|
||||||
|
*
|
||||||
|
* This should never happen, always use ww_mutex_unlock.
|
||||||
|
*/
|
||||||
|
DEBUG_LOCKS_WARN_ON(ww->ctx);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Not quite done after calling ww_acquire_done() ?
|
||||||
|
*/
|
||||||
|
DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);
|
||||||
|
|
||||||
|
if (ww_ctx->contending_lock) {
|
||||||
|
/*
|
||||||
|
* After -EDEADLK you tried to
|
||||||
|
* acquire a different ww_mutex? Bad!
|
||||||
|
*/
|
||||||
|
DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* You called ww_mutex_lock after receiving -EDEADLK,
|
||||||
|
* but 'forgot' to unlock everything else first?
|
||||||
|
*/
|
||||||
|
DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
|
||||||
|
ww_ctx->contending_lock = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Naughty, using a different class will lead to undefined behavior!
|
||||||
|
*/
|
||||||
|
DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
|
||||||
|
#endif
|
||||||
|
ww_ctx->acquired++;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* after acquiring lock with fastpath or when we lost out in contested
|
||||||
|
* slowpath, set ctx and wake up any waiters so they can recheck.
|
||||||
|
*
|
||||||
|
* This function is never called when CONFIG_DEBUG_LOCK_ALLOC is set,
|
||||||
|
* as the fastpath and opportunistic spinning are disabled in that case.
|
||||||
|
*/
|
||||||
|
static __always_inline void
|
||||||
|
ww_mutex_set_context_fastpath(struct ww_mutex *lock,
|
||||||
|
struct ww_acquire_ctx *ctx)
|
||||||
|
{
|
||||||
|
unsigned long flags;
|
||||||
|
struct mutex_waiter *cur;
|
||||||
|
|
||||||
|
ww_mutex_lock_acquired(lock, ctx);
|
||||||
|
|
||||||
|
lock->ctx = ctx;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The lock->ctx update should be visible on all cores before
|
||||||
|
* the atomic read is done, otherwise contended waiters might be
|
||||||
|
* missed. The contended waiters will either see ww_ctx == NULL
|
||||||
|
* and keep spinning, or it will acquire wait_lock, add itself
|
||||||
|
* to waiter list and sleep.
|
||||||
|
*/
|
||||||
|
smp_mb(); /* ^^^ */
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Check if lock is contended, if not there is nobody to wake up
|
||||||
|
*/
|
||||||
|
if (likely(atomic_read(&lock->base.count) == 0))
|
||||||
|
return;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Uh oh, we raced in fastpath, wake up everyone in this case,
|
||||||
|
* so they can see the new lock->ctx.
|
||||||
|
*/
|
||||||
|
spin_lock_mutex(&lock->base.wait_lock, flags);
|
||||||
|
list_for_each_entry(cur, &lock->base.wait_list, list) {
|
||||||
|
debug_mutex_wake_waiter(&lock->base, cur);
|
||||||
|
wake_up_process(cur->task);
|
||||||
|
}
|
||||||
|
spin_unlock_mutex(&lock->base.wait_lock, flags);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
|
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
|
||||||
/*
|
/*
|
||||||
* In order to avoid a stampede of mutex spinners from acquiring the mutex
|
* In order to avoid a stampede of mutex spinners from acquiring the mutex
|
||||||
|
|
@ -180,6 +266,129 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock)
|
||||||
*/
|
*/
|
||||||
return retval;
|
return retval;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Atomically try to take the lock when it is available
|
||||||
|
*/
|
||||||
|
static inline bool mutex_try_to_acquire(struct mutex *lock)
|
||||||
|
{
|
||||||
|
return !mutex_is_locked(lock) &&
|
||||||
|
(atomic_cmpxchg(&lock->count, 1, 0) == 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Optimistic spinning.
|
||||||
|
*
|
||||||
|
* We try to spin for acquisition when we find that the lock owner
|
||||||
|
* is currently running on a (different) CPU and while we don't
|
||||||
|
* need to reschedule. The rationale is that if the lock owner is
|
||||||
|
* running, it is likely to release the lock soon.
|
||||||
|
*
|
||||||
|
* Since this needs the lock owner, and this mutex implementation
|
||||||
|
* doesn't track the owner atomically in the lock field, we need to
|
||||||
|
* track it non-atomically.
|
||||||
|
*
|
||||||
|
* We can't do this for DEBUG_MUTEXES because that relies on wait_lock
|
||||||
|
* to serialize everything.
|
||||||
|
*
|
||||||
|
* The mutex spinners are queued up using MCS lock so that only one
|
||||||
|
* spinner can compete for the mutex. However, if mutex spinning isn't
|
||||||
|
* going to happen, there is no point in going through the lock/unlock
|
||||||
|
* overhead.
|
||||||
|
*
|
||||||
|
* Returns true when the lock was taken, otherwise false, indicating
|
||||||
|
* that we need to jump to the slowpath and sleep.
|
||||||
|
*/
|
||||||
|
static bool mutex_optimistic_spin(struct mutex *lock,
|
||||||
|
struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
|
||||||
|
{
|
||||||
|
struct task_struct *task = current;
|
||||||
|
|
||||||
|
if (!mutex_can_spin_on_owner(lock))
|
||||||
|
goto done;
|
||||||
|
|
||||||
|
if (!osq_lock(&lock->osq))
|
||||||
|
goto done;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
struct task_struct *owner;
|
||||||
|
|
||||||
|
if (use_ww_ctx && ww_ctx->acquired > 0) {
|
||||||
|
struct ww_mutex *ww;
|
||||||
|
|
||||||
|
ww = container_of(lock, struct ww_mutex, base);
|
||||||
|
/*
|
||||||
|
* If ww->ctx is set the contents are undefined, only
|
||||||
|
* by acquiring wait_lock there is a guarantee that
|
||||||
|
* they are not invalid when reading.
|
||||||
|
*
|
||||||
|
* As such, when deadlock detection needs to be
|
||||||
|
* performed the optimistic spinning cannot be done.
|
||||||
|
*/
|
||||||
|
if (ACCESS_ONCE(ww->ctx))
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* If there's an owner, wait for it to either
|
||||||
|
* release the lock or go to sleep.
|
||||||
|
*/
|
||||||
|
owner = ACCESS_ONCE(lock->owner);
|
||||||
|
if (owner && !mutex_spin_on_owner(lock, owner))
|
||||||
|
break;
|
||||||
|
|
||||||
|
/* Try to acquire the mutex if it is unlocked. */
|
||||||
|
if (mutex_try_to_acquire(lock)) {
|
||||||
|
lock_acquired(&lock->dep_map, ip);
|
||||||
|
|
||||||
|
if (use_ww_ctx) {
|
||||||
|
struct ww_mutex *ww;
|
||||||
|
ww = container_of(lock, struct ww_mutex, base);
|
||||||
|
|
||||||
|
ww_mutex_set_context_fastpath(ww, ww_ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
mutex_set_owner(lock);
|
||||||
|
osq_unlock(&lock->osq);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* When there's no owner, we might have preempted between the
|
||||||
|
* owner acquiring the lock and setting the owner field. If
|
||||||
|
* we're an RT task that will live-lock because we won't let
|
||||||
|
* the owner complete.
|
||||||
|
*/
|
||||||
|
if (!owner && (need_resched() || rt_task(task)))
|
||||||
|
break;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The cpu_relax() call is a compiler barrier which forces
|
||||||
|
* everything in this loop to be re-loaded. We don't need
|
||||||
|
* memory barriers as we'll eventually observe the right
|
||||||
|
* values at the cost of a few extra spins.
|
||||||
|
*/
|
||||||
|
cpu_relax_lowlatency();
|
||||||
|
}
|
||||||
|
|
||||||
|
osq_unlock(&lock->osq);
|
||||||
|
done:
|
||||||
|
/*
|
||||||
|
* If we fell out of the spin path because of need_resched(),
|
||||||
|
* reschedule now, before we try-lock the mutex. This avoids getting
|
||||||
|
* scheduled out right after we obtained the mutex.
|
||||||
|
*/
|
||||||
|
if (need_resched())
|
||||||
|
schedule_preempt_disabled();
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
static bool mutex_optimistic_spin(struct mutex *lock,
|
||||||
|
struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
__visible __used noinline
|
__visible __used noinline
|
||||||
|
|
@ -277,91 +486,6 @@ __mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
|
|
||||||
struct ww_acquire_ctx *ww_ctx)
|
|
||||||
{
|
|
||||||
#ifdef CONFIG_DEBUG_MUTEXES
|
|
||||||
/*
|
|
||||||
* If this WARN_ON triggers, you used ww_mutex_lock to acquire,
|
|
||||||
* but released with a normal mutex_unlock in this call.
|
|
||||||
*
|
|
||||||
* This should never happen, always use ww_mutex_unlock.
|
|
||||||
*/
|
|
||||||
DEBUG_LOCKS_WARN_ON(ww->ctx);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Not quite done after calling ww_acquire_done() ?
|
|
||||||
*/
|
|
||||||
DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);
|
|
||||||
|
|
||||||
if (ww_ctx->contending_lock) {
|
|
||||||
/*
|
|
||||||
* After -EDEADLK you tried to
|
|
||||||
* acquire a different ww_mutex? Bad!
|
|
||||||
*/
|
|
||||||
DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* You called ww_mutex_lock after receiving -EDEADLK,
|
|
||||||
* but 'forgot' to unlock everything else first?
|
|
||||||
*/
|
|
||||||
DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
|
|
||||||
ww_ctx->contending_lock = NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Naughty, using a different class will lead to undefined behavior!
|
|
||||||
*/
|
|
||||||
DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
|
|
||||||
#endif
|
|
||||||
ww_ctx->acquired++;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* after acquiring lock with fastpath or when we lost out in contested
|
|
||||||
* slowpath, set ctx and wake up any waiters so they can recheck.
|
|
||||||
*
|
|
||||||
* This function is never called when CONFIG_DEBUG_LOCK_ALLOC is set,
|
|
||||||
* as the fastpath and opportunistic spinning are disabled in that case.
|
|
||||||
*/
|
|
||||||
static __always_inline void
|
|
||||||
ww_mutex_set_context_fastpath(struct ww_mutex *lock,
|
|
||||||
struct ww_acquire_ctx *ctx)
|
|
||||||
{
|
|
||||||
unsigned long flags;
|
|
||||||
struct mutex_waiter *cur;
|
|
||||||
|
|
||||||
ww_mutex_lock_acquired(lock, ctx);
|
|
||||||
|
|
||||||
lock->ctx = ctx;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* The lock->ctx update should be visible on all cores before
|
|
||||||
* the atomic read is done, otherwise contended waiters might be
|
|
||||||
* missed. The contended waiters will either see ww_ctx == NULL
|
|
||||||
* and keep spinning, or it will acquire wait_lock, add itself
|
|
||||||
* to waiter list and sleep.
|
|
||||||
*/
|
|
||||||
smp_mb(); /* ^^^ */
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Check if lock is contended, if not there is nobody to wake up
|
|
||||||
*/
|
|
||||||
if (likely(atomic_read(&lock->base.count) == 0))
|
|
||||||
return;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Uh oh, we raced in fastpath, wake up everyone in this case,
|
|
||||||
* so they can see the new lock->ctx.
|
|
||||||
*/
|
|
||||||
spin_lock_mutex(&lock->base.wait_lock, flags);
|
|
||||||
list_for_each_entry(cur, &lock->base.wait_list, list) {
|
|
||||||
debug_mutex_wake_waiter(&lock->base, cur);
|
|
||||||
wake_up_process(cur->task);
|
|
||||||
}
|
|
||||||
spin_unlock_mutex(&lock->base.wait_lock, flags);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Lock a mutex (possibly interruptible), slowpath:
|
* Lock a mutex (possibly interruptible), slowpath:
|
||||||
*/
|
*/
|
||||||
|
|
@ -378,104 +502,12 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
|
||||||
preempt_disable();
|
preempt_disable();
|
||||||
mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
|
mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
|
||||||
|
|
||||||
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
|
if (mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx)) {
|
||||||
/*
|
/* got the lock, yay! */
|
||||||
* Optimistic spinning.
|
preempt_enable();
|
||||||
*
|
return 0;
|
||||||
* We try to spin for acquisition when we find that the lock owner
|
|
||||||
* is currently running on a (different) CPU and while we don't
|
|
||||||
* need to reschedule. The rationale is that if the lock owner is
|
|
||||||
* running, it is likely to release the lock soon.
|
|
||||||
*
|
|
||||||
* Since this needs the lock owner, and this mutex implementation
|
|
||||||
* doesn't track the owner atomically in the lock field, we need to
|
|
||||||
* track it non-atomically.
|
|
||||||
*
|
|
||||||
* We can't do this for DEBUG_MUTEXES because that relies on wait_lock
|
|
||||||
* to serialize everything.
|
|
||||||
*
|
|
||||||
* The mutex spinners are queued up using MCS lock so that only one
|
|
||||||
* spinner can compete for the mutex. However, if mutex spinning isn't
|
|
||||||
* going to happen, there is no point in going through the lock/unlock
|
|
||||||
* overhead.
|
|
||||||
*/
|
|
||||||
if (!mutex_can_spin_on_owner(lock))
|
|
||||||
goto slowpath;
|
|
||||||
|
|
||||||
if (!osq_lock(&lock->osq))
|
|
||||||
goto slowpath;
|
|
||||||
|
|
||||||
for (;;) {
|
|
||||||
struct task_struct *owner;
|
|
||||||
|
|
||||||
if (use_ww_ctx && ww_ctx->acquired > 0) {
|
|
||||||
struct ww_mutex *ww;
|
|
||||||
|
|
||||||
ww = container_of(lock, struct ww_mutex, base);
|
|
||||||
/*
|
|
||||||
* If ww->ctx is set the contents are undefined, only
|
|
||||||
* by acquiring wait_lock there is a guarantee that
|
|
||||||
* they are not invalid when reading.
|
|
||||||
*
|
|
||||||
* As such, when deadlock detection needs to be
|
|
||||||
* performed the optimistic spinning cannot be done.
|
|
||||||
*/
|
|
||||||
if (ACCESS_ONCE(ww->ctx))
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* If there's an owner, wait for it to either
|
|
||||||
* release the lock or go to sleep.
|
|
||||||
*/
|
|
||||||
owner = ACCESS_ONCE(lock->owner);
|
|
||||||
if (owner && !mutex_spin_on_owner(lock, owner))
|
|
||||||
break;
|
|
||||||
|
|
||||||
/* Try to acquire the mutex if it is unlocked. */
|
|
||||||
if (!mutex_is_locked(lock) &&
|
|
||||||
(atomic_cmpxchg(&lock->count, 1, 0) == 1)) {
|
|
||||||
lock_acquired(&lock->dep_map, ip);
|
|
||||||
if (use_ww_ctx) {
|
|
||||||
struct ww_mutex *ww;
|
|
||||||
ww = container_of(lock, struct ww_mutex, base);
|
|
||||||
|
|
||||||
ww_mutex_set_context_fastpath(ww, ww_ctx);
|
|
||||||
}
|
|
||||||
|
|
||||||
mutex_set_owner(lock);
|
|
||||||
osq_unlock(&lock->osq);
|
|
||||||
preempt_enable();
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* When there's no owner, we might have preempted between the
|
|
||||||
* owner acquiring the lock and setting the owner field. If
|
|
||||||
* we're an RT task that will live-lock because we won't let
|
|
||||||
* the owner complete.
|
|
||||||
*/
|
|
||||||
if (!owner && (need_resched() || rt_task(task)))
|
|
||||||
break;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* The cpu_relax() call is a compiler barrier which forces
|
|
||||||
* everything in this loop to be re-loaded. We don't need
|
|
||||||
* memory barriers as we'll eventually observe the right
|
|
||||||
* values at the cost of a few extra spins.
|
|
||||||
*/
|
|
||||||
cpu_relax_lowlatency();
|
|
||||||
}
|
}
|
||||||
osq_unlock(&lock->osq);
|
|
||||||
slowpath:
|
|
||||||
/*
|
|
||||||
* If we fell out of the spin path because of need_resched(),
|
|
||||||
* reschedule now, before we try-lock the mutex. This avoids getting
|
|
||||||
* scheduled out right after we obtained the mutex.
|
|
||||||
*/
|
|
||||||
if (need_resched())
|
|
||||||
schedule_preempt_disabled();
|
|
||||||
#endif
|
|
||||||
spin_lock_mutex(&lock->wait_lock, flags);
|
spin_lock_mutex(&lock->wait_lock, flags);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
||||||
Загрузка…
Ссылка в новой задаче