memcg: replace ss->id_lock with a rwlock
While back-porting Johannes Weiner's patch "mm: memcg-aware global reclaim" for an internal effort, we noticed a significant performance regression during page-reclaim heavy workloads due to high contention of the ss->id_lock. This lock protects idr map, and serializes calls to idr_get_next() in css_get_next() (which is used during the memcg hierarchy walk). Since idr_get_next() is just doing a look up, we need only serialize it with respect to idr_remove()/idr_get_new(). By making the ss->id_lock a rwlock, contention is greatly reduced and performance improves. Tested: cat a 256m file from a ramdisk in a 128m container 50 times on each core (one file + container per core) in parallel on a NUMA machine. Result is the time for the test to complete in 1 of the containers. Both kernels included Johannes' memcg-aware global reclaim patches. Before rwlock patch: 1710.778s After rwlock patch: 152.227s Signed-off-by: Andrew Bresticker <abrestic@google.com> Cc: Paul Menage <menage@gmail.com> Cc: Li Zefan <lizf@cn.fujitsu.com> Acked-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Cc: Ying Han <yinghan@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Родитель
080d676de0
Коммит
c1e2ee2dc4
|
@ -516,7 +516,7 @@ struct cgroup_subsys {
|
|||
struct list_head sibling;
|
||||
/* used when use_id == true */
|
||||
struct idr idr;
|
||||
spinlock_t id_lock;
|
||||
rwlock_t id_lock;
|
||||
|
||||
/* should be defined only by modular subsystems */
|
||||
struct module *module;
|
||||
|
|
|
@ -4883,9 +4883,9 @@ void free_css_id(struct cgroup_subsys *ss, struct cgroup_subsys_state *css)
|
|||
|
||||
rcu_assign_pointer(id->css, NULL);
|
||||
rcu_assign_pointer(css->id, NULL);
|
||||
spin_lock(&ss->id_lock);
|
||||
write_lock(&ss->id_lock);
|
||||
idr_remove(&ss->idr, id->id);
|
||||
spin_unlock(&ss->id_lock);
|
||||
write_unlock(&ss->id_lock);
|
||||
kfree_rcu(id, rcu_head);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(free_css_id);
|
||||
|
@ -4911,10 +4911,10 @@ static struct css_id *get_new_cssid(struct cgroup_subsys *ss, int depth)
|
|||
error = -ENOMEM;
|
||||
goto err_out;
|
||||
}
|
||||
spin_lock(&ss->id_lock);
|
||||
write_lock(&ss->id_lock);
|
||||
/* Don't use 0. allocates an ID of 1-65535 */
|
||||
error = idr_get_new_above(&ss->idr, newid, 1, &myid);
|
||||
spin_unlock(&ss->id_lock);
|
||||
write_unlock(&ss->id_lock);
|
||||
|
||||
/* Returns error when there are no free spaces for new ID.*/
|
||||
if (error) {
|
||||
|
@ -4929,9 +4929,9 @@ static struct css_id *get_new_cssid(struct cgroup_subsys *ss, int depth)
|
|||
return newid;
|
||||
remove_idr:
|
||||
error = -ENOSPC;
|
||||
spin_lock(&ss->id_lock);
|
||||
write_lock(&ss->id_lock);
|
||||
idr_remove(&ss->idr, myid);
|
||||
spin_unlock(&ss->id_lock);
|
||||
write_unlock(&ss->id_lock);
|
||||
err_out:
|
||||
kfree(newid);
|
||||
return ERR_PTR(error);
|
||||
|
@ -4943,7 +4943,7 @@ static int __init_or_module cgroup_init_idr(struct cgroup_subsys *ss,
|
|||
{
|
||||
struct css_id *newid;
|
||||
|
||||
spin_lock_init(&ss->id_lock);
|
||||
rwlock_init(&ss->id_lock);
|
||||
idr_init(&ss->idr);
|
||||
|
||||
newid = get_new_cssid(ss, 0);
|
||||
|
@ -5038,9 +5038,9 @@ css_get_next(struct cgroup_subsys *ss, int id,
|
|||
* scan next entry from bitmap(tree), tmpid is updated after
|
||||
* idr_get_next().
|
||||
*/
|
||||
spin_lock(&ss->id_lock);
|
||||
read_lock(&ss->id_lock);
|
||||
tmp = idr_get_next(&ss->idr, &tmpid);
|
||||
spin_unlock(&ss->id_lock);
|
||||
read_unlock(&ss->id_lock);
|
||||
|
||||
if (!tmp)
|
||||
break;
|
||||
|
|
Загрузка…
Ссылка в новой задаче