diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 10846b9656aa..031682e7ef0c 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -157,12 +157,46 @@ struct page_cgroup { struct list_head lru; /* per cgroup LRU list */ struct page *page; struct mem_cgroup *mem_cgroup; - int flags; + unsigned long flags; }; -#define PAGE_CGROUP_FLAG_CACHE (0x1) /* charged as cache */ -#define PAGE_CGROUP_FLAG_ACTIVE (0x2) /* page is active in this cgroup */ -#define PAGE_CGROUP_FLAG_FILE (0x4) /* page is file system backed */ -#define PAGE_CGROUP_FLAG_UNEVICTABLE (0x8) /* page is unevictableable */ + +enum { + /* flags for mem_cgroup */ + PCG_CACHE, /* charged as cache */ + /* flags for LRU placement */ + PCG_ACTIVE, /* page is active in this cgroup */ + PCG_FILE, /* page is file system backed */ + PCG_UNEVICTABLE, /* page is unevictableable */ +}; + +#define TESTPCGFLAG(uname, lname) \ +static inline int PageCgroup##uname(struct page_cgroup *pc) \ + { return test_bit(PCG_##lname, &pc->flags); } + +#define SETPCGFLAG(uname, lname) \ +static inline void SetPageCgroup##uname(struct page_cgroup *pc)\ + { set_bit(PCG_##lname, &pc->flags); } + +#define CLEARPCGFLAG(uname, lname) \ +static inline void ClearPageCgroup##uname(struct page_cgroup *pc) \ + { clear_bit(PCG_##lname, &pc->flags); } + + +/* Cache flag is set only once (at allocation) */ +TESTPCGFLAG(Cache, CACHE) + +/* LRU management flags (from global-lru definition) */ +TESTPCGFLAG(File, FILE) +SETPCGFLAG(File, FILE) +CLEARPCGFLAG(File, FILE) + +TESTPCGFLAG(Active, ACTIVE) +SETPCGFLAG(Active, ACTIVE) +CLEARPCGFLAG(Active, ACTIVE) + +TESTPCGFLAG(Unevictable, UNEVICTABLE) +SETPCGFLAG(Unevictable, UNEVICTABLE) +CLEARPCGFLAG(Unevictable, UNEVICTABLE) static int page_cgroup_nid(struct page_cgroup *pc) { @@ -177,15 +211,25 @@ static enum zone_type page_cgroup_zid(struct page_cgroup *pc) enum charge_type { MEM_CGROUP_CHARGE_TYPE_CACHE = 0, MEM_CGROUP_CHARGE_TYPE_MAPPED, - MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */ MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */ + MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */ + NR_CHARGE_TYPE, +}; + +static const unsigned long +pcg_default_flags[NR_CHARGE_TYPE] = { + ((1 << PCG_CACHE) | (1 << PCG_FILE)), + ((1 << PCG_ACTIVE)), + ((1 << PCG_ACTIVE) | (1 << PCG_CACHE)), + 0, }; /* * Always modified under lru lock. Then, not necessary to preempt_disable() */ -static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, int flags, - bool charge) +static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, + struct page_cgroup *pc, + bool charge) { int val = (charge)? 1 : -1; struct mem_cgroup_stat *stat = &mem->stat; @@ -194,7 +238,7 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, int flags, VM_BUG_ON(!irqs_disabled()); cpustat = &stat->cpustat[smp_processor_id()]; - if (flags & PAGE_CGROUP_FLAG_CACHE) + if (PageCgroupCache(pc)) __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_CACHE, val); else __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_RSS, val); @@ -295,18 +339,18 @@ static void __mem_cgroup_remove_list(struct mem_cgroup_per_zone *mz, { int lru = LRU_BASE; - if (pc->flags & PAGE_CGROUP_FLAG_UNEVICTABLE) + if (PageCgroupUnevictable(pc)) lru = LRU_UNEVICTABLE; else { - if (pc->flags & PAGE_CGROUP_FLAG_ACTIVE) + if (PageCgroupActive(pc)) lru += LRU_ACTIVE; - if (pc->flags & PAGE_CGROUP_FLAG_FILE) + if (PageCgroupFile(pc)) lru += LRU_FILE; } MEM_CGROUP_ZSTAT(mz, lru) -= 1; - mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, false); + mem_cgroup_charge_statistics(pc->mem_cgroup, pc, false); list_del(&pc->lru); } @@ -315,27 +359,27 @@ static void __mem_cgroup_add_list(struct mem_cgroup_per_zone *mz, { int lru = LRU_BASE; - if (pc->flags & PAGE_CGROUP_FLAG_UNEVICTABLE) + if (PageCgroupUnevictable(pc)) lru = LRU_UNEVICTABLE; else { - if (pc->flags & PAGE_CGROUP_FLAG_ACTIVE) + if (PageCgroupActive(pc)) lru += LRU_ACTIVE; - if (pc->flags & PAGE_CGROUP_FLAG_FILE) + if (PageCgroupFile(pc)) lru += LRU_FILE; } MEM_CGROUP_ZSTAT(mz, lru) += 1; list_add(&pc->lru, &mz->lists[lru]); - mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, true); + mem_cgroup_charge_statistics(pc->mem_cgroup, pc, true); } static void __mem_cgroup_move_lists(struct page_cgroup *pc, enum lru_list lru) { struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc); - int active = pc->flags & PAGE_CGROUP_FLAG_ACTIVE; - int file = pc->flags & PAGE_CGROUP_FLAG_FILE; - int unevictable = pc->flags & PAGE_CGROUP_FLAG_UNEVICTABLE; + int active = PageCgroupActive(pc); + int file = PageCgroupFile(pc); + int unevictable = PageCgroupUnevictable(pc); enum lru_list from = unevictable ? LRU_UNEVICTABLE : (LRU_FILE * !!file + !!active); @@ -343,16 +387,20 @@ static void __mem_cgroup_move_lists(struct page_cgroup *pc, enum lru_list lru) return; MEM_CGROUP_ZSTAT(mz, from) -= 1; - + /* + * However this is done under mz->lru_lock, another flags, which + * are not related to LRU, will be modified from out-of-lock. + * We have to use atomic set/clear flags. + */ if (is_unevictable_lru(lru)) { - pc->flags &= ~PAGE_CGROUP_FLAG_ACTIVE; - pc->flags |= PAGE_CGROUP_FLAG_UNEVICTABLE; + ClearPageCgroupActive(pc); + SetPageCgroupUnevictable(pc); } else { if (is_active_lru(lru)) - pc->flags |= PAGE_CGROUP_FLAG_ACTIVE; + SetPageCgroupActive(pc); else - pc->flags &= ~PAGE_CGROUP_FLAG_ACTIVE; - pc->flags &= ~PAGE_CGROUP_FLAG_UNEVICTABLE; + ClearPageCgroupActive(pc); + ClearPageCgroupUnevictable(pc); } MEM_CGROUP_ZSTAT(mz, lru) += 1; @@ -589,16 +637,7 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm, * If a page is accounted as a page cache, insert to inactive list. * If anon, insert to active list. */ - if (ctype == MEM_CGROUP_CHARGE_TYPE_CACHE) { - pc->flags = PAGE_CGROUP_FLAG_CACHE; - if (page_is_file_cache(page)) - pc->flags |= PAGE_CGROUP_FLAG_FILE; - else - pc->flags |= PAGE_CGROUP_FLAG_ACTIVE; - } else if (ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED) - pc->flags = PAGE_CGROUP_FLAG_ACTIVE; - else /* MEM_CGROUP_CHARGE_TYPE_SHMEM */ - pc->flags = PAGE_CGROUP_FLAG_CACHE | PAGE_CGROUP_FLAG_ACTIVE; + pc->flags = pcg_default_flags[ctype]; lock_page_cgroup(page); if (unlikely(page_get_page_cgroup(page))) { @@ -677,8 +716,12 @@ int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, if (unlikely(!mm)) mm = &init_mm; - return mem_cgroup_charge_common(page, mm, gfp_mask, + if (page_is_file_cache(page)) + return mem_cgroup_charge_common(page, mm, gfp_mask, MEM_CGROUP_CHARGE_TYPE_CACHE, NULL); + else + return mem_cgroup_charge_common(page, mm, gfp_mask, + MEM_CGROUP_CHARGE_TYPE_SHMEM, NULL); } /* @@ -706,8 +749,7 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) VM_BUG_ON(pc->page != page); if ((ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED) - && ((pc->flags & PAGE_CGROUP_FLAG_CACHE) - || page_mapped(page))) + && ((PageCgroupCache(pc) || page_mapped(page)))) goto unlock; mz = page_cgroup_zoneinfo(pc); @@ -758,7 +800,7 @@ int mem_cgroup_prepare_migration(struct page *page, struct page *newpage) if (pc) { mem = pc->mem_cgroup; css_get(&mem->css); - if (pc->flags & PAGE_CGROUP_FLAG_CACHE) { + if (PageCgroupCache(pc)) { if (page_is_file_cache(page)) ctype = MEM_CGROUP_CHARGE_TYPE_CACHE; else