hugetlb: move refcounting in hugepage allocation inside hugetlb_lock
Currently alloc_huge_page() raises page refcount outside hugetlb_lock. but it causes race when dequeue_hwpoison_huge_page() runs concurrently with alloc_huge_page(). To avoid it, this patch moves set_page_refcounted() in hugetlb_lock. Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> Signed-off-by: Wu Fengguang <fengguang.wu@intel.com> Acked-by: Mel Gorman <mel@csn.ul.ie> Reviewed-by: Christoph Lameter <cl@linux.com> Signed-off-by: Andi Kleen <ak@linux.intel.com>
This commit is contained in:
Родитель
6de2b1aab9
Коммит
a9869b837c
35
mm/hugetlb.c
35
mm/hugetlb.c
|
@ -509,6 +509,7 @@ static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
|
||||||
return NULL;
|
return NULL;
|
||||||
page = list_entry(h->hugepage_freelists[nid].next, struct page, lru);
|
page = list_entry(h->hugepage_freelists[nid].next, struct page, lru);
|
||||||
list_del(&page->lru);
|
list_del(&page->lru);
|
||||||
|
set_page_refcounted(page);
|
||||||
h->free_huge_pages--;
|
h->free_huge_pages--;
|
||||||
h->free_huge_pages_node[nid]--;
|
h->free_huge_pages_node[nid]--;
|
||||||
return page;
|
return page;
|
||||||
|
@ -868,12 +869,6 @@ static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
|
||||||
|
|
||||||
spin_lock(&hugetlb_lock);
|
spin_lock(&hugetlb_lock);
|
||||||
if (page) {
|
if (page) {
|
||||||
/*
|
|
||||||
* This page is now managed by the hugetlb allocator and has
|
|
||||||
* no users -- drop the buddy allocator's reference.
|
|
||||||
*/
|
|
||||||
put_page_testzero(page);
|
|
||||||
VM_BUG_ON(page_count(page));
|
|
||||||
r_nid = page_to_nid(page);
|
r_nid = page_to_nid(page);
|
||||||
set_compound_page_dtor(page, free_huge_page);
|
set_compound_page_dtor(page, free_huge_page);
|
||||||
/*
|
/*
|
||||||
|
@ -936,16 +931,13 @@ retry:
|
||||||
spin_unlock(&hugetlb_lock);
|
spin_unlock(&hugetlb_lock);
|
||||||
for (i = 0; i < needed; i++) {
|
for (i = 0; i < needed; i++) {
|
||||||
page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
|
page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
|
||||||
if (!page) {
|
if (!page)
|
||||||
/*
|
/*
|
||||||
* We were not able to allocate enough pages to
|
* We were not able to allocate enough pages to
|
||||||
* satisfy the entire reservation so we free what
|
* satisfy the entire reservation so we free what
|
||||||
* we've allocated so far.
|
* we've allocated so far.
|
||||||
*/
|
*/
|
||||||
spin_lock(&hugetlb_lock);
|
|
||||||
needed = 0;
|
|
||||||
goto free;
|
goto free;
|
||||||
}
|
|
||||||
|
|
||||||
list_add(&page->lru, &surplus_list);
|
list_add(&page->lru, &surplus_list);
|
||||||
}
|
}
|
||||||
|
@ -972,31 +964,31 @@ retry:
|
||||||
needed += allocated;
|
needed += allocated;
|
||||||
h->resv_huge_pages += delta;
|
h->resv_huge_pages += delta;
|
||||||
ret = 0;
|
ret = 0;
|
||||||
free:
|
|
||||||
|
spin_unlock(&hugetlb_lock);
|
||||||
/* Free the needed pages to the hugetlb pool */
|
/* Free the needed pages to the hugetlb pool */
|
||||||
list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
|
list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
|
||||||
if ((--needed) < 0)
|
if ((--needed) < 0)
|
||||||
break;
|
break;
|
||||||
list_del(&page->lru);
|
list_del(&page->lru);
|
||||||
|
/*
|
||||||
|
* This page is now managed by the hugetlb allocator and has
|
||||||
|
* no users -- drop the buddy allocator's reference.
|
||||||
|
*/
|
||||||
|
put_page_testzero(page);
|
||||||
|
VM_BUG_ON(page_count(page));
|
||||||
enqueue_huge_page(h, page);
|
enqueue_huge_page(h, page);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Free unnecessary surplus pages to the buddy allocator */
|
/* Free unnecessary surplus pages to the buddy allocator */
|
||||||
|
free:
|
||||||
if (!list_empty(&surplus_list)) {
|
if (!list_empty(&surplus_list)) {
|
||||||
spin_unlock(&hugetlb_lock);
|
|
||||||
list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
|
list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
|
||||||
list_del(&page->lru);
|
list_del(&page->lru);
|
||||||
/*
|
put_page(page);
|
||||||
* The page has a reference count of zero already, so
|
|
||||||
* call free_huge_page directly instead of using
|
|
||||||
* put_page. This must be done with hugetlb_lock
|
|
||||||
* unlocked which is safe because free_huge_page takes
|
|
||||||
* hugetlb_lock before deciding how to free the page.
|
|
||||||
*/
|
|
||||||
free_huge_page(page);
|
|
||||||
}
|
}
|
||||||
spin_lock(&hugetlb_lock);
|
|
||||||
}
|
}
|
||||||
|
spin_lock(&hugetlb_lock);
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
@ -1123,7 +1115,6 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
set_page_refcounted(page);
|
|
||||||
set_page_private(page, (unsigned long) mapping);
|
set_page_private(page, (unsigned long) mapping);
|
||||||
|
|
||||||
vma_commit_reservation(h, vma, addr);
|
vma_commit_reservation(h, vma, addr);
|
||||||
|
|
Загрузка…
Ссылка в новой задаче