From fec52486682f7b3af2ffc28bcacc6e2ad6f7e1e6 Mon Sep 17 00:00:00 2001 From: Zev Weiss Date: Mon, 11 Mar 2019 23:27:58 -0700 Subject: [PATCH 01/38] tools/testing/selftests/sysctl/sysctl.sh: add tests for >32-bit values written to 32-bit integers Patch series "sysctl: fix range-checking in do_proc_dointvec_minmax_conv()", v2. After being left with an unusable system after a typo executing something like 'echo $((1<<24)) > /proc/sys/vm/max_map_count', I found that do_proc_dointvec_minmax_conv() was missing a check to ensure that the converted value actually fits in an int. The first of the following patches enhances the sysctl selftest such that it detects this problem; the second provides a minimal fix (suitable for -stable) such that the selftest passes. The third patch then performs a more thorough refactoring to eliminate the code duplication that led to the bug in the first place (maintaining the passing status of the selftest). This patch (of 3): At present this exposes a bug in do_proc_dointvec_minmax_conv() (it fails to check for values that are too wide to fit in an int). Link: http://lkml.kernel.org/r/20190207123426.9202-2-zev@bewilderbeest.net Signed-off-by: Zev Weiss Cc: Luis Chamberlain Cc: Kees Cook Cc: Brendan Higgins Cc: Iurii Zaikin Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- tools/testing/selftests/sysctl/sysctl.sh | 55 ++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tools/testing/selftests/sysctl/sysctl.sh b/tools/testing/selftests/sysctl/sysctl.sh index 584eb8ea780a..780ce7123374 100755 --- a/tools/testing/selftests/sysctl/sysctl.sh +++ b/tools/testing/selftests/sysctl/sysctl.sh @@ -290,6 +290,58 @@ run_numerictests() test_rc } +check_failure() +{ + echo -n "Testing that $1 fails as expected..." + reset_vals + TEST_STR="$1" + orig="$(cat $TARGET)" + echo -n "$TEST_STR" > $TARGET 2> /dev/null + + # write should fail and $TARGET should retain its original value + if [ $? = 0 ] || [ "$(cat $TARGET)" != "$orig" ]; then + echo "FAIL" >&2 + rc=1 + else + echo "ok" + fi + test_rc +} + +run_wideint_tests() +{ + # sysctl conversion functions receive a boolean sign and ulong + # magnitude; here we list the magnitudes we want to test (each of + # which will be tested in both positive and negative forms). Since + # none of these values fit in 32 bits, writing them to an int- or + # uint-typed sysctl should fail. + local magnitudes=( + # common boundary-condition values (zero, +1, -1, INT_MIN, + # and INT_MAX respectively) if truncated to lower 32 bits + # (potential for being falsely deemed in range) + 0x0000000100000000 + 0x0000000100000001 + 0x00000001ffffffff + 0x0000000180000000 + 0x000000017fffffff + + # these look like negatives, but without a leading '-' are + # actually large positives (should be rejected as above + # despite being zero/+1/-1/INT_MIN/INT_MAX in the lower 32) + 0xffffffff00000000 + 0xffffffff00000001 + 0xffffffffffffffff + 0xffffffff80000000 + 0xffffffff7fffffff + ) + + for sign in '' '-'; do + for mag in "${magnitudes[@]}"; do + check_failure "${sign}${mag}" + done + done +} + # Your test must accept digits 3 and 4 to use this run_limit_digit() { @@ -556,6 +608,7 @@ sysctl_test_0001() TEST_STR=$(( $ORIG + 1 )) run_numerictests + run_wideint_tests run_limit_digit } @@ -580,6 +633,7 @@ sysctl_test_0003() TEST_STR=$(( $ORIG + 1 )) run_numerictests + run_wideint_tests run_limit_digit run_limit_digit_int } @@ -592,6 +646,7 @@ sysctl_test_0004() TEST_STR=$(( $ORIG + 1 )) run_numerictests + run_wideint_tests run_limit_digit run_limit_digit_uint } From 8cf7630b29701d364f8df4a50e4f1f5e752b2778 Mon Sep 17 00:00:00 2001 From: Zev Weiss Date: Mon, 11 Mar 2019 23:28:02 -0700 Subject: [PATCH 02/38] kernel/sysctl.c: add missing range check in do_proc_dointvec_minmax_conv This bug has apparently existed since the introduction of this function in the pre-git era (4500e91754d3 in Thomas Gleixner's history.git, "[NET]: Add proc_dointvec_userhz_jiffies, use it for proper handling of neighbour sysctls."). As a minimal fix we can simply duplicate the corresponding check in do_proc_dointvec_conv(). Link: http://lkml.kernel.org/r/20190207123426.9202-3-zev@bewilderbeest.net Signed-off-by: Zev Weiss Cc: Brendan Higgins Cc: Iurii Zaikin Cc: Kees Cook Cc: Luis Chamberlain Cc: [2.6.2+] Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/sysctl.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 7bb3988425ee..0854197e0e67 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2645,7 +2645,16 @@ static int do_proc_dointvec_minmax_conv(bool *negp, unsigned long *lvalp, { struct do_proc_dointvec_minmax_conv_param *param = data; if (write) { - int val = *negp ? -*lvalp : *lvalp; + int val; + if (*negp) { + if (*lvalp > (unsigned long) INT_MAX + 1) + return -EINVAL; + val = -*lvalp; + } else { + if (*lvalp > (unsigned long) INT_MAX) + return -EINVAL; + val = *lvalp; + } if ((param->min && *param->min > val) || (param->max && *param->max < val)) return -EINVAL; From 2bc4fc60fb3e1a4cae429b889720755ebe9085bb Mon Sep 17 00:00:00 2001 From: Zev Weiss Date: Mon, 11 Mar 2019 23:28:06 -0700 Subject: [PATCH 03/38] kernel/sysctl.c: define minmax conv functions in terms of non-minmax versions do_proc_do[u]intvec_minmax_conv() had included open-coded versions of do_proc_do[u]intvec_conv(); the duplication led to buggy inconsistencies (missing range checks). To reduce the likelihood of such problems in the future, we can instead refactor both to be defined in terms of their non-bounded counterparts (plus the added check). Link: http://lkml.kernel.org/r/20190207165138.5oud57vq4ozwb4kh@hatter.bewilderbeest.net Signed-off-by: Zev Weiss Cc: Brendan Higgins Cc: Iurii Zaikin Cc: Kees Cook Cc: Luis Chamberlain Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/sysctl.c | 59 ++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 0854197e0e67..e5da394d1ca3 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2643,32 +2643,25 @@ static int do_proc_dointvec_minmax_conv(bool *negp, unsigned long *lvalp, int *valp, int write, void *data) { + int tmp, ret; struct do_proc_dointvec_minmax_conv_param *param = data; + /* + * If writing, first do so via a temporary local int so we can + * bounds-check it before touching *valp. + */ + int *ip = write ? &tmp : valp; + + ret = do_proc_dointvec_conv(negp, lvalp, ip, write, data); + if (ret) + return ret; + if (write) { - int val; - if (*negp) { - if (*lvalp > (unsigned long) INT_MAX + 1) - return -EINVAL; - val = -*lvalp; - } else { - if (*lvalp > (unsigned long) INT_MAX) - return -EINVAL; - val = *lvalp; - } - if ((param->min && *param->min > val) || - (param->max && *param->max < val)) + if ((param->min && *param->min > tmp) || + (param->max && *param->max < tmp)) return -EINVAL; - *valp = val; - } else { - int val = *valp; - if (val < 0) { - *negp = true; - *lvalp = -(unsigned long)val; - } else { - *negp = false; - *lvalp = (unsigned long)val; - } + *valp = tmp; } + return 0; } @@ -2717,22 +2710,22 @@ static int do_proc_douintvec_minmax_conv(unsigned long *lvalp, unsigned int *valp, int write, void *data) { + int ret; + unsigned int tmp; struct do_proc_douintvec_minmax_conv_param *param = data; + /* write via temporary local uint for bounds-checking */ + unsigned int *up = write ? &tmp : valp; + + ret = do_proc_douintvec_conv(lvalp, up, write, data); + if (ret) + return ret; if (write) { - unsigned int val = *lvalp; - - if (*lvalp > UINT_MAX) - return -EINVAL; - - if ((param->min && *param->min > val) || - (param->max && *param->max < val)) + if ((param->min && *param->min > tmp) || + (param->max && *param->max < tmp)) return -ERANGE; - *valp = val; - } else { - unsigned int val = *valp; - *lvalp = (unsigned long) val; + *valp = tmp; } return 0; From b57e622e6da9048c96fa0ed6943834949a398e3f Mon Sep 17 00:00:00 2001 From: Souptick Joarder Date: Mon, 11 Mar 2019 23:28:10 -0700 Subject: [PATCH 04/38] mm/hmm: convert to use vm_fault_t MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert to use vm_fault_t type as return type for fault handler. kbuild reported warning during testing of *mm-create-the-new-vm_fault_t-type.patch* available in below link - https://patchwork.kernel.org/patch/10752741/ kernel/memremap.c:46:34: warning: incorrect type in return expression (different base types) kernel/memremap.c:46:34: expected restricted vm_fault_t kernel/memremap.c:46:34: got int This patch has fixed the warnings and also hmm_devmem_fault() is converted to return vm_fault_t to avoid further warnings. [sfr@canb.auug.org.au: drm/nouveau/dmem: update for struct hmm_devmem_ops member change] Link: http://lkml.kernel.org/r/20190220174407.753d94e5@canb.auug.org.au Link: http://lkml.kernel.org/r/20190110145900.GA1317@jordon-HP-15-Notebook-PC Signed-off-by: Souptick Joarder Signed-off-by: Stephen Rothwell Reviewed-by: Jérôme Glisse Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/gpu/drm/nouveau/nouveau_dmem.c | 2 +- include/linux/hmm.h | 4 ++-- mm/hmm.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_dmem.c b/drivers/gpu/drm/nouveau/nouveau_dmem.c index 8be7a83ced9b..aa9fec80492d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dmem.c +++ b/drivers/gpu/drm/nouveau/nouveau_dmem.c @@ -261,7 +261,7 @@ static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = { .finalize_and_map = nouveau_dmem_fault_finalize_and_map, }; -static int +static vm_fault_t nouveau_dmem_fault(struct hmm_devmem *devmem, struct vm_area_struct *vma, unsigned long addr, diff --git a/include/linux/hmm.h b/include/linux/hmm.h index 66f9ebbb1df3..ad50b7b4f141 100644 --- a/include/linux/hmm.h +++ b/include/linux/hmm.h @@ -468,7 +468,7 @@ struct hmm_devmem_ops { * Note that mmap semaphore is held in read mode at least when this * callback occurs, hence the vma is valid upon callback entry. */ - int (*fault)(struct hmm_devmem *devmem, + vm_fault_t (*fault)(struct hmm_devmem *devmem, struct vm_area_struct *vma, unsigned long addr, const struct page *page, @@ -511,7 +511,7 @@ struct hmm_devmem_ops { * chunk, as an optimization. It must, however, prioritize the faulting address * over all the others. */ -typedef int (*dev_page_fault_t)(struct vm_area_struct *vma, +typedef vm_fault_t (*dev_page_fault_t)(struct vm_area_struct *vma, unsigned long addr, const struct page *page, unsigned int flags, diff --git a/mm/hmm.c b/mm/hmm.c index a04e4b810610..fe1cd87e49ac 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -990,7 +990,7 @@ static void hmm_devmem_ref_kill(struct percpu_ref *ref) percpu_ref_kill(ref); } -static int hmm_devmem_fault(struct vm_area_struct *vma, +static vm_fault_t hmm_devmem_fault(struct vm_area_struct *vma, unsigned long addr, const struct page *page, unsigned int flags, From b5420237ec817b0b5f729a674c81ace0865c3b3b Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Mon, 11 Mar 2019 23:28:13 -0700 Subject: [PATCH 05/38] mm: refactor readahead defines in mm.h All users of VM_MAX_READAHEAD actually convert it to kbytes and then to pages. Define the macro explicitly as (SZ_128K / PAGE_SIZE). This simplifies the expression in every filesystem. Also rename the macro to VM_READAHEAD_PAGES to properly convey its meaning. Finally remove unused VM_MIN_READAHEAD [akpm@linux-foundation.org: fix fs/io_uring.c, per Stephen] Link: http://lkml.kernel.org/r/20181221144053.24318-1-nborisov@suse.com Signed-off-by: Nikolay Borisov Reviewed-by: Matthew Wilcox Reviewed-by: David Hildenbrand Cc: Jens Axboe Cc: Eric Van Hensbergen Cc: Latchesar Ionkov Cc: Dominique Martinet Cc: David Howells Cc: Chris Mason Cc: Josef Bacik Cc: David Sterba Cc: Miklos Szeredi Cc: Stephen Rothwell Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- block/blk-core.c | 3 +-- fs/9p/vfs_super.c | 2 +- fs/afs/super.c | 2 +- fs/btrfs/disk-io.c | 2 +- fs/fuse/inode.c | 2 +- fs/io_uring.c | 2 +- include/linux/mm.h | 4 ++-- 7 files changed, 8 insertions(+), 9 deletions(-) diff --git a/block/blk-core.c b/block/blk-core.c index 6b78ec56a4f2..4673ebe42255 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -500,8 +500,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) if (!q->stats) goto fail_stats; - q->backing_dev_info->ra_pages = - (VM_MAX_READAHEAD * 1024) / PAGE_SIZE; + q->backing_dev_info->ra_pages = VM_READAHEAD_PAGES; q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK; q->backing_dev_info->name = "block"; q->node = node_id; diff --git a/fs/9p/vfs_super.c b/fs/9p/vfs_super.c index 48ce50484e80..10d3bd3f534b 100644 --- a/fs/9p/vfs_super.c +++ b/fs/9p/vfs_super.c @@ -92,7 +92,7 @@ v9fs_fill_super(struct super_block *sb, struct v9fs_session_info *v9ses, return ret; if (v9ses->cache) - sb->s_bdi->ra_pages = (VM_MAX_READAHEAD * 1024)/PAGE_SIZE; + sb->s_bdi->ra_pages = VM_READAHEAD_PAGES; sb->s_flags |= SB_ACTIVE | SB_DIRSYNC; if (!v9ses->cache) diff --git a/fs/afs/super.c b/fs/afs/super.c index dcd07fe99871..e684f6769b15 100644 --- a/fs/afs/super.c +++ b/fs/afs/super.c @@ -399,7 +399,7 @@ static int afs_fill_super(struct super_block *sb, ret = super_setup_bdi(sb); if (ret) return ret; - sb->s_bdi->ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_SIZE; + sb->s_bdi->ra_pages = VM_READAHEAD_PAGES; /* allocate the root inode and dentry */ if (as->dyn_root) { diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index f0cdb53f3e2d..6fe9197f6ee4 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -2958,7 +2958,7 @@ int open_ctree(struct super_block *sb, sb->s_bdi->congested_fn = btrfs_congested_fn; sb->s_bdi->congested_data = fs_info; sb->s_bdi->capabilities |= BDI_CAP_CGROUP_WRITEBACK; - sb->s_bdi->ra_pages = VM_MAX_READAHEAD * SZ_1K / PAGE_SIZE; + sb->s_bdi->ra_pages = VM_READAHEAD_PAGES; sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super); sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index c2d4099429be..16750ed591ae 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1010,7 +1010,7 @@ static int fuse_bdi_init(struct fuse_conn *fc, struct super_block *sb) if (err) return err; - sb->s_bdi->ra_pages = (VM_MAX_READAHEAD * 1024) / PAGE_SIZE; + sb->s_bdi->ra_pages = VM_READAHEAD_PAGES; /* fuse does it's own writeback accounting */ sb->s_bdi->capabilities = BDI_CAP_NO_ACCT_WB | BDI_CAP_STRICTLIMIT; diff --git a/fs/io_uring.c b/fs/io_uring.c index 5d99376d2369..c88088d92613 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -923,7 +923,7 @@ static void io_async_list_note(int rw, struct io_kiocb *req, size_t len) /* Use 8x RA size as a decent limiter for both reads/writes */ max_pages = filp->f_ra.ra_pages; if (!max_pages) - max_pages = VM_MAX_READAHEAD >> (PAGE_SHIFT - 10); + max_pages = VM_READAHEAD_PAGES; max_pages *= 8; /* If max pages are exceeded, reset the state */ diff --git a/include/linux/mm.h b/include/linux/mm.h index 5801ee849f36..76769749b5a5 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -26,6 +26,7 @@ #include #include #include +#include struct mempolicy; struct anon_vma; @@ -2402,8 +2403,7 @@ int __must_check write_one_page(struct page *page); void task_dirty_inc(struct task_struct *tsk); /* readahead.c */ -#define VM_MAX_READAHEAD 128 /* kbytes */ -#define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */ +#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) int force_page_cache_readahead(struct address_space *mapping, struct file *filp, pgoff_t offset, unsigned long nr_to_read); From d5a572a4cb1e484d8d8c79bdfd16aaa18fa1470b Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Mon, 11 Mar 2019 23:28:51 -0700 Subject: [PATCH 06/38] proc: calculate end pointer for /proc/*/* lookup at compile time Compilers like to transform loops like for (i = 0; i < n; i++) { [use p[i]] } into for (p = p0; p < end; p++) { ... } Do it by hand, so that it results in overall simpler loop and smaller code. Space savings: $ ./scripts/bloat-o-meter ../vmlinux-001 ../obj/vmlinux add/remove: 0/0 grow/shrink: 2/1 up/down: 4/-9 (-5) Function old new delta proc_tid_base_lookup 17 19 +2 proc_tgid_base_lookup 17 19 +2 proc_pident_lookup 179 170 -9 The same could be done to proc_pident_readdir(), but the code becomes bigger for some reason. [sfr@canb.auug.org.au: merge fix for proc_pident_lookup() API change] Link: http://lkml.kernel.org/r/20190131160135.4a8ae70b@canb.auug.org.au Link: http://lkml.kernel.org/r/20190114200422.GB9680@avx2 Signed-off-by: Alexey Dobriyan Signed-off-by: Stephen Rothwell Cc: James Morris Cc: Alexey Dobriyan Cc: Casey Schaufler Cc: Kees Cook Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/base.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/fs/proc/base.c b/fs/proc/base.c index 5ab1849971b4..1b548d060d28 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2459,11 +2459,10 @@ static struct dentry *proc_pident_instantiate(struct dentry *dentry, static struct dentry *proc_pident_lookup(struct inode *dir, struct dentry *dentry, - const struct pid_entry *ents, - unsigned int nents) + const struct pid_entry *p, + const struct pid_entry *end) { struct task_struct *task = get_proc_task(dir); - const struct pid_entry *p, *last; struct dentry *res = ERR_PTR(-ENOENT); if (!task) @@ -2473,8 +2472,7 @@ static struct dentry *proc_pident_lookup(struct inode *dir, * Yes, it does not scale. And it should not. Don't add * new entries into /proc// without very good reasons. */ - last = &ents[nents]; - for (p = ents; p < last; p++) { + for (; p < end; p++) { if (p->len != dentry->d_name.len) continue; if (!memcmp(dentry->d_name.name, p->name, p->len)) { @@ -2610,7 +2608,7 @@ static struct dentry *proc_##LSM##_attr_dir_lookup(struct inode *dir, \ { \ return proc_pident_lookup(dir, dentry, \ LSM##_attr_dir_stuff, \ - ARRAY_SIZE(LSM##_attr_dir_stuff)); \ + LSM##_attr_dir_stuff + ARRAY_SIZE(LSM##_attr_dir_stuff)); \ } \ \ static const struct inode_operations proc_##LSM##_attr_dir_inode_ops = { \ @@ -2655,7 +2653,8 @@ static struct dentry *proc_attr_dir_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { return proc_pident_lookup(dir, dentry, - attr_dir_stuff, ARRAY_SIZE(attr_dir_stuff)); + attr_dir_stuff, + attr_dir_stuff + ARRAY_SIZE(attr_dir_stuff)); } static const struct inode_operations proc_attr_dir_inode_operations = { @@ -3091,7 +3090,8 @@ static const struct file_operations proc_tgid_base_operations = { static struct dentry *proc_tgid_base_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { return proc_pident_lookup(dir, dentry, - tgid_base_stuff, ARRAY_SIZE(tgid_base_stuff)); + tgid_base_stuff, + tgid_base_stuff + ARRAY_SIZE(tgid_base_stuff)); } static const struct inode_operations proc_tgid_base_inode_operations = { @@ -3463,7 +3463,8 @@ static int proc_tid_base_readdir(struct file *file, struct dir_context *ctx) static struct dentry *proc_tid_base_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { return proc_pident_lookup(dir, dentry, - tid_base_stuff, ARRAY_SIZE(tid_base_stuff)); + tid_base_stuff, + tid_base_stuff + ARRAY_SIZE(tid_base_stuff)); } static const struct file_operations proc_tid_base_operations = { From fb054d0d91dbe3c67c25c68cf17bf7e0b7214ca7 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:28:55 -0700 Subject: [PATCH 07/38] openrisc: prefer memblock APIs returning virtual address Patch series "Refine memblock API", v2. Current memblock API is quite extensive and, which is more annoying, duplicated. Except the low-level functions that allow searching for a free memory region and marking it as reserved, memblock provides three (well, two and a half) sets of functions to allocate memory. There are several overlapping functions that return a physical address and there are functions that return virtual address. Those that return the virtual address may also clear the allocated memory. And, on top of all that, some allocators panic and some return NULL in case of error. This set tries to reduce the mess, and trim down the amount of memblock allocation methods. Patches 1-10 consolidate the functions that return physical address of the allocated memory Patches 11-13 are some trivial cleanups Patches 14-19 add checks for the return value of memblock_alloc*() and panics in case of errors. The patches 14-18 include some minor refactoring to have better readability of the resulting code and patch 19 is a mechanical addition of if (!ptr) panic(); after memblock_alloc*() calls. And, finally, patches 20 and 21 remove panic() calls memblock and _nopanic variants from memblock. This patch (of 21): The allocation of the page tables memory in openrics uses memblock_phys_alloc() and then converts the returned physical address to virtual one. Use memblock_alloc_raw() and add a panic() if the allocation fails. Link: http://lkml.kernel.org/r/1548057848-15136-2-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Acked-by: Stafford Horne Cc: Catalin Marinas Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Heiko Carstens Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Rich Felker Cc: Richard Weinberger Cc: Rob Herring Cc: Russell King Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Cc: Christophe Leroy Cc: Guo Ren [c-sky] Cc: Juergen Gross [Xen] Cc: Rob Herring Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/openrisc/mm/init.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/openrisc/mm/init.c b/arch/openrisc/mm/init.c index d157310eb377..caeb4184e8a6 100644 --- a/arch/openrisc/mm/init.c +++ b/arch/openrisc/mm/init.c @@ -105,7 +105,10 @@ static void __init map_ram(void) } /* Alloc one page for holding PTE's... */ - pte = (pte_t *) __va(memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE)); + pte = memblock_alloc_raw(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate page for PTEs\n", + __func__); set_pmd(pme, __pmd(_KERNPG_TABLE + __pa(pte))); /* Fill the newly allocated page with PTE'S */ From 1269f7b83f2cf79c2dce3b7658df38e51fd4b0ab Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 11 Mar 2019 23:29:00 -0700 Subject: [PATCH 08/38] powerpc: use memblock functions returning virtual address Since only the virtual address of allocated blocks is used, lets use functions returning directly virtual address. Those functions have the advantage of also zeroing the block. [rppt@linux.ibm.com: powerpc: remove duplicated alloc_stack() function] Link: http://lkml.kernel.org/r/20190226064032.GA5873@rapoport-lnx [rppt@linux.ibm.com: updated error message in alloc_stack() to be more verbose] [rppt@linux.ibm.com: convereted several additional call sites ] Link: http://lkml.kernel.org/r/1548057848-15136-3-git-send-email-rppt@linux.ibm.com Signed-off-by: Christophe Leroy Signed-off-by: Mike Rapoport Acked-by: Michael Ellerman Cc: Catalin Marinas Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/powerpc/kernel/dt_cpu_ftrs.c | 3 +-- arch/powerpc/kernel/paca.c | 6 +++++- arch/powerpc/kernel/prom.c | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c index e49bd5efcfe6..28c076c771de 100644 --- a/arch/powerpc/kernel/dt_cpu_ftrs.c +++ b/arch/powerpc/kernel/dt_cpu_ftrs.c @@ -810,7 +810,6 @@ static int __init process_cpufeatures_node(unsigned long node, int len; f = &dt_cpu_features[i]; - memset(f, 0, sizeof(struct dt_cpu_feature)); f->node = node; @@ -1005,7 +1004,7 @@ static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char /* Count and allocate space for cpu features */ of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, &nr_dt_cpu_features); - dt_cpu_features = __va(memblock_phys_alloc(sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE)); + dt_cpu_features = memblock_alloc(sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE); cpufeatures_setup_start(isa); diff --git a/arch/powerpc/kernel/paca.c b/arch/powerpc/kernel/paca.c index 8c890c6557ed..e7382abee868 100644 --- a/arch/powerpc/kernel/paca.c +++ b/arch/powerpc/kernel/paca.c @@ -196,7 +196,11 @@ void __init allocate_paca_ptrs(void) paca_nr_cpu_ids = nr_cpu_ids; paca_ptrs_size = sizeof(struct paca_struct *) * nr_cpu_ids; - paca_ptrs = __va(memblock_phys_alloc(paca_ptrs_size, SMP_CACHE_BYTES)); + paca_ptrs = memblock_alloc_raw(paca_ptrs_size, SMP_CACHE_BYTES); + if (!paca_ptrs) + panic("Failed to allocate %d bytes for paca pointers\n", + paca_ptrs_size); + memset(paca_ptrs, 0x88, paca_ptrs_size); } diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index 4181ec715f88..4221527b082f 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -126,7 +126,10 @@ static void __init move_device_tree(void) if ((memory_limit && (start + size) > PHYSICAL_START + memory_limit) || !memblock_is_memory(start + size - 1) || overlaps_crashkernel(start, size) || overlaps_initrd(start, size)) { - p = __va(memblock_phys_alloc(size, PAGE_SIZE)); + p = memblock_alloc_raw(size, PAGE_SIZE); + if (!p) + panic("Failed to allocate %lu bytes to move device tree\n", + size); memcpy(p, initial_boot_params, size); initial_boot_params = p; DBG("Moved device tree to 0x%px\n", p); From f240ec09bb8a08003a6e2cb51682705ab19b78cd Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:29:06 -0700 Subject: [PATCH 09/38] memblock: replace memblock_alloc_base(ANYWHERE) with memblock_phys_alloc The calls to memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ANYWHERE) and memblock_phys_alloc(size, align) are equivalent as both try to allocate 'size' bytes with 'align' alignment anywhere in the memory and panic if hte allocation fails. The conversion is done using the following semantic patch: @@ expression size, align; @@ - memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ANYWHERE) + memblock_phys_alloc(size, align) Link: http://lkml.kernel.org/r/1548057848-15136-4-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/arm/mm/init.c | 2 +- arch/sh/boards/mach-ap325rxa/setup.c | 2 +- arch/sh/boards/mach-ecovec24/setup.c | 4 ++-- arch/sh/boards/mach-kfr2r09/setup.c | 2 +- arch/sh/boards/mach-migor/setup.c | 2 +- arch/sh/boards/mach-se/7724/setup.c | 4 ++-- arch/xtensa/mm/kasan_init.c | 3 +-- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 478ea8b7db87..b76b90eb9356 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -205,7 +205,7 @@ phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align) BUG_ON(!arm_memblock_steal_permitted); - phys = memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ANYWHERE); + phys = memblock_phys_alloc(size, align); memblock_free(phys, size); memblock_remove(phys, size); diff --git a/arch/sh/boards/mach-ap325rxa/setup.c b/arch/sh/boards/mach-ap325rxa/setup.c index 7899b4f51fdd..97774424fbee 100644 --- a/arch/sh/boards/mach-ap325rxa/setup.c +++ b/arch/sh/boards/mach-ap325rxa/setup.c @@ -556,7 +556,7 @@ static void __init ap325rxa_mv_mem_reserve(void) phys_addr_t phys; phys_addr_t size = CEU_BUFFER_MEMORY_SIZE; - phys = memblock_alloc_base(size, PAGE_SIZE, MEMBLOCK_ALLOC_ANYWHERE); + phys = memblock_phys_alloc(size, PAGE_SIZE); memblock_free(phys, size); memblock_remove(phys, size); diff --git a/arch/sh/boards/mach-ecovec24/setup.c b/arch/sh/boards/mach-ecovec24/setup.c index 3c59019c4695..d329bf3be487 100644 --- a/arch/sh/boards/mach-ecovec24/setup.c +++ b/arch/sh/boards/mach-ecovec24/setup.c @@ -1476,12 +1476,12 @@ static void __init ecovec_mv_mem_reserve(void) phys_addr_t phys; phys_addr_t size = CEU_BUFFER_MEMORY_SIZE; - phys = memblock_alloc_base(size, PAGE_SIZE, MEMBLOCK_ALLOC_ANYWHERE); + phys = memblock_phys_alloc(size, PAGE_SIZE); memblock_free(phys, size); memblock_remove(phys, size); ceu0_dma_membase = phys; - phys = memblock_alloc_base(size, PAGE_SIZE, MEMBLOCK_ALLOC_ANYWHERE); + phys = memblock_phys_alloc(size, PAGE_SIZE); memblock_free(phys, size); memblock_remove(phys, size); ceu1_dma_membase = phys; diff --git a/arch/sh/boards/mach-kfr2r09/setup.c b/arch/sh/boards/mach-kfr2r09/setup.c index b8bf67c86eab..5c258ae9c43a 100644 --- a/arch/sh/boards/mach-kfr2r09/setup.c +++ b/arch/sh/boards/mach-kfr2r09/setup.c @@ -630,7 +630,7 @@ static void __init kfr2r09_mv_mem_reserve(void) phys_addr_t phys; phys_addr_t size = CEU_BUFFER_MEMORY_SIZE; - phys = memblock_alloc_base(size, PAGE_SIZE, MEMBLOCK_ALLOC_ANYWHERE); + phys = memblock_phys_alloc(size, PAGE_SIZE); memblock_free(phys, size); memblock_remove(phys, size); diff --git a/arch/sh/boards/mach-migor/setup.c b/arch/sh/boards/mach-migor/setup.c index 03579faa4ce3..193d91bb84bf 100644 --- a/arch/sh/boards/mach-migor/setup.c +++ b/arch/sh/boards/mach-migor/setup.c @@ -630,7 +630,7 @@ static void __init migor_mv_mem_reserve(void) phys_addr_t phys; phys_addr_t size = CEU_BUFFER_MEMORY_SIZE; - phys = memblock_alloc_base(size, PAGE_SIZE, MEMBLOCK_ALLOC_ANYWHERE); + phys = memblock_phys_alloc(size, PAGE_SIZE); memblock_free(phys, size); memblock_remove(phys, size); diff --git a/arch/sh/boards/mach-se/7724/setup.c b/arch/sh/boards/mach-se/7724/setup.c index 13c2d3ce78f4..5c7aa37bfd86 100644 --- a/arch/sh/boards/mach-se/7724/setup.c +++ b/arch/sh/boards/mach-se/7724/setup.c @@ -963,12 +963,12 @@ static void __init ms7724se_mv_mem_reserve(void) phys_addr_t phys; phys_addr_t size = CEU_BUFFER_MEMORY_SIZE; - phys = memblock_alloc_base(size, PAGE_SIZE, MEMBLOCK_ALLOC_ANYWHERE); + phys = memblock_phys_alloc(size, PAGE_SIZE); memblock_free(phys, size); memblock_remove(phys, size); ceu0_dma_membase = phys; - phys = memblock_alloc_base(size, PAGE_SIZE, MEMBLOCK_ALLOC_ANYWHERE); + phys = memblock_phys_alloc(size, PAGE_SIZE); memblock_free(phys, size); memblock_remove(phys, size); ceu1_dma_membase = phys; diff --git a/arch/xtensa/mm/kasan_init.c b/arch/xtensa/mm/kasan_init.c index 1734cda6bc4a..48dbb03f4f6f 100644 --- a/arch/xtensa/mm/kasan_init.c +++ b/arch/xtensa/mm/kasan_init.c @@ -52,8 +52,7 @@ static void __init populate(void *start, void *end) for (k = 0; k < PTRS_PER_PTE; ++k, ++j) { phys_addr_t phys = - memblock_alloc_base(PAGE_SIZE, PAGE_SIZE, - MEMBLOCK_ALLOC_ANYWHERE); + memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); set_pte(pte + j, pfn_pte(PHYS_PFN(phys), PAGE_KERNEL)); } From 53d818d2747ca84f1a87a0006b903523cd5bf0cd Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:29:11 -0700 Subject: [PATCH 10/38] memblock: drop memblock_alloc_base_nid() memblock_alloc_base_nid() is a oneliner wrapper for memblock_alloc_range_nid() without any side effect. Replace it's usage by the direct calls to memblock_alloc_range_nid(). Link: http://lkml.kernel.org/r/1548057848-15136-5-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/memblock.h | 3 --- mm/memblock.c | 15 ++++----------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 859b55b66db2..4db53f7c6b17 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -446,9 +446,6 @@ static inline bool memblock_bottom_up(void) phys_addr_t __init memblock_alloc_range(phys_addr_t size, phys_addr_t align, phys_addr_t start, phys_addr_t end, enum memblock_flags flags); -phys_addr_t memblock_alloc_base_nid(phys_addr_t size, - phys_addr_t align, phys_addr_t max_addr, - int nid, enum memblock_flags flags); phys_addr_t memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr); phys_addr_t __memblock_alloc_base(phys_addr_t size, phys_addr_t align, diff --git a/mm/memblock.c b/mm/memblock.c index 470601115892..e9e440cfd210 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1289,21 +1289,14 @@ phys_addr_t __init memblock_alloc_range(phys_addr_t size, phys_addr_t align, flags); } -phys_addr_t __init memblock_alloc_base_nid(phys_addr_t size, - phys_addr_t align, phys_addr_t max_addr, - int nid, enum memblock_flags flags) -{ - return memblock_alloc_range_nid(size, align, 0, max_addr, nid, flags); -} - phys_addr_t __init memblock_phys_alloc_nid(phys_addr_t size, phys_addr_t align, int nid) { enum memblock_flags flags = choose_memblock_flags(); phys_addr_t ret; again: - ret = memblock_alloc_base_nid(size, align, MEMBLOCK_ALLOC_ACCESSIBLE, - nid, flags); + ret = memblock_alloc_range_nid(size, align, 0, + MEMBLOCK_ALLOC_ACCESSIBLE, nid, flags); if (!ret && (flags & MEMBLOCK_MIRROR)) { flags &= ~MEMBLOCK_MIRROR; @@ -1314,8 +1307,8 @@ again: phys_addr_t __init __memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr) { - return memblock_alloc_base_nid(size, align, max_addr, NUMA_NO_NODE, - MEMBLOCK_NONE); + return memblock_alloc_range_nid(size, align, 0, max_addr, NUMA_NO_NODE, + MEMBLOCK_NONE); } phys_addr_t __init memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr) From 8a770c2a83eaf4c3d493ca4056abd6d6ddce6f18 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:29:16 -0700 Subject: [PATCH 11/38] memblock: emphasize that memblock_alloc_range() returns a physical address Rename memblock_alloc_range() to memblock_phys_alloc_range() to emphasize that it returns a physical address. While on it, remove the 'enum memblock_flags' parameter from this function as its only user anyway sets it to MEMBLOCK_NONE, which is the default for the most of memblock allocations. Link: http://lkml.kernel.org/r/1548057848-15136-6-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/memblock.h | 5 ++--- mm/cma.c | 10 ++++------ mm/memblock.c | 9 +++++---- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 4db53f7c6b17..251cd66b151b 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -325,6 +325,8 @@ static inline int memblock_get_region_node(const struct memblock_region *r) #define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL #endif +phys_addr_t memblock_phys_alloc_range(phys_addr_t size, phys_addr_t align, + phys_addr_t start, phys_addr_t end); phys_addr_t memblock_phys_alloc_nid(phys_addr_t size, phys_addr_t align, int nid); phys_addr_t memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid); @@ -443,9 +445,6 @@ static inline bool memblock_bottom_up(void) return memblock.bottom_up; } -phys_addr_t __init memblock_alloc_range(phys_addr_t size, phys_addr_t align, - phys_addr_t start, phys_addr_t end, - enum memblock_flags flags); phys_addr_t memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr); phys_addr_t __memblock_alloc_base(phys_addr_t size, phys_addr_t align, diff --git a/mm/cma.c b/mm/cma.c index f4f3a8a57d86..bb2d333ffcb3 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -327,16 +327,14 @@ int __init cma_declare_contiguous(phys_addr_t base, * memory in case of failure. */ if (base < highmem_start && limit > highmem_start) { - addr = memblock_alloc_range(size, alignment, - highmem_start, limit, - MEMBLOCK_NONE); + addr = memblock_phys_alloc_range(size, alignment, + highmem_start, limit); limit = highmem_start; } if (!addr) { - addr = memblock_alloc_range(size, alignment, base, - limit, - MEMBLOCK_NONE); + addr = memblock_phys_alloc_range(size, alignment, base, + limit); if (!addr) { ret = -ENOMEM; goto err; diff --git a/mm/memblock.c b/mm/memblock.c index e9e440cfd210..eb785ea6757b 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1281,12 +1281,13 @@ static phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size, return 0; } -phys_addr_t __init memblock_alloc_range(phys_addr_t size, phys_addr_t align, - phys_addr_t start, phys_addr_t end, - enum memblock_flags flags) +phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, + phys_addr_t align, + phys_addr_t start, + phys_addr_t end) { return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE, - flags); + MEMBLOCK_NONE); } phys_addr_t __init memblock_phys_alloc_nid(phys_addr_t size, phys_addr_t align, int nid) From 337555744e6e39dd1d87698c6084dd88a606d60a Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:29:21 -0700 Subject: [PATCH 12/38] memblock: memblock_phys_alloc_try_nid(): don't panic The memblock_phys_alloc_try_nid() function tries to allocate memory from the requested node and then falls back to allocation from any node in the system. The memblock_alloc_base() fallback used by this function panics if the allocation fails. Replace the memblock_alloc_base() fallback with the direct call to memblock_alloc_range_nid() and update the memblock_phys_alloc_try_nid() callers to check the returned value and panic in case of error. Link: http://lkml.kernel.org/r/1548057848-15136-7-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Acked-by: Michael Ellerman [powerpc] Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/arm64/mm/numa.c | 4 ++++ arch/powerpc/mm/numa.c | 4 ++++ mm/memblock.c | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/arch/arm64/mm/numa.c b/arch/arm64/mm/numa.c index 7a0a555b366a..06a6f264f2dd 100644 --- a/arch/arm64/mm/numa.c +++ b/arch/arm64/mm/numa.c @@ -237,6 +237,10 @@ static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn) pr_info("Initmem setup node %d []\n", nid); nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid); + if (!nd_pa) + panic("Cannot allocate %zu bytes for node %d data\n", + nd_size, nid); + nd = __va(nd_pa); /* report and initialize */ diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index ac49e4158e50..f976676004ad 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -788,6 +788,10 @@ static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn) int tnid; nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid); + if (!nd_pa) + panic("Cannot allocate %zu bytes for node %d data\n", + nd_size, nid); + nd = __va(nd_pa); /* report and initialize */ diff --git a/mm/memblock.c b/mm/memblock.c index eb785ea6757b..ac57bd3082bb 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1336,7 +1336,9 @@ phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t ali if (res) return res; - return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); + return memblock_alloc_range_nid(size, align, 0, + MEMBLOCK_ALLOC_ACCESSIBLE, + NUMA_NO_NODE, MEMBLOCK_NONE); } /** From ecc3e771f4ca98c52a072e41804434b4979bdf84 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:29:26 -0700 Subject: [PATCH 13/38] memblock: memblock_phys_alloc(): don't panic Make the memblock_phys_alloc() function an inline wrapper for memblock_phys_alloc_range() and update the memblock_phys_alloc() callers to check the returned value and panic in case of error. Link: http://lkml.kernel.org/r/1548057848-15136-8-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/arm/mm/init.c | 4 ++++ arch/arm64/mm/mmu.c | 2 ++ arch/powerpc/sysdev/dart_iommu.c | 3 +++ arch/s390/kernel/crash_dump.c | 3 +++ arch/s390/kernel/setup.c | 3 +++ arch/sh/boards/mach-ap325rxa/setup.c | 3 +++ arch/sh/boards/mach-ecovec24/setup.c | 6 ++++++ arch/sh/boards/mach-kfr2r09/setup.c | 3 +++ arch/sh/boards/mach-migor/setup.c | 3 +++ arch/sh/boards/mach-se/7724/setup.c | 6 ++++++ arch/xtensa/mm/kasan_init.c | 3 +++ include/linux/memblock.h | 7 ++++++- mm/memblock.c | 5 ----- 13 files changed, 45 insertions(+), 6 deletions(-) diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index b76b90eb9356..15dddfe43319 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -206,6 +206,10 @@ phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align) BUG_ON(!arm_memblock_steal_permitted); phys = memblock_phys_alloc(size, align); + if (!phys) + panic("Failed to steal %pa bytes at %pS\n", + &size, (void *)_RET_IP_); + memblock_free(phys, size); memblock_remove(phys, size); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 402b6495ff58..e97f018ff740 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -103,6 +103,8 @@ static phys_addr_t __init early_pgtable_alloc(void) void *ptr; phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); + if (!phys) + panic("Failed to allocate page table page\n"); /* * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE diff --git a/arch/powerpc/sysdev/dart_iommu.c b/arch/powerpc/sysdev/dart_iommu.c index fc5c5c23303e..2a751795ec1e 100644 --- a/arch/powerpc/sysdev/dart_iommu.c +++ b/arch/powerpc/sysdev/dart_iommu.c @@ -265,6 +265,9 @@ static void allocate_dart(void) * prefetching into invalid pages and corrupting data */ tmp = memblock_phys_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE); + if (!tmp) + panic("DART: table allocation failed\n"); + dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) & DARTMAP_RPNMASK); diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 97eae3871868..f96a5857bbfd 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -61,6 +61,9 @@ struct save_area * __init save_area_alloc(bool is_boot_cpu) struct save_area *sa; sa = (void *) memblock_phys_alloc(sizeof(*sa), 8); + if (!sa) + panic("Failed to allocate save area\n"); + if (is_boot_cpu) list_add(&sa->list, &dump_save_areas); else diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 12934e8fbb91..d7920f3e76c6 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -966,6 +966,9 @@ static void __init setup_randomness(void) vmms = (struct sysinfo_3_2_2 *) memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); + if (!vmms) + panic("Failed to allocate memory for sysinfo structure\n"); + if (stsi(vmms, 3, 2, 2) == 0 && vmms->count) add_device_randomness(&vmms->vm, sizeof(vmms->vm[0]) * vmms->count); memblock_free((unsigned long) vmms, PAGE_SIZE); diff --git a/arch/sh/boards/mach-ap325rxa/setup.c b/arch/sh/boards/mach-ap325rxa/setup.c index 97774424fbee..8301a4378f50 100644 --- a/arch/sh/boards/mach-ap325rxa/setup.c +++ b/arch/sh/boards/mach-ap325rxa/setup.c @@ -557,6 +557,9 @@ static void __init ap325rxa_mv_mem_reserve(void) phys_addr_t size = CEU_BUFFER_MEMORY_SIZE; phys = memblock_phys_alloc(size, PAGE_SIZE); + if (!phys) + panic("Failed to allocate CEU memory\n"); + memblock_free(phys, size); memblock_remove(phys, size); diff --git a/arch/sh/boards/mach-ecovec24/setup.c b/arch/sh/boards/mach-ecovec24/setup.c index d329bf3be487..34e5414c5563 100644 --- a/arch/sh/boards/mach-ecovec24/setup.c +++ b/arch/sh/boards/mach-ecovec24/setup.c @@ -1477,11 +1477,17 @@ static void __init ecovec_mv_mem_reserve(void) phys_addr_t size = CEU_BUFFER_MEMORY_SIZE; phys = memblock_phys_alloc(size, PAGE_SIZE); + if (!phys) + panic("Failed to allocate CEU0 memory\n"); + memblock_free(phys, size); memblock_remove(phys, size); ceu0_dma_membase = phys; phys = memblock_phys_alloc(size, PAGE_SIZE); + if (!phys) + panic("Failed to allocate CEU1 memory\n"); + memblock_free(phys, size); memblock_remove(phys, size); ceu1_dma_membase = phys; diff --git a/arch/sh/boards/mach-kfr2r09/setup.c b/arch/sh/boards/mach-kfr2r09/setup.c index 5c258ae9c43a..1cf9a47ac90e 100644 --- a/arch/sh/boards/mach-kfr2r09/setup.c +++ b/arch/sh/boards/mach-kfr2r09/setup.c @@ -631,6 +631,9 @@ static void __init kfr2r09_mv_mem_reserve(void) phys_addr_t size = CEU_BUFFER_MEMORY_SIZE; phys = memblock_phys_alloc(size, PAGE_SIZE); + if (!phys) + panic("Failed to allocate CEU memory\n"); + memblock_free(phys, size); memblock_remove(phys, size); diff --git a/arch/sh/boards/mach-migor/setup.c b/arch/sh/boards/mach-migor/setup.c index 193d91bb84bf..90702740f207 100644 --- a/arch/sh/boards/mach-migor/setup.c +++ b/arch/sh/boards/mach-migor/setup.c @@ -631,6 +631,9 @@ static void __init migor_mv_mem_reserve(void) phys_addr_t size = CEU_BUFFER_MEMORY_SIZE; phys = memblock_phys_alloc(size, PAGE_SIZE); + if (!phys) + panic("Failed to allocate CEU memory\n"); + memblock_free(phys, size); memblock_remove(phys, size); diff --git a/arch/sh/boards/mach-se/7724/setup.c b/arch/sh/boards/mach-se/7724/setup.c index 5c7aa37bfd86..3674064816c7 100644 --- a/arch/sh/boards/mach-se/7724/setup.c +++ b/arch/sh/boards/mach-se/7724/setup.c @@ -964,11 +964,17 @@ static void __init ms7724se_mv_mem_reserve(void) phys_addr_t size = CEU_BUFFER_MEMORY_SIZE; phys = memblock_phys_alloc(size, PAGE_SIZE); + if (!phys) + panic("Failed to allocate CEU0 memory\n"); + memblock_free(phys, size); memblock_remove(phys, size); ceu0_dma_membase = phys; phys = memblock_phys_alloc(size, PAGE_SIZE); + if (!phys) + panic("Failed to allocate CEU1 memory\n"); + memblock_free(phys, size); memblock_remove(phys, size); ceu1_dma_membase = phys; diff --git a/arch/xtensa/mm/kasan_init.c b/arch/xtensa/mm/kasan_init.c index 48dbb03f4f6f..4852848a0c28 100644 --- a/arch/xtensa/mm/kasan_init.c +++ b/arch/xtensa/mm/kasan_init.c @@ -54,6 +54,9 @@ static void __init populate(void *start, void *end) phys_addr_t phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); + if (!phys) + panic("Failed to allocate page table page\n"); + set_pte(pte + j, pfn_pte(PHYS_PFN(phys), PAGE_KERNEL)); } } diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 251cd66b151b..7caecb42bfea 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -330,7 +330,12 @@ phys_addr_t memblock_phys_alloc_range(phys_addr_t size, phys_addr_t align, phys_addr_t memblock_phys_alloc_nid(phys_addr_t size, phys_addr_t align, int nid); phys_addr_t memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid); -phys_addr_t memblock_phys_alloc(phys_addr_t size, phys_addr_t align); +static inline phys_addr_t memblock_phys_alloc(phys_addr_t size, + phys_addr_t align) +{ + return memblock_phys_alloc_range(size, align, 0, + MEMBLOCK_ALLOC_ACCESSIBLE); +} void *memblock_alloc_try_nid_raw(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, phys_addr_t max_addr, diff --git a/mm/memblock.c b/mm/memblock.c index ac57bd3082bb..d0b76bb7340d 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1325,11 +1325,6 @@ phys_addr_t __init memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys return alloc; } -phys_addr_t __init memblock_phys_alloc(phys_addr_t size, phys_addr_t align) -{ - return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); -} - phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) { phys_addr_t res = memblock_phys_alloc_nid(size, align, nid); From 42b46aeff2e366bad54bd1c069b7b5381d9be8b3 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:29:31 -0700 Subject: [PATCH 14/38] memblock: drop __memblock_alloc_base() The __memblock_alloc_base() function tries to allocate a memory up to the limit specified by its max_addr parameter. Depending on the value of this parameter, the __memblock_alloc_base() can is replaced with the appropriate memblock_phys_alloc*() variant. Link: http://lkml.kernel.org/r/1548057848-15136-9-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Acked-by: Rob Herring Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/sh/kernel/machine_kexec.c | 3 ++- arch/x86/kernel/e820.c | 2 +- arch/x86/mm/numa.c | 12 ++++-------- drivers/of/of_reserved_mem.c | 7 ++----- include/linux/memblock.h | 2 -- mm/memblock.c | 9 ++------- 6 files changed, 11 insertions(+), 24 deletions(-) diff --git a/arch/sh/kernel/machine_kexec.c b/arch/sh/kernel/machine_kexec.c index b9f9f1a5afdc..63d63a36f6f2 100644 --- a/arch/sh/kernel/machine_kexec.c +++ b/arch/sh/kernel/machine_kexec.c @@ -168,7 +168,8 @@ void __init reserve_crashkernel(void) crash_size = PAGE_ALIGN(resource_size(&crashk_res)); if (!crashk_res.start) { unsigned long max = memblock_end_of_DRAM() - memory_limit; - crashk_res.start = __memblock_alloc_base(crash_size, PAGE_SIZE, max); + crashk_res.start = memblock_phys_alloc_range(crash_size, + PAGE_SIZE, 0, max); if (!crashk_res.start) { pr_err("crashkernel allocation failed\n"); goto disable; diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index a687d10da417..5203ee4e6435 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c @@ -775,7 +775,7 @@ u64 __init e820__memblock_alloc_reserved(u64 size, u64 align) { u64 addr; - addr = __memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); + addr = memblock_phys_alloc(size, align); if (addr) { e820__range_update_kexec(addr, size, E820_TYPE_RAM, E820_TYPE_RESERVED); pr_info("update e820_table_kexec for e820__memblock_alloc_reserved()\n"); diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index 12c1b7a83ed7..dfb6c4df639a 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -195,15 +195,11 @@ static void __init alloc_node_data(int nid) * Allocate node data. Try node-local memory and then any node. * Never allocate in DMA zone. */ - nd_pa = memblock_phys_alloc_nid(nd_size, SMP_CACHE_BYTES, nid); + nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid); if (!nd_pa) { - nd_pa = __memblock_alloc_base(nd_size, SMP_CACHE_BYTES, - MEMBLOCK_ALLOC_ACCESSIBLE); - if (!nd_pa) { - pr_err("Cannot find %zu bytes in any node (initial node: %d)\n", - nd_size, nid); - return; - } + pr_err("Cannot find %zu bytes in any node (initial node: %d)\n", + nd_size, nid); + return; } nd = __va(nd_pa); diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c index e773063c6de9..8c07d7da5256 100644 --- a/drivers/of/of_reserved_mem.c +++ b/drivers/of/of_reserved_mem.c @@ -31,13 +31,10 @@ static int __init early_init_dt_alloc_reserved_memory_arch(phys_addr_t size, phys_addr_t *res_base) { phys_addr_t base; - /* - * We use __memblock_alloc_base() because memblock_alloc_base() - * panic()s on allocation failure. - */ + end = !end ? MEMBLOCK_ALLOC_ANYWHERE : end; align = !align ? SMP_CACHE_BYTES : align; - base = __memblock_alloc_base(size, align, end); + base = memblock_phys_alloc_range(size, align, 0, end); if (!base) return -ENOMEM; diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 7caecb42bfea..017aeb223b24 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -452,8 +452,6 @@ static inline bool memblock_bottom_up(void) phys_addr_t memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr); -phys_addr_t __memblock_alloc_base(phys_addr_t size, phys_addr_t align, - phys_addr_t max_addr); phys_addr_t memblock_phys_mem_size(void); phys_addr_t memblock_reserved_size(void); phys_addr_t memblock_mem_size(unsigned long limit_pfn); diff --git a/mm/memblock.c b/mm/memblock.c index d0b76bb7340d..5b6aeb8108d9 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1306,17 +1306,12 @@ again: return ret; } -phys_addr_t __init __memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr) -{ - return memblock_alloc_range_nid(size, align, 0, max_addr, NUMA_NO_NODE, - MEMBLOCK_NONE); -} - phys_addr_t __init memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr) { phys_addr_t alloc; - alloc = __memblock_alloc_base(size, align, max_addr); + alloc = memblock_alloc_range_nid(size, align, 0, max_addr, NUMA_NO_NODE, + MEMBLOCK_NONE); if (alloc == 0) panic("ERROR: Failed to allocate %pa bytes below %pa.\n", From 0ba9e6edd4c2e563a9b34c8a46649218814a363f Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:29:35 -0700 Subject: [PATCH 15/38] memblock: drop memblock_alloc_base() The memblock_alloc_base() function tries to allocate a memory up to the limit specified by its max_addr parameter and panics if the allocation fails. Replace its usage with memblock_phys_alloc_range() and make the callers check the return value and panic in case of error. Link: http://lkml.kernel.org/r/1548057848-15136-10-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Acked-by: Michael Ellerman [powerpc] Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/powerpc/kernel/rtas.c | 6 +++++- arch/powerpc/mm/hash_utils_64.c | 8 ++++++-- arch/s390/kernel/smp.c | 6 +++++- drivers/macintosh/smu.c | 2 +- include/linux/memblock.h | 2 -- mm/memblock.c | 14 -------------- 6 files changed, 17 insertions(+), 21 deletions(-) diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c index de35bd8f047f..fbc676160adf 100644 --- a/arch/powerpc/kernel/rtas.c +++ b/arch/powerpc/kernel/rtas.c @@ -1187,7 +1187,11 @@ void __init rtas_initialize(void) ibm_suspend_me_token = rtas_token("ibm,suspend-me"); } #endif - rtas_rmo_buf = memblock_alloc_base(RTAS_RMOBUF_MAX, PAGE_SIZE, rtas_region); + rtas_rmo_buf = memblock_phys_alloc_range(RTAS_RMOBUF_MAX, PAGE_SIZE, + 0, rtas_region); + if (!rtas_rmo_buf) + panic("ERROR: RTAS: Failed to allocate %lx bytes below %pa\n", + PAGE_SIZE, &rtas_region); #ifdef CONFIG_RTAS_ERROR_LOGGING rtas_last_error_token = rtas_token("rtas-last-error"); diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 3d4b2399192f..880a366c229c 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -882,8 +882,12 @@ static void __init htab_initialize(void) } #endif /* CONFIG_PPC_CELL */ - table = memblock_alloc_base(htab_size_bytes, htab_size_bytes, - limit); + table = memblock_phys_alloc_range(htab_size_bytes, + htab_size_bytes, + 0, limit); + if (!table) + panic("ERROR: Failed to allocate %pa bytes below %pa\n", + &htab_size_bytes, &limit); DBG("Hash table allocated at %lx, size: %lx\n", table, htab_size_bytes); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index b198ece2aad6..5e3cccc408b8 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -656,7 +656,11 @@ void __init smp_save_dump_cpus(void) /* No previous system present, normal boot. */ return; /* Allocate a page as dumping area for the store status sigps */ - page = memblock_alloc_base(PAGE_SIZE, PAGE_SIZE, 1UL << 31); + page = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, 1UL << 31); + if (!page) + panic("ERROR: Failed to allocate %x bytes below %lx\n", + PAGE_SIZE, 1UL << 31); + /* Set multi-threading state to the previous system. */ pcpu_set_smt(sclp.mtid_prev); boot_cpu_addr = stap(); diff --git a/drivers/macintosh/smu.c b/drivers/macintosh/smu.c index 0a0b8e1f4236..42cf68d15da3 100644 --- a/drivers/macintosh/smu.c +++ b/drivers/macintosh/smu.c @@ -485,7 +485,7 @@ int __init smu_init (void) * SMU based G5s need some memory below 2Gb. Thankfully this is * called at a time where memblock is still available. */ - smu_cmdbuf_abs = memblock_alloc_base(4096, 4096, 0x80000000UL); + smu_cmdbuf_abs = memblock_phys_alloc_range(4096, 4096, 0, 0x80000000UL); if (smu_cmdbuf_abs == 0) { printk(KERN_ERR "SMU: Command buffer allocation failed !\n"); ret = -EINVAL; diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 017aeb223b24..0c8375120322 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -450,8 +450,6 @@ static inline bool memblock_bottom_up(void) return memblock.bottom_up; } -phys_addr_t memblock_alloc_base(phys_addr_t size, phys_addr_t align, - phys_addr_t max_addr); phys_addr_t memblock_phys_mem_size(void); phys_addr_t memblock_reserved_size(void); phys_addr_t memblock_mem_size(unsigned long limit_pfn); diff --git a/mm/memblock.c b/mm/memblock.c index 5b6aeb8108d9..42fe65447d8b 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1306,20 +1306,6 @@ again: return ret; } -phys_addr_t __init memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr) -{ - phys_addr_t alloc; - - alloc = memblock_alloc_range_nid(size, align, 0, max_addr, NUMA_NO_NODE, - MEMBLOCK_NONE); - - if (alloc == 0) - panic("ERROR: Failed to allocate %pa bytes below %pa.\n", - &size, &max_addr); - - return alloc; -} - phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) { phys_addr_t res = memblock_phys_alloc_nid(size, align, nid); From 92d12f9544b7b133b54cb64f687f3f45fce0043c Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:29:41 -0700 Subject: [PATCH 16/38] memblock: refactor internal allocation functions Currently, memblock has several internal functions with overlapping functionality. They all call memblock_find_in_range_node() to find free memory and then reserve the allocated range and mark it with kmemleak. However, there is difference in the allocation constraints and in fallback strategies. The allocations returning physical address first attempt to find free memory on the specified node within mirrored memory regions, then retry on the same node without the requirement for memory mirroring and finally fall back to all available memory. The allocations returning virtual address start with clamping the allowed range to memblock.current_limit, attempt to allocate from the specified node from regions with mirroring and with user defined minimal address. If such allocation fails, next attempt is done with node restriction lifted. Next, the allocation is retried with minimal address reset to zero and at last without the requirement for mirrored regions. Let's consolidate various fallbacks handling and make them more consistent for physical and virtual variants. Most of the fallback handling is moved to memblock_alloc_range_nid() and it now handles node and mirror fallbacks. The memblock_alloc_internal() uses memblock_alloc_range_nid() to get a physical address of the allocated range and converts it to virtual address. The fallback for allocation below the specified minimal address remains in memblock_alloc_internal() because memblock_alloc_range_nid() is used by CMA with exact requirement for lower bounds. The memblock_phys_alloc_nid() function is completely dropped as it is not used anywhere outside memblock and its only usage can be replaced by a call to memblock_alloc_range_nid(). [rppt@linux.ibm.com: fix parameter order in memblock_phys_alloc_try_nid()] Link: http://lkml.kernel.org/r/20190203113915.GC8620@rapoport-lnx Link: http://lkml.kernel.org/r/1548057848-15136-11-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Tested-by: Michael Ellerman Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/memblock.h | 1 - mm/memblock.c | 171 ++++++++++++++++++--------------------- 2 files changed, 77 insertions(+), 95 deletions(-) diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 0c8375120322..c1315c331a8e 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -327,7 +327,6 @@ static inline int memblock_get_region_node(const struct memblock_region *r) phys_addr_t memblock_phys_alloc_range(phys_addr_t size, phys_addr_t align, phys_addr_t start, phys_addr_t end); -phys_addr_t memblock_phys_alloc_nid(phys_addr_t size, phys_addr_t align, int nid); phys_addr_t memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid); static inline phys_addr_t memblock_phys_alloc(phys_addr_t size, diff --git a/mm/memblock.c b/mm/memblock.c index 42fe65447d8b..31e89dac9a23 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1255,30 +1255,84 @@ int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, } #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ +/** + * memblock_alloc_range_nid - allocate boot memory block + * @size: size of memory block to be allocated in bytes + * @align: alignment of the region and block's size + * @start: the lower bound of the memory region to allocate (phys address) + * @end: the upper bound of the memory region to allocate (phys address) + * @nid: nid of the free area to find, %NUMA_NO_NODE for any node + * + * The allocation is performed from memory region limited by + * memblock.current_limit if @max_addr == %MEMBLOCK_ALLOC_ACCESSIBLE. + * + * If the specified node can not hold the requested memory the + * allocation falls back to any node in the system + * + * For systems with memory mirroring, the allocation is attempted first + * from the regions with mirroring enabled and then retried from any + * memory region. + * + * In addition, function sets the min_count to 0 using kmemleak_alloc_phys for + * allocated boot memory block, so that it is never reported as leaks. + * + * Return: + * Physical address of allocated memory block on success, %0 on failure. + */ static phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size, phys_addr_t align, phys_addr_t start, - phys_addr_t end, int nid, - enum memblock_flags flags) + phys_addr_t end, int nid) { + enum memblock_flags flags = choose_memblock_flags(); phys_addr_t found; + if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) + nid = NUMA_NO_NODE; + if (!align) { /* Can't use WARNs this early in boot on powerpc */ dump_stack(); align = SMP_CACHE_BYTES; } + if (end > memblock.current_limit) + end = memblock.current_limit; + +again: found = memblock_find_in_range_node(size, align, start, end, nid, flags); - if (found && !memblock_reserve(found, size)) { + if (found && !memblock_reserve(found, size)) + goto done; + + if (nid != NUMA_NO_NODE) { + found = memblock_find_in_range_node(size, align, start, + end, NUMA_NO_NODE, + flags); + if (found && !memblock_reserve(found, size)) + goto done; + } + + if (flags & MEMBLOCK_MIRROR) { + flags &= ~MEMBLOCK_MIRROR; + pr_warn("Could not allocate %pap bytes of mirrored memory\n", + &size); + goto again; + } + + return 0; + +done: + /* Skip kmemleak for kasan_init() due to high volume. */ + if (end != MEMBLOCK_ALLOC_KASAN) /* - * The min_count is set to 0 so that memblock allocations are - * never reported as leaks. + * The min_count is set to 0 so that memblock allocated + * blocks are never reported as leaks. This is because many + * of these blocks are only referred via the physical + * address which is not looked up by kmemleak. */ kmemleak_alloc_phys(found, size, 0, 0); - return found; - } - return 0; + + return found; } phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, @@ -1286,35 +1340,13 @@ phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, phys_addr_t start, phys_addr_t end) { - return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE, - MEMBLOCK_NONE); -} - -phys_addr_t __init memblock_phys_alloc_nid(phys_addr_t size, phys_addr_t align, int nid) -{ - enum memblock_flags flags = choose_memblock_flags(); - phys_addr_t ret; - -again: - ret = memblock_alloc_range_nid(size, align, 0, - MEMBLOCK_ALLOC_ACCESSIBLE, nid, flags); - - if (!ret && (flags & MEMBLOCK_MIRROR)) { - flags &= ~MEMBLOCK_MIRROR; - goto again; - } - return ret; + return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE); } phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) { - phys_addr_t res = memblock_phys_alloc_nid(size, align, nid); - - if (res) - return res; return memblock_alloc_range_nid(size, align, 0, - MEMBLOCK_ALLOC_ACCESSIBLE, - NUMA_NO_NODE, MEMBLOCK_NONE); + MEMBLOCK_ALLOC_ACCESSIBLE, nid); } /** @@ -1325,19 +1357,13 @@ phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t ali * @max_addr: the upper bound of the memory region to allocate (phys address) * @nid: nid of the free area to find, %NUMA_NO_NODE for any node * + * Allocates memory block using memblock_alloc_range_nid() and + * converts the returned physical address to virtual. + * * The @min_addr limit is dropped if it can not be satisfied and the allocation - * will fall back to memory below @min_addr. Also, allocation may fall back - * to any node in the system if the specified node can not - * hold the requested memory. - * - * The allocation is performed from memory region limited by - * memblock.current_limit if @max_addr == %MEMBLOCK_ALLOC_ACCESSIBLE. - * - * The phys address of allocated boot memory block is converted to virtual and - * allocated memory is reset to 0. - * - * In addition, function sets the min_count to 0 using kmemleak_alloc for - * allocated boot memory block, so that it is never reported as leaks. + * will fall back to memory below @min_addr. Other constraints, such + * as node and mirrored memory will be handled again in + * memblock_alloc_range_nid(). * * Return: * Virtual address of allocated memory block on success, NULL on failure. @@ -1348,11 +1374,6 @@ static void * __init memblock_alloc_internal( int nid) { phys_addr_t alloc; - void *ptr; - enum memblock_flags flags = choose_memblock_flags(); - - if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) - nid = NUMA_NO_NODE; /* * Detect any accidental use of these APIs after slab is ready, as at @@ -1362,54 +1383,16 @@ static void * __init memblock_alloc_internal( if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, nid); - if (!align) { - dump_stack(); - align = SMP_CACHE_BYTES; - } + alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid); - if (max_addr > memblock.current_limit) - max_addr = memblock.current_limit; -again: - alloc = memblock_find_in_range_node(size, align, min_addr, max_addr, - nid, flags); - if (alloc && !memblock_reserve(alloc, size)) - goto done; + /* retry allocation without lower limit */ + if (!alloc && min_addr) + alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid); - if (nid != NUMA_NO_NODE) { - alloc = memblock_find_in_range_node(size, align, min_addr, - max_addr, NUMA_NO_NODE, - flags); - if (alloc && !memblock_reserve(alloc, size)) - goto done; - } + if (!alloc) + return NULL; - if (min_addr) { - min_addr = 0; - goto again; - } - - if (flags & MEMBLOCK_MIRROR) { - flags &= ~MEMBLOCK_MIRROR; - pr_warn("Could not allocate %pap bytes of mirrored memory\n", - &size); - goto again; - } - - return NULL; -done: - ptr = phys_to_virt(alloc); - - /* Skip kmemleak for kasan_init() due to high volume. */ - if (max_addr != MEMBLOCK_ALLOC_KASAN) - /* - * The min_count is set to 0 so that bootmem allocated - * blocks are never reported as leaks. This is because many - * of these blocks are only referred via the physical - * address which is not looked up by kmemleak. - */ - kmemleak_alloc(ptr, size, 0, 0); - - return ptr; + return phys_to_virt(alloc); } /** From c366ea89fa40f244d1210e74485fce110835b71b Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:29:46 -0700 Subject: [PATCH 17/38] memblock: make memblock_find_in_range_node() and choose_memblock_flags() static These functions are not used outside memblock. Make them static. Link: http://lkml.kernel.org/r/1548057848-15136-12-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/memblock.h | 4 ---- mm/memblock.c | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/include/linux/memblock.h b/include/linux/memblock.h index c1315c331a8e..c077227e6d53 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -108,9 +108,6 @@ void memblock_discard(void); #define memblock_dbg(fmt, ...) \ if (memblock_debug) printk(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__) -phys_addr_t memblock_find_in_range_node(phys_addr_t size, phys_addr_t align, - phys_addr_t start, phys_addr_t end, - int nid, enum memblock_flags flags); phys_addr_t memblock_find_in_range(phys_addr_t start, phys_addr_t end, phys_addr_t size, phys_addr_t align); void memblock_allow_resize(void); @@ -127,7 +124,6 @@ int memblock_clear_hotplug(phys_addr_t base, phys_addr_t size); int memblock_mark_mirror(phys_addr_t base, phys_addr_t size); int memblock_mark_nomap(phys_addr_t base, phys_addr_t size); int memblock_clear_nomap(phys_addr_t base, phys_addr_t size); -enum memblock_flags choose_memblock_flags(void); unsigned long memblock_free_all(void); void reset_node_managed_pages(pg_data_t *pgdat); diff --git a/mm/memblock.c b/mm/memblock.c index 31e89dac9a23..618f94a1eedb 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -132,7 +132,7 @@ static int memblock_can_resize __initdata_memblock; static int memblock_memory_in_slab __initdata_memblock = 0; static int memblock_reserved_in_slab __initdata_memblock = 0; -enum memblock_flags __init_memblock choose_memblock_flags(void) +static enum memblock_flags __init_memblock choose_memblock_flags(void) { return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE; } @@ -261,7 +261,7 @@ __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end, * Return: * Found address on success, 0 on failure. */ -phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, +static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, phys_addr_t align, phys_addr_t start, phys_addr_t end, int nid, enum memblock_flags flags) From 9415673e3e19f76b1b83be2bc8844bdeff87a05d Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:29:50 -0700 Subject: [PATCH 18/38] arch: use memblock_alloc() instead of memblock_alloc_from(size, align, 0) The last parameter of memblock_alloc_from() is the lower limit for the memory allocation. When it is 0, the call is equivalent to memblock_alloc(). Link: http://lkml.kernel.org/r/1548057848-15136-13-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Acked-by: Paul Burton # MIPS part Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/alpha/kernel/core_cia.c | 2 +- arch/alpha/kernel/pci_iommu.c | 4 ++-- arch/alpha/kernel/setup.c | 2 +- arch/ia64/kernel/mca.c | 3 +-- arch/mips/kernel/traps.c | 2 +- arch/sparc/kernel/prom_32.c | 2 +- arch/sparc/mm/init_32.c | 2 +- arch/sparc/mm/srmmu.c | 10 +++++----- 8 files changed, 13 insertions(+), 14 deletions(-) diff --git a/arch/alpha/kernel/core_cia.c b/arch/alpha/kernel/core_cia.c index 867e8730b0c5..466cd44d8b36 100644 --- a/arch/alpha/kernel/core_cia.c +++ b/arch/alpha/kernel/core_cia.c @@ -331,7 +331,7 @@ cia_prepare_tbia_workaround(int window) long i; /* Use minimal 1K map. */ - ppte = memblock_alloc_from(CIA_BROKEN_TBIA_SIZE, 32768, 0); + ppte = memblock_alloc(CIA_BROKEN_TBIA_SIZE, 32768); pte = (virt_to_phys(ppte) >> (PAGE_SHIFT - 1)) | 1; for (i = 0; i < CIA_BROKEN_TBIA_SIZE / sizeof(unsigned long); ++i) diff --git a/arch/alpha/kernel/pci_iommu.c b/arch/alpha/kernel/pci_iommu.c index aa0f50d0f823..e4cf77b07742 100644 --- a/arch/alpha/kernel/pci_iommu.c +++ b/arch/alpha/kernel/pci_iommu.c @@ -87,13 +87,13 @@ iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base, printk("%s: couldn't allocate arena ptes from node %d\n" " falling back to system-wide allocation\n", __func__, nid); - arena->ptes = memblock_alloc_from(mem_size, align, 0); + arena->ptes = memblock_alloc(mem_size, align); } #else /* CONFIG_DISCONTIGMEM */ arena = memblock_alloc(sizeof(*arena), SMP_CACHE_BYTES); - arena->ptes = memblock_alloc_from(mem_size, align, 0); + arena->ptes = memblock_alloc(mem_size, align); #endif /* CONFIG_DISCONTIGMEM */ diff --git a/arch/alpha/kernel/setup.c b/arch/alpha/kernel/setup.c index 4b5b1b244f86..5d4c76a77a9f 100644 --- a/arch/alpha/kernel/setup.c +++ b/arch/alpha/kernel/setup.c @@ -293,7 +293,7 @@ move_initrd(unsigned long mem_limit) unsigned long size; size = initrd_end - initrd_start; - start = memblock_alloc_from(PAGE_ALIGN(size), PAGE_SIZE, 0); + start = memblock_alloc(PAGE_ALIGN(size), PAGE_SIZE); if (!start || __pa(start) + size > mem_limit) { initrd_start = initrd_end = 0; return NULL; diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index 91bd1e129379..74d148bd4876 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -1835,8 +1835,7 @@ format_mca_init_stack(void *mca_data, unsigned long offset, /* Caller prevents this from being called after init */ static void * __ref mca_bootmem(void) { - return memblock_alloc_from(sizeof(struct ia64_mca_cpu), - KERNEL_STACK_SIZE, 0); + return memblock_alloc(sizeof(struct ia64_mca_cpu), KERNEL_STACK_SIZE); } /* Do per-CPU MCA-related initialization. */ diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 42d411125690..fc511ecefec6 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -2293,7 +2293,7 @@ void __init trap_init(void) phys_addr_t ebase_pa; ebase = (unsigned long) - memblock_alloc_from(size, 1 << fls(size), 0); + memblock_alloc(size, 1 << fls(size)); /* * Try to ensure ebase resides in KSeg0 if possible. diff --git a/arch/sparc/kernel/prom_32.c b/arch/sparc/kernel/prom_32.c index 42d7f2a7da6d..38940afaa696 100644 --- a/arch/sparc/kernel/prom_32.c +++ b/arch/sparc/kernel/prom_32.c @@ -32,7 +32,7 @@ void * __init prom_early_alloc(unsigned long size) { void *ret; - ret = memblock_alloc_from(size, SMP_CACHE_BYTES, 0UL); + ret = memblock_alloc(size, SMP_CACHE_BYTES); if (ret != NULL) memset(ret, 0, size); diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c index d900952bfc5f..a8ff29821bdb 100644 --- a/arch/sparc/mm/init_32.c +++ b/arch/sparc/mm/init_32.c @@ -264,7 +264,7 @@ void __init mem_init(void) i = last_valid_pfn >> ((20 - PAGE_SHIFT) + 5); i += 1; sparc_valid_addr_bitmap = (unsigned long *) - memblock_alloc_from(i << 2, SMP_CACHE_BYTES, 0UL); + memblock_alloc(i << 2, SMP_CACHE_BYTES); if (sparc_valid_addr_bitmap == NULL) { prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n"); diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index b609362e846f..a400ec319564 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -303,13 +303,13 @@ static void __init srmmu_nocache_init(void) bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT; - srmmu_nocache_pool = memblock_alloc_from(srmmu_nocache_size, - SRMMU_NOCACHE_ALIGN_MAX, 0UL); + srmmu_nocache_pool = memblock_alloc(srmmu_nocache_size, + SRMMU_NOCACHE_ALIGN_MAX); memset(srmmu_nocache_pool, 0, srmmu_nocache_size); srmmu_nocache_bitmap = - memblock_alloc_from(BITS_TO_LONGS(bitmap_bits) * sizeof(long), - SMP_CACHE_BYTES, 0UL); + memblock_alloc(BITS_TO_LONGS(bitmap_bits) * sizeof(long), + SMP_CACHE_BYTES); bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits); srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); @@ -467,7 +467,7 @@ static void __init sparc_context_init(int numctx) unsigned long size; size = numctx * sizeof(struct ctx_list); - ctx_list_pool = memblock_alloc_from(size, SMP_CACHE_BYTES, 0UL); + ctx_list_pool = memblock_alloc(size, SMP_CACHE_BYTES); for (ctx = 0; ctx < numctx; ctx++) { struct ctx_list *clist; From 0240dfd5b4de79980a484cef8ddf0d52d52bce40 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:29:56 -0700 Subject: [PATCH 19/38] arch: don't memset(0) memory returned by memblock_alloc() memblock_alloc() already clears the allocated memory, no point in doing it twice. Link: http://lkml.kernel.org/r/1548057848-15136-14-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Acked-by: Geert Uytterhoeven [m68k] Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/c6x/mm/init.c | 1 - arch/h8300/mm/init.c | 1 - arch/ia64/kernel/mca.c | 2 -- arch/m68k/mm/mcfmmu.c | 1 - arch/microblaze/mm/init.c | 6 ++---- arch/sparc/kernel/prom_32.c | 2 -- 6 files changed, 2 insertions(+), 11 deletions(-) diff --git a/arch/c6x/mm/init.c b/arch/c6x/mm/init.c index af5ada0520be..e83c04654238 100644 --- a/arch/c6x/mm/init.c +++ b/arch/c6x/mm/init.c @@ -40,7 +40,6 @@ void __init paging_init(void) empty_zero_page = (unsigned long) memblock_alloc(PAGE_SIZE, PAGE_SIZE); - memset((void *)empty_zero_page, 0, PAGE_SIZE); /* * Set up user data space diff --git a/arch/h8300/mm/init.c b/arch/h8300/mm/init.c index 6519252ac4db..a1578904ad4e 100644 --- a/arch/h8300/mm/init.c +++ b/arch/h8300/mm/init.c @@ -68,7 +68,6 @@ void __init paging_init(void) * to a couple of allocated pages. */ empty_zero_page = (unsigned long)memblock_alloc(PAGE_SIZE, PAGE_SIZE); - memset((void *)empty_zero_page, 0, PAGE_SIZE); /* * Set up SFC/DFC registers (user data space). diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index 74d148bd4876..370bc349619a 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -400,8 +400,6 @@ ia64_log_init(int sal_info_type) // set up OS data structures to hold error info IA64_LOG_ALLOCATE(sal_info_type, max_size); - memset(IA64_LOG_CURR_BUFFER(sal_info_type), 0, max_size); - memset(IA64_LOG_NEXT_BUFFER(sal_info_type), 0, max_size); } /* diff --git a/arch/m68k/mm/mcfmmu.c b/arch/m68k/mm/mcfmmu.c index 0de4999a3810..492f953db31b 100644 --- a/arch/m68k/mm/mcfmmu.c +++ b/arch/m68k/mm/mcfmmu.c @@ -44,7 +44,6 @@ void __init paging_init(void) int i; empty_zero_page = (void *) memblock_alloc(PAGE_SIZE, PAGE_SIZE); - memset((void *) empty_zero_page, 0, PAGE_SIZE); pg_dir = swapper_pg_dir; memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir)); diff --git a/arch/microblaze/mm/init.c b/arch/microblaze/mm/init.c index 44f4b8910c21..bd1cd4bff449 100644 --- a/arch/microblaze/mm/init.c +++ b/arch/microblaze/mm/init.c @@ -376,10 +376,8 @@ void * __ref zalloc_maybe_bootmem(size_t size, gfp_t mask) if (mem_init_done) p = kzalloc(size, mask); - else { + else p = memblock_alloc(size, SMP_CACHE_BYTES); - if (p) - memset(p, 0, size); - } + return p; } diff --git a/arch/sparc/kernel/prom_32.c b/arch/sparc/kernel/prom_32.c index 38940afaa696..e7126ca4ead7 100644 --- a/arch/sparc/kernel/prom_32.c +++ b/arch/sparc/kernel/prom_32.c @@ -33,8 +33,6 @@ void * __init prom_early_alloc(unsigned long size) void *ret; ret = memblock_alloc(size, SMP_CACHE_BYTES); - if (ret != NULL) - memset(ret, 0, size); prom_early_allocated += size; From d80db5c1ed8ddcb836766adfe773b6a6768df831 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:00 -0700 Subject: [PATCH 20/38] ia64: add checks for the return value of memblock_alloc*() Add panic() calls if memblock_alloc*() returns NULL. Most of the changes are simply addition of if(!ptr) panic(); statements after the calls to memblock_alloc*() variants. Exceptions are create_mem_map_page_table() and ia64_log_init() that were slightly refactored to accommodate the change. Link: http://lkml.kernel.org/r/1548057848-15136-15-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/ia64/kernel/mca.c | 20 +++++++++++------ arch/ia64/mm/contig.c | 8 +++++-- arch/ia64/mm/discontig.c | 4 ++++ arch/ia64/mm/init.c | 38 ++++++++++++++++++++++++++------- arch/ia64/mm/tlb.c | 6 ++++++ arch/ia64/sn/kernel/io_common.c | 3 +++ arch/ia64/sn/kernel/setup.c | 12 ++++++++++- 7 files changed, 74 insertions(+), 17 deletions(-) diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index 370bc349619a..5cabb3fd159f 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -359,11 +359,6 @@ typedef struct ia64_state_log_s static ia64_state_log_t ia64_state_log[IA64_MAX_LOG_TYPES]; -#define IA64_LOG_ALLOCATE(it, size) \ - {ia64_state_log[it].isl_log[IA64_LOG_CURR_INDEX(it)] = \ - (ia64_err_rec_t *)memblock_alloc(size, SMP_CACHE_BYTES); \ - ia64_state_log[it].isl_log[IA64_LOG_NEXT_INDEX(it)] = \ - (ia64_err_rec_t *)memblock_alloc(size, SMP_CACHE_BYTES);} #define IA64_LOG_LOCK_INIT(it) spin_lock_init(&ia64_state_log[it].isl_lock) #define IA64_LOG_LOCK(it) spin_lock_irqsave(&ia64_state_log[it].isl_lock, s) #define IA64_LOG_UNLOCK(it) spin_unlock_irqrestore(&ia64_state_log[it].isl_lock,s) @@ -378,6 +373,19 @@ static ia64_state_log_t ia64_state_log[IA64_MAX_LOG_TYPES]; #define IA64_LOG_CURR_BUFFER(it) (void *)((ia64_state_log[it].isl_log[IA64_LOG_CURR_INDEX(it)])) #define IA64_LOG_COUNT(it) ia64_state_log[it].isl_count +static inline void ia64_log_allocate(int it, u64 size) +{ + ia64_state_log[it].isl_log[IA64_LOG_CURR_INDEX(it)] = + (ia64_err_rec_t *)memblock_alloc(size, SMP_CACHE_BYTES); + if (!ia64_state_log[it].isl_log[IA64_LOG_CURR_INDEX(it)]) + panic("%s: Failed to allocate %llu bytes\n", __func__, size); + + ia64_state_log[it].isl_log[IA64_LOG_NEXT_INDEX(it)] = + (ia64_err_rec_t *)memblock_alloc(size, SMP_CACHE_BYTES); + if (!ia64_state_log[it].isl_log[IA64_LOG_NEXT_INDEX(it)]) + panic("%s: Failed to allocate %llu bytes\n", __func__, size); +} + /* * ia64_log_init * Reset the OS ia64 log buffer @@ -399,7 +407,7 @@ ia64_log_init(int sal_info_type) return; // set up OS data structures to hold error info - IA64_LOG_ALLOCATE(sal_info_type, max_size); + ia64_log_allocate(sal_info_type, max_size); } /* diff --git a/arch/ia64/mm/contig.c b/arch/ia64/mm/contig.c index 6e447234205c..d29fb6b9fa33 100644 --- a/arch/ia64/mm/contig.c +++ b/arch/ia64/mm/contig.c @@ -84,9 +84,13 @@ skip: static inline void alloc_per_cpu_data(void) { - cpu_data = memblock_alloc_from(PERCPU_PAGE_SIZE * num_possible_cpus(), - PERCPU_PAGE_SIZE, + size_t size = PERCPU_PAGE_SIZE * num_possible_cpus(); + + cpu_data = memblock_alloc_from(size, PERCPU_PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); + if (!cpu_data) + panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", + __func__, size, PERCPU_PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); } /** diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index f9c36750c6a4..05490dd073e6 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -454,6 +454,10 @@ static void __init *memory_less_node_alloc(int nid, unsigned long pernodesize) __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, bestnode); + if (!ptr) + panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%lx\n", + __func__, pernodesize, PERCPU_PAGE_SIZE, bestnode, + __pa(MAX_DMA_ADDRESS)); return ptr; } diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 29d841525ca1..e49200e31750 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -444,23 +444,45 @@ int __init create_mem_map_page_table(u64 start, u64 end, void *arg) for (address = start_page; address < end_page; address += PAGE_SIZE) { pgd = pgd_offset_k(address); - if (pgd_none(*pgd)) - pgd_populate(&init_mm, pgd, memblock_alloc_node(PAGE_SIZE, PAGE_SIZE, node)); + if (pgd_none(*pgd)) { + pud = memblock_alloc_node(PAGE_SIZE, PAGE_SIZE, node); + if (!pud) + goto err_alloc; + pgd_populate(&init_mm, pgd, pud); + } pud = pud_offset(pgd, address); - if (pud_none(*pud)) - pud_populate(&init_mm, pud, memblock_alloc_node(PAGE_SIZE, PAGE_SIZE, node)); + if (pud_none(*pud)) { + pmd = memblock_alloc_node(PAGE_SIZE, PAGE_SIZE, node); + if (!pmd) + goto err_alloc; + pud_populate(&init_mm, pud, pmd); + } pmd = pmd_offset(pud, address); - if (pmd_none(*pmd)) - pmd_populate_kernel(&init_mm, pmd, memblock_alloc_node(PAGE_SIZE, PAGE_SIZE, node)); + if (pmd_none(*pmd)) { + pte = memblock_alloc_node(PAGE_SIZE, PAGE_SIZE, node); + if (!pte) + goto err_alloc; + pmd_populate_kernel(&init_mm, pmd, pte); + } pte = pte_offset_kernel(pmd, address); - if (pte_none(*pte)) - set_pte(pte, pfn_pte(__pa(memblock_alloc_node(PAGE_SIZE, PAGE_SIZE, node)) >> PAGE_SHIFT, + if (pte_none(*pte)) { + void *page = memblock_alloc_node(PAGE_SIZE, PAGE_SIZE, + node); + if (!page) + goto err_alloc; + set_pte(pte, pfn_pte(__pa(page) >> PAGE_SHIFT, PAGE_KERNEL)); + } } return 0; + +err_alloc: + panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d\n", + __func__, PAGE_SIZE, PAGE_SIZE, node); + return -ENOMEM; } struct memmap_init_callback_data { diff --git a/arch/ia64/mm/tlb.c b/arch/ia64/mm/tlb.c index 9340bcb4f29c..5fc89aabdce1 100644 --- a/arch/ia64/mm/tlb.c +++ b/arch/ia64/mm/tlb.c @@ -61,8 +61,14 @@ mmu_context_init (void) { ia64_ctx.bitmap = memblock_alloc((ia64_ctx.max_ctx + 1) >> 3, SMP_CACHE_BYTES); + if (!ia64_ctx.bitmap) + panic("%s: Failed to allocate %u bytes\n", __func__, + (ia64_ctx.max_ctx + 1) >> 3); ia64_ctx.flushmap = memblock_alloc((ia64_ctx.max_ctx + 1) >> 3, SMP_CACHE_BYTES); + if (!ia64_ctx.flushmap) + panic("%s: Failed to allocate %u bytes\n", __func__, + (ia64_ctx.max_ctx + 1) >> 3); } /* diff --git a/arch/ia64/sn/kernel/io_common.c b/arch/ia64/sn/kernel/io_common.c index 8df13d0d96fa..d46847323ef6 100644 --- a/arch/ia64/sn/kernel/io_common.c +++ b/arch/ia64/sn/kernel/io_common.c @@ -394,6 +394,9 @@ void __init hubdev_init_node(nodepda_t * npda, cnodeid_t node) hubdev_info = (struct hubdev_info *)memblock_alloc_node(size, SMP_CACHE_BYTES, node); + if (!hubdev_info) + panic("%s: Failed to allocate %d bytes align=0x%x nid=%d\n", + __func__, size, SMP_CACHE_BYTES, node); npda->pdinfo = (void *)hubdev_info; } diff --git a/arch/ia64/sn/kernel/setup.c b/arch/ia64/sn/kernel/setup.c index a6d40a2c5bff..e6a5049ef503 100644 --- a/arch/ia64/sn/kernel/setup.c +++ b/arch/ia64/sn/kernel/setup.c @@ -513,6 +513,10 @@ static void __init sn_init_pdas(char **cmdline_p) nodepdaindr[cnode] = memblock_alloc_node(sizeof(nodepda_t), SMP_CACHE_BYTES, cnode); + if (!nodepdaindr[cnode]) + panic("%s: Failed to allocate %lu bytes align=0x%x nid=%d\n", + __func__, sizeof(nodepda_t), SMP_CACHE_BYTES, + cnode); memset(nodepdaindr[cnode]->phys_cpuid, -1, sizeof(nodepdaindr[cnode]->phys_cpuid)); spin_lock_init(&nodepdaindr[cnode]->ptc_lock); @@ -521,9 +525,15 @@ static void __init sn_init_pdas(char **cmdline_p) /* * Allocate & initialize nodepda for TIOs. For now, put them on node 0. */ - for (cnode = num_online_nodes(); cnode < num_cnodes; cnode++) + for (cnode = num_online_nodes(); cnode < num_cnodes; cnode++) { nodepdaindr[cnode] = memblock_alloc_node(sizeof(nodepda_t), SMP_CACHE_BYTES, 0); + if (!nodepdaindr[cnode]) + panic("%s: Failed to allocate %lu bytes align=0x%x nid=%d\n", + __func__, sizeof(nodepda_t), SMP_CACHE_BYTES, + cnode); + } + /* * Now copy the array of nodepda pointers to each nodepda. From b1e1c869ffd1d8e84beb9d9ee4adc3507d8843d5 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:10 -0700 Subject: [PATCH 21/38] sparc: add checks for the return value of memblock_alloc*() Add panic() calls if memblock_alloc*() returns NULL. Most of the changes are simply addition of if(!ptr) panic(); statements after the calls to memblock_alloc*() variants. Exceptions are pcpu_populate_pte() and kernel_map_range() that were slightly refactored to accommodate the change. Link: http://lkml.kernel.org/r/1548057848-15136-16-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Acked-by: David S. Miller Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/sparc/kernel/prom_32.c | 2 ++ arch/sparc/kernel/setup_64.c | 6 ++++++ arch/sparc/kernel/smp_64.c | 12 ++++++++++++ arch/sparc/mm/init_64.c | 11 +++++++++++ arch/sparc/mm/srmmu.c | 8 ++++++++ 5 files changed, 39 insertions(+) diff --git a/arch/sparc/kernel/prom_32.c b/arch/sparc/kernel/prom_32.c index e7126ca4ead7..869b16c96157 100644 --- a/arch/sparc/kernel/prom_32.c +++ b/arch/sparc/kernel/prom_32.c @@ -33,6 +33,8 @@ void * __init prom_early_alloc(unsigned long size) void *ret; ret = memblock_alloc(size, SMP_CACHE_BYTES); + if (!ret) + panic("%s: Failed to allocate %lu bytes\n", __func__, size); prom_early_allocated += size; diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c index 51c4d12c0853..fd2182a5c32d 100644 --- a/arch/sparc/kernel/setup_64.c +++ b/arch/sparc/kernel/setup_64.c @@ -624,8 +624,14 @@ void __init alloc_irqstack_bootmem(void) softirq_stack[i] = memblock_alloc_node(THREAD_SIZE, THREAD_SIZE, node); + if (!softirq_stack[i]) + panic("%s: Failed to allocate %lu bytes align=%lx nid=%d\n", + __func__, THREAD_SIZE, THREAD_SIZE, node); hardirq_stack[i] = memblock_alloc_node(THREAD_SIZE, THREAD_SIZE, node); + if (!hardirq_stack[i]) + panic("%s: Failed to allocate %lu bytes align=%lx nid=%d\n", + __func__, THREAD_SIZE, THREAD_SIZE, node); } } diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index f45d876983f1..a8275fea4b70 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -1628,6 +1628,8 @@ static void __init pcpu_populate_pte(unsigned long addr) pud_t *new; new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + if (!new) + goto err_alloc; pgd_populate(&init_mm, pgd, new); } @@ -1636,6 +1638,8 @@ static void __init pcpu_populate_pte(unsigned long addr) pmd_t *new; new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + if (!new) + goto err_alloc; pud_populate(&init_mm, pud, new); } @@ -1644,8 +1648,16 @@ static void __init pcpu_populate_pte(unsigned long addr) pte_t *new; new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + if (!new) + goto err_alloc; pmd_populate_kernel(&init_mm, pmd, new); } + + return; + +err_alloc: + panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); } void __init setup_per_cpu_areas(void) diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index ef340e8f209f..f2d70ff7a284 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -1809,6 +1809,8 @@ static unsigned long __ref kernel_map_range(unsigned long pstart, new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + if (!new) + goto err_alloc; alloc_bytes += PAGE_SIZE; pgd_populate(&init_mm, pgd, new); } @@ -1822,6 +1824,8 @@ static unsigned long __ref kernel_map_range(unsigned long pstart, } new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + if (!new) + goto err_alloc; alloc_bytes += PAGE_SIZE; pud_populate(&init_mm, pud, new); } @@ -1836,6 +1840,8 @@ static unsigned long __ref kernel_map_range(unsigned long pstart, } new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + if (!new) + goto err_alloc; alloc_bytes += PAGE_SIZE; pmd_populate_kernel(&init_mm, pmd, new); } @@ -1855,6 +1861,11 @@ static unsigned long __ref kernel_map_range(unsigned long pstart, } return alloc_bytes; + +err_alloc: + panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + return -ENOMEM; } static void __init flush_all_kernel_tsbs(void) diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index a400ec319564..aaebbc00d262 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -305,11 +305,17 @@ static void __init srmmu_nocache_init(void) srmmu_nocache_pool = memblock_alloc(srmmu_nocache_size, SRMMU_NOCACHE_ALIGN_MAX); + if (!srmmu_nocache_pool) + panic("%s: Failed to allocate %lu bytes align=0x%x\n", + __func__, srmmu_nocache_size, SRMMU_NOCACHE_ALIGN_MAX); memset(srmmu_nocache_pool, 0, srmmu_nocache_size); srmmu_nocache_bitmap = memblock_alloc(BITS_TO_LONGS(bitmap_bits) * sizeof(long), SMP_CACHE_BYTES); + if (!srmmu_nocache_bitmap) + panic("%s: Failed to allocate %zu bytes\n", __func__, + BITS_TO_LONGS(bitmap_bits) * sizeof(long)); bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits); srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); @@ -468,6 +474,8 @@ static void __init sparc_context_init(int numctx) size = numctx * sizeof(struct ctx_list); ctx_list_pool = memblock_alloc(size, SMP_CACHE_BYTES); + if (!ctx_list_pool) + panic("%s: Failed to allocate %lu bytes\n", __func__, size); for (ctx = 0; ctx < numctx; ctx++) { struct ctx_list *clist; From f655f40537916d4b1d6d1a023a778697c75a4fe2 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:15 -0700 Subject: [PATCH 22/38] mm/percpu: add checks for the return value of memblock_alloc*() Add panic() calls if memblock_alloc() returns NULL. The panic() format duplicates the one used by memblock itself and in order to avoid explosion with long parameters list replace open coded allocation size calculations with a local variable. Link: http://lkml.kernel.org/r/1548057848-15136-17-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/percpu.c | 73 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/mm/percpu.c b/mm/percpu.c index c5c750781628..3f9fb3086a9b 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1086,6 +1086,7 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr, struct pcpu_chunk *chunk; unsigned long aligned_addr, lcm_align; int start_offset, offset_bits, region_size, region_bits; + size_t alloc_size; /* region calculations */ aligned_addr = tmp_addr & PAGE_MASK; @@ -1101,9 +1102,12 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr, region_size = ALIGN(start_offset + map_size, lcm_align); /* allocate chunk */ - chunk = memblock_alloc(sizeof(struct pcpu_chunk) + - BITS_TO_LONGS(region_size >> PAGE_SHIFT), - SMP_CACHE_BYTES); + alloc_size = sizeof(struct pcpu_chunk) + + BITS_TO_LONGS(region_size >> PAGE_SHIFT); + chunk = memblock_alloc(alloc_size, SMP_CACHE_BYTES); + if (!chunk) + panic("%s: Failed to allocate %zu bytes\n", __func__, + alloc_size); INIT_LIST_HEAD(&chunk->list); @@ -1114,12 +1118,25 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr, chunk->nr_pages = region_size >> PAGE_SHIFT; region_bits = pcpu_chunk_map_bits(chunk); - chunk->alloc_map = memblock_alloc(BITS_TO_LONGS(region_bits) * sizeof(chunk->alloc_map[0]), - SMP_CACHE_BYTES); - chunk->bound_map = memblock_alloc(BITS_TO_LONGS(region_bits + 1) * sizeof(chunk->bound_map[0]), - SMP_CACHE_BYTES); - chunk->md_blocks = memblock_alloc(pcpu_chunk_nr_blocks(chunk) * sizeof(chunk->md_blocks[0]), - SMP_CACHE_BYTES); + alloc_size = BITS_TO_LONGS(region_bits) * sizeof(chunk->alloc_map[0]); + chunk->alloc_map = memblock_alloc(alloc_size, SMP_CACHE_BYTES); + if (!chunk->alloc_map) + panic("%s: Failed to allocate %zu bytes\n", __func__, + alloc_size); + + alloc_size = + BITS_TO_LONGS(region_bits + 1) * sizeof(chunk->bound_map[0]); + chunk->bound_map = memblock_alloc(alloc_size, SMP_CACHE_BYTES); + if (!chunk->bound_map) + panic("%s: Failed to allocate %zu bytes\n", __func__, + alloc_size); + + alloc_size = pcpu_chunk_nr_blocks(chunk) * sizeof(chunk->md_blocks[0]); + chunk->md_blocks = memblock_alloc(alloc_size, SMP_CACHE_BYTES); + if (!chunk->md_blocks) + panic("%s: Failed to allocate %zu bytes\n", __func__, + alloc_size); + pcpu_init_md_blocks(chunk); /* manage populated page bitmap */ @@ -2044,6 +2061,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, int group, unit, i; int map_size; unsigned long tmp_addr; + size_t alloc_size; #define PCPU_SETUP_BUG_ON(cond) do { \ if (unlikely(cond)) { \ @@ -2075,14 +2093,29 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, PCPU_SETUP_BUG_ON(pcpu_verify_alloc_info(ai) < 0); /* process group information and build config tables accordingly */ - group_offsets = memblock_alloc(ai->nr_groups * sizeof(group_offsets[0]), - SMP_CACHE_BYTES); - group_sizes = memblock_alloc(ai->nr_groups * sizeof(group_sizes[0]), - SMP_CACHE_BYTES); - unit_map = memblock_alloc(nr_cpu_ids * sizeof(unit_map[0]), - SMP_CACHE_BYTES); - unit_off = memblock_alloc(nr_cpu_ids * sizeof(unit_off[0]), - SMP_CACHE_BYTES); + alloc_size = ai->nr_groups * sizeof(group_offsets[0]); + group_offsets = memblock_alloc(alloc_size, SMP_CACHE_BYTES); + if (!group_offsets) + panic("%s: Failed to allocate %zu bytes\n", __func__, + alloc_size); + + alloc_size = ai->nr_groups * sizeof(group_sizes[0]); + group_sizes = memblock_alloc(alloc_size, SMP_CACHE_BYTES); + if (!group_sizes) + panic("%s: Failed to allocate %zu bytes\n", __func__, + alloc_size); + + alloc_size = nr_cpu_ids * sizeof(unit_map[0]); + unit_map = memblock_alloc(alloc_size, SMP_CACHE_BYTES); + if (!unit_map) + panic("%s: Failed to allocate %zu bytes\n", __func__, + alloc_size); + + alloc_size = nr_cpu_ids * sizeof(unit_off[0]); + unit_off = memblock_alloc(alloc_size, SMP_CACHE_BYTES); + if (!unit_off) + panic("%s: Failed to allocate %zu bytes\n", __func__, + alloc_size); for (cpu = 0; cpu < nr_cpu_ids; cpu++) unit_map[cpu] = UINT_MAX; @@ -2148,6 +2181,9 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, pcpu_nr_slots = __pcpu_size_to_slot(pcpu_unit_size) + 2; pcpu_slot = memblock_alloc(pcpu_nr_slots * sizeof(pcpu_slot[0]), SMP_CACHE_BYTES); + if (!pcpu_slot) + panic("%s: Failed to allocate %zu bytes\n", __func__, + pcpu_nr_slots * sizeof(pcpu_slot[0])); for (i = 0; i < pcpu_nr_slots; i++) INIT_LIST_HEAD(&pcpu_slot[i]); @@ -2602,6 +2638,9 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pages_size = PFN_ALIGN(unit_pages * num_possible_cpus() * sizeof(pages[0])); pages = memblock_alloc(pages_size, SMP_CACHE_BYTES); + if (!pages) + panic("%s: Failed to allocate %zu bytes\n", __func__, + pages_size); /* allocate pages */ j = 0; From f5c7310ac73ea270e3a1acdb73d1b4817f11fd67 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:20 -0700 Subject: [PATCH 23/38] init/main: add checks for the return value of memblock_alloc*() Add panic() calls if memblock_alloc() returns NULL. The panic() format duplicates the one used by memblock itself and in order to avoid explosion with long parameters list replace open coded allocation size calculations with a local variable. Link: http://lkml.kernel.org/r/1548057848-15136-18-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- init/main.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/init/main.c b/init/main.c index c86a1c8f19f4..598e278b46f7 100644 --- a/init/main.c +++ b/init/main.c @@ -373,12 +373,20 @@ static inline void smp_prepare_cpus(unsigned int maxcpus) { } */ static void __init setup_command_line(char *command_line) { - saved_command_line = - memblock_alloc(strlen(boot_command_line) + 1, SMP_CACHE_BYTES); - initcall_command_line = - memblock_alloc(strlen(boot_command_line) + 1, SMP_CACHE_BYTES); - static_command_line = memblock_alloc(strlen(command_line) + 1, - SMP_CACHE_BYTES); + size_t len = strlen(boot_command_line) + 1; + + saved_command_line = memblock_alloc(len, SMP_CACHE_BYTES); + if (!saved_command_line) + panic("%s: Failed to allocate %zu bytes\n", __func__, len); + + initcall_command_line = memblock_alloc(len, SMP_CACHE_BYTES); + if (!initcall_command_line) + panic("%s: Failed to allocate %zu bytes\n", __func__, len); + + static_command_line = memblock_alloc(len, SMP_CACHE_BYTES); + if (!static_command_line) + panic("%s: Failed to allocate %zu bytes\n", __func__, len); + strcpy(saved_command_line, boot_command_line); strcpy(static_command_line, command_line); } @@ -770,8 +778,14 @@ static int __init initcall_blacklist(char *str) pr_debug("blacklisting initcall %s\n", str_entry); entry = memblock_alloc(sizeof(*entry), SMP_CACHE_BYTES); + if (!entry) + panic("%s: Failed to allocate %zu bytes\n", + __func__, sizeof(*entry)); entry->buf = memblock_alloc(strlen(str_entry) + 1, SMP_CACHE_BYTES); + if (!entry->buf) + panic("%s: Failed to allocate %zu bytes\n", + __func__, strlen(str_entry) + 1); strcpy(entry->buf, str_entry); list_add(&entry->next, &blacklisted_initcalls); } From a0bf842e89a3842162aa8514b9bf4611c86fee10 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:26 -0700 Subject: [PATCH 24/38] swiotlb: add checks for the return value of memblock_alloc*() Add panic() calls if memblock_alloc() returns NULL. The panic() format duplicates the one used by memblock itself and in order to avoid explosion with long parameters list replace open coded allocation size calculations with a local variable. Link: http://lkml.kernel.org/r/1548057848-15136-19-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/dma/swiotlb.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 9f5851064aea..dd6a8e2d53a7 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -199,6 +199,7 @@ void __init swiotlb_update_mem_attributes(void) int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose) { unsigned long i, bytes; + size_t alloc_size; bytes = nslabs << IO_TLB_SHIFT; @@ -211,12 +212,18 @@ int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose) * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE * between io_tlb_start and io_tlb_end. */ - io_tlb_list = memblock_alloc( - PAGE_ALIGN(io_tlb_nslabs * sizeof(int)), - PAGE_SIZE); - io_tlb_orig_addr = memblock_alloc( - PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)), - PAGE_SIZE); + alloc_size = PAGE_ALIGN(io_tlb_nslabs * sizeof(int)); + io_tlb_list = memblock_alloc(alloc_size, PAGE_SIZE); + if (!io_tlb_list) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, alloc_size, PAGE_SIZE); + + alloc_size = PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)); + io_tlb_orig_addr = memblock_alloc(alloc_size, PAGE_SIZE); + if (!io_tlb_orig_addr) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, alloc_size, PAGE_SIZE); + for (i = 0; i < io_tlb_nslabs; i++) { io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); io_tlb_orig_addr[i] = INVALID_PHYS_ADDR; From 8a7f97b902f4fb0d94b355b6b3f1fbd7154cafb9 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:31 -0700 Subject: [PATCH 25/38] treewide: add checks for the return value of memblock_alloc*() Add check for the return value of memblock_alloc*() functions and call panic() in case of error. The panic message repeats the one used by panicing memblock allocators with adjustment of parameters to include only relevant ones. The replacement was mostly automated with semantic patches like the one below with manual massaging of format strings. @@ expression ptr, size, align; @@ ptr = memblock_alloc(size, align); + if (!ptr) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", __func__, size, align); [anders.roxell@linaro.org: use '%pa' with 'phys_addr_t' type] Link: http://lkml.kernel.org/r/20190131161046.21886-1-anders.roxell@linaro.org [rppt@linux.ibm.com: fix format strings for panics after memblock_alloc] Link: http://lkml.kernel.org/r/1548950940-15145-1-git-send-email-rppt@linux.ibm.com [rppt@linux.ibm.com: don't panic if the allocation in sparse_buffer_init fails] Link: http://lkml.kernel.org/r/20190131074018.GD28876@rapoport-lnx [akpm@linux-foundation.org: fix xtensa printk warning] Link: http://lkml.kernel.org/r/1548057848-15136-20-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Signed-off-by: Anders Roxell Reviewed-by: Guo Ren [c-sky] Acked-by: Paul Burton [MIPS] Acked-by: Heiko Carstens [s390] Reviewed-by: Juergen Gross [Xen] Reviewed-by: Geert Uytterhoeven [m68k] Acked-by: Max Filippov [xtensa] Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Mark Salter Cc: Matt Turner Cc: Michael Ellerman Cc: Michal Simek Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/alpha/kernel/core_cia.c | 3 +++ arch/alpha/kernel/core_marvel.c | 6 ++++++ arch/alpha/kernel/pci-noop.c | 13 +++++++++++-- arch/alpha/kernel/pci.c | 11 ++++++++++- arch/alpha/kernel/pci_iommu.c | 12 ++++++++++++ arch/arc/mm/highmem.c | 4 ++++ arch/arm/kernel/setup.c | 6 ++++++ arch/arm/mm/mmu.c | 14 +++++++++++++- arch/arm64/kernel/setup.c | 8 +++++--- arch/arm64/mm/kasan_init.c | 10 ++++++++++ arch/c6x/mm/dma-coherent.c | 4 ++++ arch/c6x/mm/init.c | 3 +++ arch/csky/mm/highmem.c | 5 +++++ arch/h8300/mm/init.c | 3 +++ arch/m68k/atari/stram.c | 4 ++++ arch/m68k/mm/init.c | 3 +++ arch/m68k/mm/mcfmmu.c | 6 ++++++ arch/m68k/mm/motorola.c | 9 +++++++++ arch/m68k/mm/sun3mmu.c | 6 ++++++ arch/m68k/sun3/sun3dvma.c | 3 +++ arch/microblaze/mm/init.c | 8 ++++++-- arch/mips/cavium-octeon/dma-octeon.c | 3 +++ arch/mips/kernel/setup.c | 3 +++ arch/mips/kernel/traps.c | 3 +++ arch/mips/mm/init.c | 5 +++++ arch/nds32/mm/init.c | 12 ++++++++++++ arch/openrisc/mm/ioremap.c | 8 ++++++-- arch/powerpc/kernel/dt_cpu_ftrs.c | 5 +++++ arch/powerpc/kernel/pci_32.c | 3 +++ arch/powerpc/kernel/setup-common.c | 3 +++ arch/powerpc/kernel/setup_64.c | 4 ++++ arch/powerpc/lib/alloc.c | 3 +++ arch/powerpc/mm/hash_utils_64.c | 3 +++ arch/powerpc/mm/mmu_context_nohash.c | 9 +++++++++ arch/powerpc/mm/pgtable-book3e.c | 12 ++++++++++-- arch/powerpc/mm/pgtable-book3s64.c | 3 +++ arch/powerpc/mm/pgtable-radix.c | 9 ++++++++- arch/powerpc/mm/ppc_mmu_32.c | 3 +++ arch/powerpc/platforms/pasemi/iommu.c | 3 +++ arch/powerpc/platforms/powermac/nvram.c | 3 +++ arch/powerpc/platforms/powernv/opal.c | 3 +++ arch/powerpc/platforms/powernv/pci-ioda.c | 8 ++++++++ arch/powerpc/platforms/ps3/setup.c | 3 +++ arch/powerpc/sysdev/msi_bitmap.c | 3 +++ arch/s390/kernel/setup.c | 13 +++++++++++++ arch/s390/kernel/smp.c | 5 ++++- arch/s390/kernel/topology.c | 6 ++++++ arch/s390/numa/mode_emu.c | 3 +++ arch/s390/numa/numa.c | 6 +++++- arch/sh/mm/init.c | 6 ++++++ arch/sh/mm/numa.c | 4 ++++ arch/um/drivers/net_kern.c | 3 +++ arch/um/drivers/vector_kern.c | 3 +++ arch/um/kernel/initrd.c | 2 ++ arch/um/kernel/mem.c | 16 ++++++++++++++++ arch/unicore32/kernel/setup.c | 4 ++++ arch/unicore32/mm/mmu.c | 15 +++++++++++++-- arch/x86/kernel/acpi/boot.c | 3 +++ arch/x86/kernel/apic/io_apic.c | 5 +++++ arch/x86/kernel/e820.c | 3 +++ arch/x86/platform/olpc/olpc_dt.c | 3 +++ arch/x86/xen/p2m.c | 11 +++++++++-- arch/xtensa/mm/kasan_init.c | 4 ++++ arch/xtensa/mm/mmu.c | 3 +++ drivers/clk/ti/clk.c | 3 +++ drivers/macintosh/smu.c | 3 +++ drivers/of/fdt.c | 8 +++++++- drivers/of/unittest.c | 8 +++++++- drivers/xen/swiotlb-xen.c | 7 +++++-- kernel/dma/swiotlb.c | 4 ++-- kernel/power/snapshot.c | 3 +++ lib/cpumask.c | 3 +++ mm/kasan/init.c | 10 ++++++++-- mm/sparse.c | 21 +++++++++++++++++---- 74 files changed, 411 insertions(+), 32 deletions(-) diff --git a/arch/alpha/kernel/core_cia.c b/arch/alpha/kernel/core_cia.c index 466cd44d8b36..f489170201c3 100644 --- a/arch/alpha/kernel/core_cia.c +++ b/arch/alpha/kernel/core_cia.c @@ -332,6 +332,9 @@ cia_prepare_tbia_workaround(int window) /* Use minimal 1K map. */ ppte = memblock_alloc(CIA_BROKEN_TBIA_SIZE, 32768); + if (!ppte) + panic("%s: Failed to allocate %u bytes align=0x%x\n", + __func__, CIA_BROKEN_TBIA_SIZE, 32768); pte = (virt_to_phys(ppte) >> (PAGE_SHIFT - 1)) | 1; for (i = 0; i < CIA_BROKEN_TBIA_SIZE / sizeof(unsigned long); ++i) diff --git a/arch/alpha/kernel/core_marvel.c b/arch/alpha/kernel/core_marvel.c index c1d0c18c71ca..1db9d0eb2922 100644 --- a/arch/alpha/kernel/core_marvel.c +++ b/arch/alpha/kernel/core_marvel.c @@ -83,6 +83,9 @@ mk_resource_name(int pe, int port, char *str) sprintf(tmp, "PCI %s PE %d PORT %d", str, pe, port); name = memblock_alloc(strlen(tmp) + 1, SMP_CACHE_BYTES); + if (!name) + panic("%s: Failed to allocate %zu bytes\n", __func__, + strlen(tmp) + 1); strcpy(name, tmp); return name; @@ -118,6 +121,9 @@ alloc_io7(unsigned int pe) } io7 = memblock_alloc(sizeof(*io7), SMP_CACHE_BYTES); + if (!io7) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*io7)); io7->pe = pe; raw_spin_lock_init(&io7->irq_lock); diff --git a/arch/alpha/kernel/pci-noop.c b/arch/alpha/kernel/pci-noop.c index 091cff3c68fd..ae82061edae9 100644 --- a/arch/alpha/kernel/pci-noop.c +++ b/arch/alpha/kernel/pci-noop.c @@ -34,6 +34,9 @@ alloc_pci_controller(void) struct pci_controller *hose; hose = memblock_alloc(sizeof(*hose), SMP_CACHE_BYTES); + if (!hose) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*hose)); *hose_tail = hose; hose_tail = &hose->next; @@ -44,7 +47,13 @@ alloc_pci_controller(void) struct resource * __init alloc_resource(void) { - return memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES); + void *ptr = memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES); + + if (!ptr) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(struct resource)); + + return ptr; } SYSCALL_DEFINE3(pciconfig_iobase, long, which, unsigned long, bus, @@ -54,7 +63,7 @@ SYSCALL_DEFINE3(pciconfig_iobase, long, which, unsigned long, bus, /* from hose or from bus.devfn */ if (which & IOBASE_FROM_HOSE) { - for (hose = hose_head; hose; hose = hose->next) + for (hose = hose_head; hose; hose = hose->next) if (hose->index == bus) break; if (!hose) diff --git a/arch/alpha/kernel/pci.c b/arch/alpha/kernel/pci.c index 97098127df83..64fbfb0763b2 100644 --- a/arch/alpha/kernel/pci.c +++ b/arch/alpha/kernel/pci.c @@ -393,6 +393,9 @@ alloc_pci_controller(void) struct pci_controller *hose; hose = memblock_alloc(sizeof(*hose), SMP_CACHE_BYTES); + if (!hose) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*hose)); *hose_tail = hose; hose_tail = &hose->next; @@ -403,7 +406,13 @@ alloc_pci_controller(void) struct resource * __init alloc_resource(void) { - return memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES); + void *ptr = memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES); + + if (!ptr) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(struct resource)); + + return ptr; } diff --git a/arch/alpha/kernel/pci_iommu.c b/arch/alpha/kernel/pci_iommu.c index e4cf77b07742..3034d6d936d2 100644 --- a/arch/alpha/kernel/pci_iommu.c +++ b/arch/alpha/kernel/pci_iommu.c @@ -80,6 +80,9 @@ iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base, " falling back to system-wide allocation\n", __func__, nid); arena = memblock_alloc(sizeof(*arena), SMP_CACHE_BYTES); + if (!arena) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*arena)); } arena->ptes = memblock_alloc_node(sizeof(*arena), align, nid); @@ -88,12 +91,21 @@ iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base, " falling back to system-wide allocation\n", __func__, nid); arena->ptes = memblock_alloc(mem_size, align); + if (!arena->ptes) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, mem_size, align); } #else /* CONFIG_DISCONTIGMEM */ arena = memblock_alloc(sizeof(*arena), SMP_CACHE_BYTES); + if (!arena) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*arena)); arena->ptes = memblock_alloc(mem_size, align); + if (!arena->ptes) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, mem_size, align); #endif /* CONFIG_DISCONTIGMEM */ diff --git a/arch/arc/mm/highmem.c b/arch/arc/mm/highmem.c index 48e700151810..11f57e2ced8a 100644 --- a/arch/arc/mm/highmem.c +++ b/arch/arc/mm/highmem.c @@ -124,6 +124,10 @@ static noinline pte_t * __init alloc_kmap_pgtable(unsigned long kvaddr) pmd_k = pmd_offset(pud_k, kvaddr); pte_k = (pte_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + if (!pte_k) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); + pmd_populate_kernel(&init_mm, pmd_k, pte_k); return pte_k; } diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 375b13f7e780..5d78b6ac0429 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -867,6 +867,9 @@ static void __init request_standard_resources(const struct machine_desc *mdesc) boot_alias_start = phys_to_idmap(start); if (arm_has_idmap_alias() && boot_alias_start != IDMAP_INVALID_ADDR) { res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES); + if (!res) + panic("%s: Failed to allocate %zu bytes\n", + __func__, sizeof(*res)); res->name = "System RAM (boot alias)"; res->start = boot_alias_start; res->end = phys_to_idmap(end); @@ -875,6 +878,9 @@ static void __init request_standard_resources(const struct machine_desc *mdesc) } res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES); + if (!res) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*res)); res->name = "System RAM"; res->start = start; res->end = end; diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 57de0dde3ae0..f3ce34113f89 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -721,7 +721,13 @@ EXPORT_SYMBOL(phys_mem_access_prot); static void __init *early_alloc(unsigned long sz) { - return memblock_alloc(sz, sz); + void *ptr = memblock_alloc(sz, sz); + + if (!ptr) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, sz, sz); + + return ptr; } static void *__init late_alloc(unsigned long sz) @@ -994,6 +1000,9 @@ void __init iotable_init(struct map_desc *io_desc, int nr) return; svm = memblock_alloc(sizeof(*svm) * nr, __alignof__(*svm)); + if (!svm) + panic("%s: Failed to allocate %zu bytes align=0x%zx\n", + __func__, sizeof(*svm) * nr, __alignof__(*svm)); for (md = io_desc; nr; md++, nr--) { create_mapping(md); @@ -1016,6 +1025,9 @@ void __init vm_reserve_area_early(unsigned long addr, unsigned long size, struct static_vm *svm; svm = memblock_alloc(sizeof(*svm), __alignof__(*svm)); + if (!svm) + panic("%s: Failed to allocate %zu bytes align=0x%zx\n", + __func__, sizeof(*svm), __alignof__(*svm)); vm = &svm->vm; vm->addr = (void *)addr; diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 834b321a88f8..f8482fe5a190 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -208,6 +208,7 @@ static void __init request_standard_resources(void) struct memblock_region *region; struct resource *res; unsigned long i = 0; + size_t res_size; kernel_code.start = __pa_symbol(_text); kernel_code.end = __pa_symbol(__init_begin - 1); @@ -215,9 +216,10 @@ static void __init request_standard_resources(void) kernel_data.end = __pa_symbol(_end - 1); num_standard_resources = memblock.memory.cnt; - standard_resources = memblock_alloc_low(num_standard_resources * - sizeof(*standard_resources), - SMP_CACHE_BYTES); + res_size = num_standard_resources * sizeof(*standard_resources); + standard_resources = memblock_alloc_low(res_size, SMP_CACHE_BYTES); + if (!standard_resources) + panic("%s: Failed to allocate %zu bytes\n", __func__, res_size); for_each_memblock(memory, region) { res = &standard_resources[i++]; diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index f37a86d2a69d..296de39ddee5 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -40,6 +40,11 @@ static phys_addr_t __init kasan_alloc_zeroed_page(int node) void *p = memblock_alloc_try_nid(PAGE_SIZE, PAGE_SIZE, __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_KASAN, node); + if (!p) + panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%llx\n", + __func__, PAGE_SIZE, PAGE_SIZE, node, + __pa(MAX_DMA_ADDRESS)); + return __pa(p); } @@ -48,6 +53,11 @@ static phys_addr_t __init kasan_alloc_raw_page(int node) void *p = memblock_alloc_try_nid_raw(PAGE_SIZE, PAGE_SIZE, __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_KASAN, node); + if (!p) + panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%llx\n", + __func__, PAGE_SIZE, PAGE_SIZE, node, + __pa(MAX_DMA_ADDRESS)); + return __pa(p); } diff --git a/arch/c6x/mm/dma-coherent.c b/arch/c6x/mm/dma-coherent.c index 0be289839ce0..0d3701bc88f6 100644 --- a/arch/c6x/mm/dma-coherent.c +++ b/arch/c6x/mm/dma-coherent.c @@ -138,6 +138,10 @@ void __init coherent_mem_init(phys_addr_t start, u32 size) dma_bitmap = memblock_alloc(BITS_TO_LONGS(dma_pages) * sizeof(long), sizeof(long)); + if (!dma_bitmap) + panic("%s: Failed to allocate %zu bytes align=0x%zx\n", + __func__, BITS_TO_LONGS(dma_pages) * sizeof(long), + sizeof(long)); } static void c6x_dma_sync(struct device *dev, phys_addr_t paddr, size_t size, diff --git a/arch/c6x/mm/init.c b/arch/c6x/mm/init.c index e83c04654238..fe582c3a1794 100644 --- a/arch/c6x/mm/init.c +++ b/arch/c6x/mm/init.c @@ -40,6 +40,9 @@ void __init paging_init(void) empty_zero_page = (unsigned long) memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!empty_zero_page) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); /* * Set up user data space diff --git a/arch/csky/mm/highmem.c b/arch/csky/mm/highmem.c index 53b1bfa4c462..3317b774f6dc 100644 --- a/arch/csky/mm/highmem.c +++ b/arch/csky/mm/highmem.c @@ -141,6 +141,11 @@ static void __init fixrange_init(unsigned long start, unsigned long end, for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) { if (pmd_none(*pmd)) { pte = (pte_t *) memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, PAGE_SIZE, + PAGE_SIZE); + set_pmd(pmd, __pmd(__pa(pte))); BUG_ON(pte != pte_offset_kernel(pmd, 0)); } diff --git a/arch/h8300/mm/init.c b/arch/h8300/mm/init.c index a1578904ad4e..0f04a5e9aa4f 100644 --- a/arch/h8300/mm/init.c +++ b/arch/h8300/mm/init.c @@ -68,6 +68,9 @@ void __init paging_init(void) * to a couple of allocated pages. */ empty_zero_page = (unsigned long)memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!empty_zero_page) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); /* * Set up SFC/DFC registers (user data space). diff --git a/arch/m68k/atari/stram.c b/arch/m68k/atari/stram.c index 6ffc204eb07d..6152f9f631d2 100644 --- a/arch/m68k/atari/stram.c +++ b/arch/m68k/atari/stram.c @@ -97,6 +97,10 @@ void __init atari_stram_reserve_pages(void *start_mem) pr_debug("atari_stram pool: kernel in ST-RAM, using alloc_bootmem!\n"); stram_pool.start = (resource_size_t)memblock_alloc_low(pool_size, PAGE_SIZE); + if (!stram_pool.start) + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, pool_size, PAGE_SIZE); + stram_pool.end = stram_pool.start + pool_size - 1; request_resource(&iomem_resource, &stram_pool); stram_virt_offset = 0; diff --git a/arch/m68k/mm/init.c b/arch/m68k/mm/init.c index 933c33e76a48..8868a4c9adae 100644 --- a/arch/m68k/mm/init.c +++ b/arch/m68k/mm/init.c @@ -94,6 +94,9 @@ void __init paging_init(void) high_memory = (void *) end_mem; empty_zero_page = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!empty_zero_page) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); /* * Set up SFC/DFC registers (user data space). diff --git a/arch/m68k/mm/mcfmmu.c b/arch/m68k/mm/mcfmmu.c index 492f953db31b..6cb1e41d58d0 100644 --- a/arch/m68k/mm/mcfmmu.c +++ b/arch/m68k/mm/mcfmmu.c @@ -44,6 +44,9 @@ void __init paging_init(void) int i; empty_zero_page = (void *) memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!empty_zero_page) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); pg_dir = swapper_pg_dir; memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir)); @@ -51,6 +54,9 @@ void __init paging_init(void) size = num_pages * sizeof(pte_t); size = (size + PAGE_SIZE) & ~(PAGE_SIZE-1); next_pgtable = (unsigned long) memblock_alloc(size, PAGE_SIZE); + if (!next_pgtable) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, size, PAGE_SIZE); bootmem_end = (next_pgtable + size + PAGE_SIZE) & PAGE_MASK; pg_dir += PAGE_OFFSET >> PGDIR_SHIFT; diff --git a/arch/m68k/mm/motorola.c b/arch/m68k/mm/motorola.c index 3f3d0bf36091..356601bf96d9 100644 --- a/arch/m68k/mm/motorola.c +++ b/arch/m68k/mm/motorola.c @@ -55,6 +55,9 @@ static pte_t * __init kernel_page_table(void) pte_t *ptablep; ptablep = (pte_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + if (!ptablep) + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); clear_page(ptablep); __flush_page_to_ram(ptablep); @@ -96,6 +99,9 @@ static pmd_t * __init kernel_ptr_table(void) if (((unsigned long)last_pgtable & ~PAGE_MASK) == 0) { last_pgtable = (pmd_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + if (!last_pgtable) + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); clear_page(last_pgtable); __flush_page_to_ram(last_pgtable); @@ -278,6 +284,9 @@ void __init paging_init(void) * to a couple of allocated pages */ empty_zero_page = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!empty_zero_page) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); /* * Set up SFC/DFC registers diff --git a/arch/m68k/mm/sun3mmu.c b/arch/m68k/mm/sun3mmu.c index f736db48a2e1..eca1c46bb90a 100644 --- a/arch/m68k/mm/sun3mmu.c +++ b/arch/m68k/mm/sun3mmu.c @@ -46,6 +46,9 @@ void __init paging_init(void) unsigned long size; empty_zero_page = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!empty_zero_page) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); address = PAGE_OFFSET; pg_dir = swapper_pg_dir; @@ -56,6 +59,9 @@ void __init paging_init(void) size = (size + PAGE_SIZE) & ~(PAGE_SIZE-1); next_pgtable = (unsigned long)memblock_alloc(size, PAGE_SIZE); + if (!next_pgtable) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, size, PAGE_SIZE); bootmem_end = (next_pgtable + size + PAGE_SIZE) & PAGE_MASK; /* Map whole memory from PAGE_OFFSET (0x0E000000) */ diff --git a/arch/m68k/sun3/sun3dvma.c b/arch/m68k/sun3/sun3dvma.c index 4d64711d3d47..399f3d06125f 100644 --- a/arch/m68k/sun3/sun3dvma.c +++ b/arch/m68k/sun3/sun3dvma.c @@ -269,6 +269,9 @@ void __init dvma_init(void) iommu_use = memblock_alloc(IOMMU_TOTAL_ENTRIES * sizeof(unsigned long), SMP_CACHE_BYTES); + if (!iommu_use) + panic("%s: Failed to allocate %zu bytes\n", __func__, + IOMMU_TOTAL_ENTRIES * sizeof(unsigned long)); dvma_unmap_iommu(DVMA_START, DVMA_SIZE); diff --git a/arch/microblaze/mm/init.c b/arch/microblaze/mm/init.c index bd1cd4bff449..7e97d44f6538 100644 --- a/arch/microblaze/mm/init.c +++ b/arch/microblaze/mm/init.c @@ -374,10 +374,14 @@ void * __ref zalloc_maybe_bootmem(size_t size, gfp_t mask) { void *p; - if (mem_init_done) + if (mem_init_done) { p = kzalloc(size, mask); - else + } else { p = memblock_alloc(size, SMP_CACHE_BYTES); + if (!p) + panic("%s: Failed to allocate %zu bytes\n", + __func__, size); + } return p; } diff --git a/arch/mips/cavium-octeon/dma-octeon.c b/arch/mips/cavium-octeon/dma-octeon.c index e8eb60ed99f2..11d5a4e90736 100644 --- a/arch/mips/cavium-octeon/dma-octeon.c +++ b/arch/mips/cavium-octeon/dma-octeon.c @@ -245,6 +245,9 @@ void __init plat_swiotlb_setup(void) swiotlbsize = swiotlb_nslabs << IO_TLB_SHIFT; octeon_swiotlb = memblock_alloc_low(swiotlbsize, PAGE_SIZE); + if (!octeon_swiotlb) + panic("%s: Failed to allocate %zu bytes align=%lx\n", + __func__, swiotlbsize, PAGE_SIZE); if (swiotlb_init_with_tbl(octeon_swiotlb, swiotlb_nslabs, 1) == -ENOMEM) panic("Cannot allocate SWIOTLB buffer"); diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 5151532ad959..8d1dc6c71173 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -919,6 +919,9 @@ static void __init resource_init(void) end = HIGHMEM_START - 1; res = memblock_alloc(sizeof(struct resource), SMP_CACHE_BYTES); + if (!res) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(struct resource)); res->start = start; res->end = end; diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index fc511ecefec6..98ca55d62201 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -2294,6 +2294,9 @@ void __init trap_init(void) ebase = (unsigned long) memblock_alloc(size, 1 << fls(size)); + if (!ebase) + panic("%s: Failed to allocate %lu bytes align=0x%x\n", + __func__, size, 1 << fls(size)); /* * Try to ensure ebase resides in KSeg0 if possible. diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index c3b45e248806..bbb196ad5f26 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -252,6 +252,11 @@ void __init fixrange_init(unsigned long start, unsigned long end, if (pmd_none(*pmd)) { pte = (pte_t *) memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, PAGE_SIZE, + PAGE_SIZE); + set_pmd(pmd, __pmd((unsigned long)pte)); BUG_ON(pte != pte_offset_kernel(pmd, 0)); } diff --git a/arch/nds32/mm/init.c b/arch/nds32/mm/init.c index d1e521cce317..1d03633f89a9 100644 --- a/arch/nds32/mm/init.c +++ b/arch/nds32/mm/init.c @@ -79,6 +79,9 @@ static void __init map_ram(void) /* Alloc one page for holding PTE's... */ pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); set_pmd(pme, __pmd(__pa(pte) + _PAGE_KERNEL_TABLE)); /* Fill the newly allocated page with PTE'S */ @@ -111,6 +114,9 @@ static void __init fixedrange_init(void) pud = pud_offset(pgd, vaddr); pmd = pmd_offset(pud, vaddr); fixmap_pmd_p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!fixmap_pmd_p) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); set_pmd(pmd, __pmd(__pa(fixmap_pmd_p) + _PAGE_KERNEL_TABLE)); #ifdef CONFIG_HIGHMEM @@ -123,6 +129,9 @@ static void __init fixedrange_init(void) pud = pud_offset(pgd, vaddr); pmd = pmd_offset(pud, vaddr); pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); set_pmd(pmd, __pmd(__pa(pte) + _PAGE_KERNEL_TABLE)); pkmap_page_table = pte; #endif /* CONFIG_HIGHMEM */ @@ -148,6 +157,9 @@ void __init paging_init(void) /* allocate space for empty_zero_page */ zero_page = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!zero_page) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); zone_sizes_init(); empty_zero_page = virt_to_page(zero_page); diff --git a/arch/openrisc/mm/ioremap.c b/arch/openrisc/mm/ioremap.c index 051bcb4fefd3..a8509950dbbc 100644 --- a/arch/openrisc/mm/ioremap.c +++ b/arch/openrisc/mm/ioremap.c @@ -122,10 +122,14 @@ pte_t __ref *pte_alloc_one_kernel(struct mm_struct *mm) { pte_t *pte; - if (likely(mem_init_done)) + if (likely(mem_init_done)) { pte = (pte_t *)get_zeroed_page(GFP_KERNEL); - else + } else { pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); + } return pte; } diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c index 28c076c771de..c66fd3ce6478 100644 --- a/arch/powerpc/kernel/dt_cpu_ftrs.c +++ b/arch/powerpc/kernel/dt_cpu_ftrs.c @@ -1005,6 +1005,11 @@ static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, &nr_dt_cpu_features); dt_cpu_features = memblock_alloc(sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE); + if (!dt_cpu_features) + panic("%s: Failed to allocate %zu bytes align=0x%lx\n", + __func__, + sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, + PAGE_SIZE); cpufeatures_setup_start(isa); diff --git a/arch/powerpc/kernel/pci_32.c b/arch/powerpc/kernel/pci_32.c index d3f04f2d8249..0417fda13636 100644 --- a/arch/powerpc/kernel/pci_32.c +++ b/arch/powerpc/kernel/pci_32.c @@ -205,6 +205,9 @@ pci_create_OF_bus_map(void) of_prop = memblock_alloc(sizeof(struct property) + 256, SMP_CACHE_BYTES); + if (!of_prop) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(struct property) + 256); dn = of_find_node_by_path("/"); if (dn) { memset(of_prop, -1, sizeof(struct property) + 256); diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index f17868e19e2c..2e5dfb6e0823 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -461,6 +461,9 @@ void __init smp_setup_cpu_maps(void) cpu_to_phys_id = memblock_alloc(nr_cpu_ids * sizeof(u32), __alignof__(u32)); + if (!cpu_to_phys_id) + panic("%s: Failed to allocate %zu bytes align=0x%zx\n", + __func__, nr_cpu_ids * sizeof(u32), __alignof__(u32)); for_each_node_by_type(dn, "cpu") { const __be32 *intserv; diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index ff0aac42bb33..ba404dd9ce1d 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -905,6 +905,10 @@ static void __ref init_fallback_flush(void) l1d_flush_fallback_area = memblock_alloc_try_nid(l1d_size * 2, l1d_size, MEMBLOCK_LOW_LIMIT, limit, NUMA_NO_NODE); + if (!l1d_flush_fallback_area) + panic("%s: Failed to allocate %llu bytes align=0x%llx max_addr=%pa\n", + __func__, l1d_size * 2, l1d_size, &limit); + for_each_possible_cpu(cpu) { struct paca_struct *paca = paca_ptrs[cpu]; diff --git a/arch/powerpc/lib/alloc.c b/arch/powerpc/lib/alloc.c index dedf88a76f58..ce180870bd52 100644 --- a/arch/powerpc/lib/alloc.c +++ b/arch/powerpc/lib/alloc.c @@ -15,6 +15,9 @@ void * __ref zalloc_maybe_bootmem(size_t size, gfp_t mask) p = kzalloc(size, mask); else { p = memblock_alloc(size, SMP_CACHE_BYTES); + if (!p) + panic("%s: Failed to allocate %zu bytes\n", __func__, + size); } return p; } diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 880a366c229c..0a4f939a8161 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -915,6 +915,9 @@ static void __init htab_initialize(void) linear_map_hash_slots = memblock_alloc_try_nid( linear_map_hash_count, 1, MEMBLOCK_LOW_LIMIT, ppc64_rma_size, NUMA_NO_NODE); + if (!linear_map_hash_slots) + panic("%s: Failed to allocate %lu bytes max_addr=%pa\n", + __func__, linear_map_hash_count, &ppc64_rma_size); } #endif /* CONFIG_DEBUG_PAGEALLOC */ diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c index 22d71a58167f..1945c5f19f5e 100644 --- a/arch/powerpc/mm/mmu_context_nohash.c +++ b/arch/powerpc/mm/mmu_context_nohash.c @@ -461,10 +461,19 @@ void __init mmu_context_init(void) * Allocate the maps used by context management */ context_map = memblock_alloc(CTX_MAP_SIZE, SMP_CACHE_BYTES); + if (!context_map) + panic("%s: Failed to allocate %zu bytes\n", __func__, + CTX_MAP_SIZE); context_mm = memblock_alloc(sizeof(void *) * (LAST_CONTEXT + 1), SMP_CACHE_BYTES); + if (!context_mm) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(void *) * (LAST_CONTEXT + 1)); #ifdef CONFIG_SMP stale_map[boot_cpuid] = memblock_alloc(CTX_MAP_SIZE, SMP_CACHE_BYTES); + if (!stale_map[boot_cpuid]) + panic("%s: Failed to allocate %zu bytes\n", __func__, + CTX_MAP_SIZE); cpuhp_setup_state_nocalls(CPUHP_POWERPC_MMU_CTX_PREPARE, "powerpc/mmu/ctx:prepare", diff --git a/arch/powerpc/mm/pgtable-book3e.c b/arch/powerpc/mm/pgtable-book3e.c index 53cbc7dc2df2..1032ef7aaf62 100644 --- a/arch/powerpc/mm/pgtable-book3e.c +++ b/arch/powerpc/mm/pgtable-book3e.c @@ -57,8 +57,16 @@ void vmemmap_remove_mapping(unsigned long start, static __ref void *early_alloc_pgtable(unsigned long size) { - return memblock_alloc_try_nid(size, size, MEMBLOCK_LOW_LIMIT, - __pa(MAX_DMA_ADDRESS), NUMA_NO_NODE); + void *ptr; + + ptr = memblock_alloc_try_nid(size, size, MEMBLOCK_LOW_LIMIT, + __pa(MAX_DMA_ADDRESS), NUMA_NO_NODE); + + if (!ptr) + panic("%s: Failed to allocate %lu bytes align=0x%lx max_addr=%lx\n", + __func__, size, size, __pa(MAX_DMA_ADDRESS)); + + return ptr; } /* diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c index 92a3e4c39540..a4341aba0af4 100644 --- a/arch/powerpc/mm/pgtable-book3s64.c +++ b/arch/powerpc/mm/pgtable-book3s64.c @@ -197,6 +197,9 @@ void __init mmu_partition_table_init(void) BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 36), "Partition table size too large."); /* Initialize the Partition Table with no entries */ partition_tb = memblock_alloc(patb_size, patb_size); + if (!partition_tb) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, patb_size, patb_size); /* * update partition table control register, diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c index e377684ac6ad..154472a28c77 100644 --- a/arch/powerpc/mm/pgtable-radix.c +++ b/arch/powerpc/mm/pgtable-radix.c @@ -53,13 +53,20 @@ static __ref void *early_alloc_pgtable(unsigned long size, int nid, { phys_addr_t min_addr = MEMBLOCK_LOW_LIMIT; phys_addr_t max_addr = MEMBLOCK_ALLOC_ANYWHERE; + void *ptr; if (region_start) min_addr = region_start; if (region_end) max_addr = region_end; - return memblock_alloc_try_nid(size, size, min_addr, max_addr, nid); + ptr = memblock_alloc_try_nid(size, size, min_addr, max_addr, nid); + + if (!ptr) + panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa max_addr=%pa\n", + __func__, size, size, nid, &min_addr, &max_addr); + + return ptr; } static int early_map_kernel_page(unsigned long ea, unsigned long pa, diff --git a/arch/powerpc/mm/ppc_mmu_32.c b/arch/powerpc/mm/ppc_mmu_32.c index 6c8a60b1e31d..f29d2f118b44 100644 --- a/arch/powerpc/mm/ppc_mmu_32.c +++ b/arch/powerpc/mm/ppc_mmu_32.c @@ -340,6 +340,9 @@ void __init MMU_init_hw(void) */ if ( ppc_md.progress ) ppc_md.progress("hash:find piece", 0x322); Hash = memblock_alloc(Hash_size, Hash_size); + if (!Hash) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, Hash_size, Hash_size); _SDR1 = __pa(Hash) | SDR1_LOW_BITS; Hash_end = (struct hash_pte *) ((unsigned long)Hash + Hash_size); diff --git a/arch/powerpc/platforms/pasemi/iommu.c b/arch/powerpc/platforms/pasemi/iommu.c index 86368e238f6e..044c6089462c 100644 --- a/arch/powerpc/platforms/pasemi/iommu.c +++ b/arch/powerpc/platforms/pasemi/iommu.c @@ -211,6 +211,9 @@ static int __init iob_init(struct device_node *dn) iob_l2_base = memblock_alloc_try_nid_raw(1UL << 21, 1UL << 21, MEMBLOCK_LOW_LIMIT, 0x80000000, NUMA_NO_NODE); + if (!iob_l2_base) + panic("%s: Failed to allocate %lu bytes align=0x%lx max_addr=%x\n", + __func__, 1UL << 21, 1UL << 21, 0x80000000); pr_info("IOBMAP L2 allocated at: %p\n", iob_l2_base); diff --git a/arch/powerpc/platforms/powermac/nvram.c b/arch/powerpc/platforms/powermac/nvram.c index 9360cdc408c1..86989c5779c2 100644 --- a/arch/powerpc/platforms/powermac/nvram.c +++ b/arch/powerpc/platforms/powermac/nvram.c @@ -519,6 +519,9 @@ static int __init core99_nvram_setup(struct device_node *dp, unsigned long addr) return -EINVAL; } nvram_image = memblock_alloc(NVRAM_SIZE, SMP_CACHE_BYTES); + if (!nvram_image) + panic("%s: Failed to allocate %u bytes\n", __func__, + NVRAM_SIZE); nvram_data = ioremap(addr, NVRAM_SIZE*2); nvram_naddrs = 1; /* Make sure we get the correct case */ diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c index 727a7de08635..2b0eca104f86 100644 --- a/arch/powerpc/platforms/powernv/opal.c +++ b/arch/powerpc/platforms/powernv/opal.c @@ -171,6 +171,9 @@ int __init early_init_dt_scan_recoverable_ranges(unsigned long node, * Allocate a buffer to hold the MC recoverable ranges. */ mc_recoverable_range = memblock_alloc(size, __alignof__(u64)); + if (!mc_recoverable_range) + panic("%s: Failed to allocate %u bytes align=0x%lx\n", + __func__, size, __alignof__(u64)); for (i = 0; i < mc_recoverable_range_len; i++) { mc_recoverable_range[i].start_addr = diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index fa6af52b5219..3ead4c237ed0 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -3657,6 +3657,9 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, pr_debug(" PHB-ID : 0x%016llx\n", phb_id); phb = memblock_alloc(sizeof(*phb), SMP_CACHE_BYTES); + if (!phb) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*phb)); /* Allocate PCI controller */ phb->hose = hose = pcibios_alloc_controller(np); @@ -3703,6 +3706,9 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, phb->diag_data_size = PNV_PCI_DIAG_BUF_SIZE; phb->diag_data = memblock_alloc(phb->diag_data_size, SMP_CACHE_BYTES); + if (!phb->diag_data) + panic("%s: Failed to allocate %u bytes\n", __func__, + phb->diag_data_size); /* Parse 32-bit and IO ranges (if any) */ pci_process_bridge_OF_ranges(hose, np, !hose->global_number); @@ -3762,6 +3768,8 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, pemap_off = size; size += phb->ioda.total_pe_num * sizeof(struct pnv_ioda_pe); aux = memblock_alloc(size, SMP_CACHE_BYTES); + if (!aux) + panic("%s: Failed to allocate %lu bytes\n", __func__, size); phb->ioda.pe_alloc = aux; phb->ioda.m64_segmap = aux + m64map_off; phb->ioda.m32_segmap = aux + m32map_off; diff --git a/arch/powerpc/platforms/ps3/setup.c b/arch/powerpc/platforms/ps3/setup.c index 658bfab3350b..4ce5458eb0f8 100644 --- a/arch/powerpc/platforms/ps3/setup.c +++ b/arch/powerpc/platforms/ps3/setup.c @@ -127,6 +127,9 @@ static void __init prealloc(struct ps3_prealloc *p) return; p->address = memblock_alloc(p->size, p->align); + if (!p->address) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, p->size, p->align); printk(KERN_INFO "%s: %lu bytes at %p\n", p->name, p->size, p->address); diff --git a/arch/powerpc/sysdev/msi_bitmap.c b/arch/powerpc/sysdev/msi_bitmap.c index d45450f6666a..51a679a1c403 100644 --- a/arch/powerpc/sysdev/msi_bitmap.c +++ b/arch/powerpc/sysdev/msi_bitmap.c @@ -129,6 +129,9 @@ int __ref msi_bitmap_alloc(struct msi_bitmap *bmp, unsigned int irq_count, bmp->bitmap = kzalloc(size, GFP_KERNEL); else { bmp->bitmap = memblock_alloc(size, SMP_CACHE_BYTES); + if (!bmp->bitmap) + panic("%s: Failed to allocate %u bytes\n", __func__, + size); /* the bitmap won't be freed from memblock allocator */ kmemleak_not_leak(bmp->bitmap); } diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index d7920f3e76c6..2c642af526ce 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -378,6 +378,10 @@ static void __init setup_lowcore_dat_off(void) */ BUILD_BUG_ON(sizeof(struct lowcore) != LC_PAGES * PAGE_SIZE); lc = memblock_alloc_low(sizeof(*lc), sizeof(*lc)); + if (!lc) + panic("%s: Failed to allocate %zu bytes align=%zx\n", + __func__, sizeof(*lc), sizeof(*lc)); + lc->restart_psw.mask = PSW_KERNEL_BITS; lc->restart_psw.addr = (unsigned long) restart_int_handler; lc->external_new_psw.mask = PSW_KERNEL_BITS | PSW_MASK_MCHECK; @@ -419,6 +423,9 @@ static void __init setup_lowcore_dat_off(void) * all CPUs in cast *one* of them does a PSW restart. */ restart_stack = memblock_alloc(THREAD_SIZE, THREAD_SIZE); + if (!restart_stack) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, THREAD_SIZE, THREAD_SIZE); restart_stack += STACK_INIT_OFFSET; /* @@ -495,6 +502,9 @@ static void __init setup_resources(void) for_each_memblock(memory, reg) { res = memblock_alloc(sizeof(*res), 8); + if (!res) + panic("%s: Failed to allocate %zu bytes align=0x%x\n", + __func__, sizeof(*res), 8); res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM; res->name = "System RAM"; @@ -509,6 +519,9 @@ static void __init setup_resources(void) continue; if (std_res->end > res->end) { sub_res = memblock_alloc(sizeof(*sub_res), 8); + if (!sub_res) + panic("%s: Failed to allocate %zu bytes align=0x%x\n", + __func__, sizeof(*sub_res), 8); *sub_res = *std_res; sub_res->end = res->end; std_res->start = res->end + 1; diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 5e3cccc408b8..3fe1c77c361b 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -658,7 +658,7 @@ void __init smp_save_dump_cpus(void) /* Allocate a page as dumping area for the store status sigps */ page = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0, 1UL << 31); if (!page) - panic("ERROR: Failed to allocate %x bytes below %lx\n", + panic("ERROR: Failed to allocate %lx bytes below %lx\n", PAGE_SIZE, 1UL << 31); /* Set multi-threading state to the previous system. */ @@ -770,6 +770,9 @@ void __init smp_detect_cpus(void) /* Get CPU information */ info = memblock_alloc(sizeof(*info), 8); + if (!info) + panic("%s: Failed to allocate %zu bytes align=0x%x\n", + __func__, sizeof(*info), 8); smp_get_core_info(info, 1); /* Find boot CPU type */ if (sclp.has_core_type) { diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index 8992b04c0ade..8964a3f60aad 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -520,6 +520,9 @@ static void __init alloc_masks(struct sysinfo_15_1_x *info, nr_masks = max(nr_masks, 1); for (i = 0; i < nr_masks; i++) { mask->next = memblock_alloc(sizeof(*mask->next), 8); + if (!mask->next) + panic("%s: Failed to allocate %zu bytes align=0x%x\n", + __func__, sizeof(*mask->next), 8); mask = mask->next; } } @@ -538,6 +541,9 @@ void __init topology_init_early(void) if (!MACHINE_HAS_TOPOLOGY) goto out; tl_info = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!tl_info) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); info = tl_info; store_topology(info); pr_info("The CPU configuration topology of the machine is: %d %d %d %d %d %d / %d\n", diff --git a/arch/s390/numa/mode_emu.c b/arch/s390/numa/mode_emu.c index bfba273c32c0..71a12a4f4906 100644 --- a/arch/s390/numa/mode_emu.c +++ b/arch/s390/numa/mode_emu.c @@ -313,6 +313,9 @@ static void __ref create_core_to_node_map(void) int i; emu_cores = memblock_alloc(sizeof(*emu_cores), 8); + if (!emu_cores) + panic("%s: Failed to allocate %zu bytes align=0x%x\n", + __func__, sizeof(*emu_cores), 8); for (i = 0; i < ARRAY_SIZE(emu_cores->to_node_id); i++) emu_cores->to_node_id[i] = NODE_ID_FREE; } diff --git a/arch/s390/numa/numa.c b/arch/s390/numa/numa.c index 2d1271e2a70d..8eb9e9743f5d 100644 --- a/arch/s390/numa/numa.c +++ b/arch/s390/numa/numa.c @@ -92,8 +92,12 @@ static void __init numa_setup_memory(void) } while (cur_base < end_of_dram); /* Allocate and fill out node_data */ - for (nid = 0; nid < MAX_NUMNODES; nid++) + for (nid = 0; nid < MAX_NUMNODES; nid++) { NODE_DATA(nid) = memblock_alloc(sizeof(pg_data_t), 8); + if (!NODE_DATA(nid)) + panic("%s: Failed to allocate %zu bytes align=0x%x\n", + __func__, sizeof(pg_data_t), 8); + } for_each_online_node(nid) { unsigned long start_pfn, end_pfn; diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index a0fa4de03dd5..fceefd92016f 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -128,6 +128,9 @@ static pmd_t * __init one_md_table_init(pud_t *pud) pmd_t *pmd; pmd = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!pmd) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); pud_populate(&init_mm, pud, pmd); BUG_ON(pmd != pmd_offset(pud, 0)); } @@ -141,6 +144,9 @@ static pte_t * __init one_page_table_init(pmd_t *pmd) pte_t *pte; pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); pmd_populate_kernel(&init_mm, pmd, pte); BUG_ON(pte != pte_offset_kernel(pmd, 0)); } diff --git a/arch/sh/mm/numa.c b/arch/sh/mm/numa.c index c4bde6148810..f7e4439deb17 100644 --- a/arch/sh/mm/numa.c +++ b/arch/sh/mm/numa.c @@ -43,6 +43,10 @@ void __init setup_bootmem_node(int nid, unsigned long start, unsigned long end) /* Node-local pgdat */ NODE_DATA(nid) = memblock_alloc_node(sizeof(struct pglist_data), SMP_CACHE_BYTES, nid); + if (!NODE_DATA(nid)) + panic("%s: Failed to allocate %zu bytes align=0x%x nid=%d\n", + __func__, sizeof(struct pglist_data), SMP_CACHE_BYTES, + nid); NODE_DATA(nid)->node_start_pfn = start_pfn; NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn; diff --git a/arch/um/drivers/net_kern.c b/arch/um/drivers/net_kern.c index d80cfb1d9430..6e5be5fb4143 100644 --- a/arch/um/drivers/net_kern.c +++ b/arch/um/drivers/net_kern.c @@ -649,6 +649,9 @@ static int __init eth_setup(char *str) } new = memblock_alloc(sizeof(*new), SMP_CACHE_BYTES); + if (!new) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*new)); INIT_LIST_HEAD(&new->list); new->index = n; diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c index 046fa9ea0ccc..596e7056f376 100644 --- a/arch/um/drivers/vector_kern.c +++ b/arch/um/drivers/vector_kern.c @@ -1576,6 +1576,9 @@ static int __init vector_setup(char *str) return 1; } new = memblock_alloc(sizeof(*new), SMP_CACHE_BYTES); + if (!new) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*new)); INIT_LIST_HEAD(&new->list); new->unit = n; new->arguments = str; diff --git a/arch/um/kernel/initrd.c b/arch/um/kernel/initrd.c index ce169ea87e61..1dcd310cb34d 100644 --- a/arch/um/kernel/initrd.c +++ b/arch/um/kernel/initrd.c @@ -37,6 +37,8 @@ int __init read_initrd(void) } area = memblock_alloc(size, SMP_CACHE_BYTES); + if (!area) + panic("%s: Failed to allocate %llu bytes\n", __func__, size); if (load_initrd(initrd, area, size) == -1) return 0; diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c index 799b571a8f88..99aa11bf53d1 100644 --- a/arch/um/kernel/mem.c +++ b/arch/um/kernel/mem.c @@ -66,6 +66,10 @@ static void __init one_page_table_init(pmd_t *pmd) if (pmd_none(*pmd)) { pte_t *pte = (pte_t *) memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); + set_pmd(pmd, __pmd(_KERNPG_TABLE + (unsigned long) __pa(pte))); if (pte != pte_offset_kernel(pmd, 0)) @@ -77,6 +81,10 @@ static void __init one_md_table_init(pud_t *pud) { #ifdef CONFIG_3_LEVEL_PGTABLES pmd_t *pmd_table = (pmd_t *) memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + if (!pmd_table) + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); + set_pud(pud, __pud(_KERNPG_TABLE + (unsigned long) __pa(pmd_table))); if (pmd_table != pmd_offset(pud, 0)) BUG(); @@ -126,6 +134,10 @@ static void __init fixaddr_user_init( void) fixrange_init( FIXADDR_USER_START, FIXADDR_USER_END, swapper_pg_dir); v = (unsigned long) memblock_alloc_low(size, PAGE_SIZE); + if (!v) + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, size, PAGE_SIZE); + memcpy((void *) v , (void *) FIXADDR_USER_START, size); p = __pa(v); for ( ; size > 0; size -= PAGE_SIZE, vaddr += PAGE_SIZE, @@ -146,6 +158,10 @@ void __init paging_init(void) empty_zero_page = (unsigned long *) memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + if (!empty_zero_page) + panic("%s: Failed to allocate %lu bytes align=%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); + for (i = 0; i < ARRAY_SIZE(zones_size); i++) zones_size[i] = 0; diff --git a/arch/unicore32/kernel/setup.c b/arch/unicore32/kernel/setup.c index 4b0cb68c355a..d3239cf2e837 100644 --- a/arch/unicore32/kernel/setup.c +++ b/arch/unicore32/kernel/setup.c @@ -207,6 +207,10 @@ request_standard_resources(struct meminfo *mi) continue; res = memblock_alloc_low(sizeof(*res), SMP_CACHE_BYTES); + if (!res) + panic("%s: Failed to allocate %zu bytes align=%x\n", + __func__, sizeof(*res), SMP_CACHE_BYTES); + res->name = "System RAM"; res->start = mi->bank[i].start; res->end = mi->bank[i].start + mi->bank[i].size - 1; diff --git a/arch/unicore32/mm/mmu.c b/arch/unicore32/mm/mmu.c index a40219291965..aa2060beb408 100644 --- a/arch/unicore32/mm/mmu.c +++ b/arch/unicore32/mm/mmu.c @@ -145,8 +145,13 @@ static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot) { if (pmd_none(*pmd)) { - pte_t *pte = memblock_alloc(PTRS_PER_PTE * sizeof(pte_t), - PTRS_PER_PTE * sizeof(pte_t)); + size_t size = PTRS_PER_PTE * sizeof(pte_t); + pte_t *pte = memblock_alloc(size, size); + + if (!pte) + panic("%s: Failed to allocate %zu bytes align=%zx\n", + __func__, size, size); + __pmd_populate(pmd, __pa(pte) | prot); } BUG_ON(pmd_bad(*pmd)); @@ -349,6 +354,9 @@ static void __init devicemaps_init(void) * Allocate the vector page early. */ vectors = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!vectors) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); for (addr = VMALLOC_END; addr; addr += PGDIR_SIZE) pmd_clear(pmd_off_k(addr)); @@ -426,6 +434,9 @@ void __init paging_init(void) /* allocate the zero page. */ zero_page = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!zero_page) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); bootmem_init(); diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 2624de16cd7a..8dcbf6890714 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -935,6 +935,9 @@ static int __init acpi_parse_hpet(struct acpi_table_header *table) #define HPET_RESOURCE_NAME_SIZE 9 hpet_res = memblock_alloc(sizeof(*hpet_res) + HPET_RESOURCE_NAME_SIZE, SMP_CACHE_BYTES); + if (!hpet_res) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*hpet_res) + HPET_RESOURCE_NAME_SIZE); hpet_res->name = (void *)&hpet_res[1]; hpet_res->flags = IORESOURCE_MEM; diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 264e3221d923..53aa234a6803 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -2581,6 +2581,8 @@ static struct resource * __init ioapic_setup_resources(void) n *= nr_ioapics; mem = memblock_alloc(n, SMP_CACHE_BYTES); + if (!mem) + panic("%s: Failed to allocate %lu bytes\n", __func__, n); res = (void *)mem; mem += sizeof(struct resource) * nr_ioapics; @@ -2625,6 +2627,9 @@ fake_ioapic_page: #endif ioapic_phys = (unsigned long)memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (!ioapic_phys) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); ioapic_phys = __pa(ioapic_phys); } set_fixmap_nocache(idx, ioapic_phys); diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index 5203ee4e6435..6831c8437951 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c @@ -1092,6 +1092,9 @@ void __init e820__reserve_resources(void) res = memblock_alloc(sizeof(*res) * e820_table->nr_entries, SMP_CACHE_BYTES); + if (!res) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*res) * e820_table->nr_entries); e820_res = res; for (i = 0; i < e820_table->nr_entries; i++) { diff --git a/arch/x86/platform/olpc/olpc_dt.c b/arch/x86/platform/olpc/olpc_dt.c index b4ab779f1d47..ac9e7bf49b66 100644 --- a/arch/x86/platform/olpc/olpc_dt.c +++ b/arch/x86/platform/olpc/olpc_dt.c @@ -141,6 +141,9 @@ void * __init prom_early_alloc(unsigned long size) * wasted bootmem) and hand off chunks of it to callers. */ res = memblock_alloc(chunk_size, SMP_CACHE_BYTES); + if (!res) + panic("%s: Failed to allocate %zu bytes\n", __func__, + chunk_size); BUG_ON(!res); prom_early_allocated += chunk_size; memset(res, 0, chunk_size); diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c index 055e37e43541..95ce9b5be411 100644 --- a/arch/x86/xen/p2m.c +++ b/arch/x86/xen/p2m.c @@ -181,8 +181,15 @@ static void p2m_init_identity(unsigned long *p2m, unsigned long pfn) static void * __ref alloc_p2m_page(void) { - if (unlikely(!slab_is_available())) - return memblock_alloc(PAGE_SIZE, PAGE_SIZE); + if (unlikely(!slab_is_available())) { + void *ptr = memblock_alloc(PAGE_SIZE, PAGE_SIZE); + + if (!ptr) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE); + + return ptr; + } return (void *)__get_free_page(GFP_KERNEL); } diff --git a/arch/xtensa/mm/kasan_init.c b/arch/xtensa/mm/kasan_init.c index 4852848a0c28..af7152560bc3 100644 --- a/arch/xtensa/mm/kasan_init.c +++ b/arch/xtensa/mm/kasan_init.c @@ -45,6 +45,10 @@ static void __init populate(void *start, void *end) pmd_t *pmd = pmd_offset(pgd, vaddr); pte_t *pte = memblock_alloc(n_pages * sizeof(pte_t), PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, n_pages * sizeof(pte_t), PAGE_SIZE); + pr_debug("%s: %p - %p\n", __func__, start, end); for (i = j = 0; i < n_pmds; ++i) { diff --git a/arch/xtensa/mm/mmu.c b/arch/xtensa/mm/mmu.c index a4dcfd39bc5c..2fb7d1172228 100644 --- a/arch/xtensa/mm/mmu.c +++ b/arch/xtensa/mm/mmu.c @@ -32,6 +32,9 @@ static void * __init init_pmd(unsigned long vaddr, unsigned long n_pages) __func__, vaddr, n_pages); pte = memblock_alloc_low(n_pages * sizeof(pte_t), PAGE_SIZE); + if (!pte) + panic("%s: Failed to allocate %zu bytes align=%lx\n", + __func__, n_pages * sizeof(pte_t), PAGE_SIZE); for (i = 0; i < n_pages; ++i) pte_clear(NULL, 0, pte + i); diff --git a/drivers/clk/ti/clk.c b/drivers/clk/ti/clk.c index d0cd58534781..5d7fb2eecce4 100644 --- a/drivers/clk/ti/clk.c +++ b/drivers/clk/ti/clk.c @@ -351,6 +351,9 @@ void __init omap2_clk_legacy_provider_init(int index, void __iomem *mem) struct clk_iomap *io; io = memblock_alloc(sizeof(*io), SMP_CACHE_BYTES); + if (!io) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(*io)); io->mem = mem; diff --git a/drivers/macintosh/smu.c b/drivers/macintosh/smu.c index 42cf68d15da3..6a844125cf2d 100644 --- a/drivers/macintosh/smu.c +++ b/drivers/macintosh/smu.c @@ -493,6 +493,9 @@ int __init smu_init (void) } smu = memblock_alloc(sizeof(struct smu_device), SMP_CACHE_BYTES); + if (!smu) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(struct smu_device)); spin_lock_init(&smu->lock); INIT_LIST_HEAD(&smu->cmd_list); diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index 9cc1461aac7d..4734223ab702 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -1181,7 +1181,13 @@ int __init __weak early_init_dt_reserve_memory_arch(phys_addr_t base, static void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align) { - return memblock_alloc(size, align); + void *ptr = memblock_alloc(size, align); + + if (!ptr) + panic("%s: Failed to allocate %llu bytes align=0x%llx\n", + __func__, size, align); + + return ptr; } bool __init early_init_dt_verify(void *params) diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 66037511f2d7..cccde756b510 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -2241,7 +2241,13 @@ static struct device_node *overlay_base_root; static void * __init dt_alloc_memory(u64 size, u64 align) { - return memblock_alloc(size, align); + void *ptr = memblock_alloc(size, align); + + if (!ptr) + panic("%s: Failed to allocate %llu bytes align=0x%llx\n", + __func__, size, align); + + return ptr; } /* diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index bb7888429be6..877baf2a94f4 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -214,10 +214,13 @@ retry: /* * Get IO TLB memory from any location. */ - if (early) + if (early) { xen_io_tlb_start = memblock_alloc(PAGE_ALIGN(bytes), PAGE_SIZE); - else { + if (!xen_io_tlb_start) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_ALIGN(bytes), PAGE_SIZE); + } else { #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT)) #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT) while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) { diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index dd6a8e2d53a7..56ac77a80b1f 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -215,13 +215,13 @@ int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose) alloc_size = PAGE_ALIGN(io_tlb_nslabs * sizeof(int)); io_tlb_list = memblock_alloc(alloc_size, PAGE_SIZE); if (!io_tlb_list) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + panic("%s: Failed to allocate %zu bytes align=0x%lx\n", __func__, alloc_size, PAGE_SIZE); alloc_size = PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)); io_tlb_orig_addr = memblock_alloc(alloc_size, PAGE_SIZE); if (!io_tlb_orig_addr) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + panic("%s: Failed to allocate %zu bytes align=0x%lx\n", __func__, alloc_size, PAGE_SIZE); for (i = 0; i < io_tlb_nslabs; i++) { diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index 4802b039b89f..f08a1e4ee1d4 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -965,6 +965,9 @@ void __init __register_nosave_region(unsigned long start_pfn, /* This allocation cannot fail */ region = memblock_alloc(sizeof(struct nosave_region), SMP_CACHE_BYTES); + if (!region) + panic("%s: Failed to allocate %zu bytes\n", __func__, + sizeof(struct nosave_region)); } region->start_pfn = start_pfn; region->end_pfn = end_pfn; diff --git a/lib/cpumask.c b/lib/cpumask.c index 087a3e9a0202..0cb672eb107c 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -165,6 +165,9 @@ EXPORT_SYMBOL(zalloc_cpumask_var); void __init alloc_bootmem_cpumask_var(cpumask_var_t *mask) { *mask = memblock_alloc(cpumask_size(), SMP_CACHE_BYTES); + if (!*mask) + panic("%s: Failed to allocate %u bytes\n", __func__, + cpumask_size()); } /** diff --git a/mm/kasan/init.c b/mm/kasan/init.c index fcaa1ca03175..ce45c491ebcd 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -83,8 +83,14 @@ static inline bool kasan_early_shadow_page_entry(pte_t pte) static __init void *early_alloc(size_t size, int node) { - return memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS), - MEMBLOCK_ALLOC_ACCESSIBLE, node); + void *ptr = memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS), + MEMBLOCK_ALLOC_ACCESSIBLE, node); + + if (!ptr) + panic("%s: Failed to allocate %zu bytes align=%zx nid=%d from=%llx\n", + __func__, size, size, node, (u64)__pa(MAX_DMA_ADDRESS)); + + return ptr; } static void __ref zero_pte_populate(pmd_t *pmd, unsigned long addr, diff --git a/mm/sparse.c b/mm/sparse.c index 77a0554fa5bd..7397fb4e78b4 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -65,11 +65,15 @@ static noinline struct mem_section __ref *sparse_index_alloc(int nid) unsigned long array_size = SECTIONS_PER_ROOT * sizeof(struct mem_section); - if (slab_is_available()) + if (slab_is_available()) { section = kzalloc_node(array_size, GFP_KERNEL, nid); - else + } else { section = memblock_alloc_node(array_size, SMP_CACHE_BYTES, nid); + if (!section) + panic("%s: Failed to allocate %lu bytes nid=%d\n", + __func__, array_size, nid); + } return section; } @@ -218,6 +222,9 @@ void __init memory_present(int nid, unsigned long start, unsigned long end) size = sizeof(struct mem_section*) * NR_SECTION_ROOTS; align = 1 << (INTERNODE_CACHE_SHIFT); mem_section = memblock_alloc(size, align); + if (!mem_section) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, size, align); } #endif @@ -404,13 +411,18 @@ struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid, { unsigned long size = section_map_size(); struct page *map = sparse_buffer_alloc(size); + phys_addr_t addr = __pa(MAX_DMA_ADDRESS); if (map) return map; map = memblock_alloc_try_nid(size, - PAGE_SIZE, __pa(MAX_DMA_ADDRESS), + PAGE_SIZE, addr, MEMBLOCK_ALLOC_ACCESSIBLE, nid); + if (!map) + panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n", + __func__, size, PAGE_SIZE, nid, &addr); + return map; } #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ @@ -420,10 +432,11 @@ static void *sparsemap_buf_end __meminitdata; static void __init sparse_buffer_init(unsigned long size, int nid) { + phys_addr_t addr = __pa(MAX_DMA_ADDRESS); WARN_ON(sparsemap_buf); /* forgot to call sparse_buffer_fini()? */ sparsemap_buf = memblock_alloc_try_nid_raw(size, PAGE_SIZE, - __pa(MAX_DMA_ADDRESS), + addr, MEMBLOCK_ALLOC_ACCESSIBLE, nid); sparsemap_buf_end = sparsemap_buf + size; } From c0dbe825a9f11d68725b01b9ed311f7c44ca9166 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:37 -0700 Subject: [PATCH 26/38] memblock: memblock_alloc_try_nid: don't panic As all the memblock_alloc*() users are now checking the return value and panic() in case of error, the panic() call can be removed from the core memblock allocator, namely memblock_alloc_try_nid(). Link: http://lkml.kernel.org/r/1548057848-15136-21-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Kroah-Hartman Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Petr Mladek Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memblock.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/mm/memblock.c b/mm/memblock.c index 618f94a1eedb..a838c50ca9a8 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1469,7 +1469,7 @@ void * __init memblock_alloc_try_nid_nopanic( } /** - * memblock_alloc_try_nid - allocate boot memory block with panicking + * memblock_alloc_try_nid - allocate boot memory block * @size: size of memory block to be allocated in bytes * @align: alignment of the region and block's size * @min_addr: the lower bound of the memory region from where the allocation @@ -1479,9 +1479,8 @@ void * __init memblock_alloc_try_nid_nopanic( * allocate only from memory limited by memblock.current_limit value * @nid: nid of the free area to find, %NUMA_NO_NODE for any node * - * Public panicking version of memblock_alloc_try_nid_nopanic() - * which provides debug information (including caller info), if enabled, - * and panics if the request can not be satisfied. + * Public function, provides additional debug information (including caller + * info), if enabled. This function zeroes the allocated memory. * * Return: * Virtual address of allocated memory block on success, NULL on failure. @@ -1498,14 +1497,10 @@ void * __init memblock_alloc_try_nid( &max_addr, (void *)_RET_IP_); ptr = memblock_alloc_internal(size, align, min_addr, max_addr, nid); - if (ptr) { + if (ptr) memset(ptr, 0, size); - return ptr; - } - panic("%s: Failed to allocate %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa\n", - __func__, (u64)size, (u64)align, nid, &min_addr, &max_addr); - return NULL; + return ptr; } /** From 26fb3dae0a1ec78bdde4b5b72e0e709503e8c596 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:42 -0700 Subject: [PATCH 27/38] memblock: drop memblock_alloc_*_nopanic() variants As all the memblock allocation functions return NULL in case of error rather than panic(), the duplicates with _nopanic suffix can be removed. Link: http://lkml.kernel.org/r/1548057848-15136-22-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Acked-by: Greg Kroah-Hartman Reviewed-by: Petr Mladek [printk] Cc: Catalin Marinas Cc: Christophe Leroy Cc: Christoph Hellwig Cc: "David S. Miller" Cc: Dennis Zhou Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Guan Xuetao Cc: Guo Ren Cc: Guo Ren [c-sky] Cc: Heiko Carstens Cc: Juergen Gross [Xen] Cc: Mark Salter Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Paul Burton Cc: Richard Weinberger Cc: Rich Felker Cc: Rob Herring Cc: Rob Herring Cc: Russell King Cc: Stafford Horne Cc: Tony Luck Cc: Vineet Gupta Cc: Yoshinori Sato Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/arc/kernel/unwind.c | 3 +-- arch/sh/mm/init.c | 2 +- arch/x86/kernel/setup_percpu.c | 10 +++++----- arch/x86/mm/kasan_init_64.c | 14 ++++++++------ drivers/firmware/memmap.c | 2 +- drivers/usb/early/xhci-dbc.c | 2 +- include/linux/memblock.h | 35 ---------------------------------- kernel/dma/swiotlb.c | 2 +- kernel/printk/printk.c | 9 +-------- mm/memblock.c | 35 ---------------------------------- mm/page_alloc.c | 10 +++++----- mm/page_ext.c | 2 +- mm/percpu.c | 11 ++++------- mm/sparse.c | 6 ++---- 14 files changed, 31 insertions(+), 112 deletions(-) diff --git a/arch/arc/kernel/unwind.c b/arch/arc/kernel/unwind.c index d34f69eb1a95..271e9fafa479 100644 --- a/arch/arc/kernel/unwind.c +++ b/arch/arc/kernel/unwind.c @@ -181,8 +181,7 @@ static void init_unwind_hdr(struct unwind_table *table, */ static void *__init unw_hdr_alloc_early(unsigned long sz) { - return memblock_alloc_from_nopanic(sz, sizeof(unsigned int), - MAX_DMA_ADDRESS); + return memblock_alloc_from(sz, sizeof(unsigned int), MAX_DMA_ADDRESS); } static void *unw_hdr_alloc(unsigned long sz) diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index fceefd92016f..70621324db41 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -202,7 +202,7 @@ void __init allocate_pgdat(unsigned int nid) get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); #ifdef CONFIG_NEED_MULTIPLE_NODES - NODE_DATA(nid) = memblock_alloc_try_nid_nopanic( + NODE_DATA(nid) = memblock_alloc_try_nid( sizeof(struct pglist_data), SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, MEMBLOCK_ALLOC_ACCESSIBLE, nid); diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index 13af08827eef..4bf46575568a 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c @@ -106,22 +106,22 @@ static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size, void *ptr; if (!node_online(node) || !NODE_DATA(node)) { - ptr = memblock_alloc_from_nopanic(size, align, goal); + ptr = memblock_alloc_from(size, align, goal); pr_info("cpu %d has no node %d or node-local memory\n", cpu, node); pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", cpu, size, __pa(ptr)); } else { - ptr = memblock_alloc_try_nid_nopanic(size, align, goal, - MEMBLOCK_ALLOC_ACCESSIBLE, - node); + ptr = memblock_alloc_try_nid(size, align, goal, + MEMBLOCK_ALLOC_ACCESSIBLE, + node); pr_debug("per cpu data for cpu%d %lu bytes on node%d at %016lx\n", cpu, size, node, __pa(ptr)); } return ptr; #else - return memblock_alloc_from_nopanic(size, align, goal); + return memblock_alloc_from(size, align, goal); #endif } diff --git a/arch/x86/mm/kasan_init_64.c b/arch/x86/mm/kasan_init_64.c index 462fde83b515..8dc0fc0b1382 100644 --- a/arch/x86/mm/kasan_init_64.c +++ b/arch/x86/mm/kasan_init_64.c @@ -24,14 +24,16 @@ extern struct range pfn_mapped[E820_MAX_ENTRIES]; static p4d_t tmp_p4d_table[MAX_PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE); -static __init void *early_alloc(size_t size, int nid, bool panic) +static __init void *early_alloc(size_t size, int nid, bool should_panic) { - if (panic) - return memblock_alloc_try_nid(size, size, - __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid); - else - return memblock_alloc_try_nid_nopanic(size, size, + void *ptr = memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid); + + if (!ptr && should_panic) + panic("%pS: Failed to allocate page, nid=%d from=%lx\n", + (void *)_RET_IP_, nid, __pa(MAX_DMA_ADDRESS)); + + return ptr; } static void __init kasan_populate_pmd(pmd_t *pmd, unsigned long addr, diff --git a/drivers/firmware/memmap.c b/drivers/firmware/memmap.c index ec4fd253a4e9..d168c87c7d30 100644 --- a/drivers/firmware/memmap.c +++ b/drivers/firmware/memmap.c @@ -333,7 +333,7 @@ int __init firmware_map_add_early(u64 start, u64 end, const char *type) { struct firmware_map_entry *entry; - entry = memblock_alloc_nopanic(sizeof(struct firmware_map_entry), + entry = memblock_alloc(sizeof(struct firmware_map_entry), SMP_CACHE_BYTES); if (WARN_ON(!entry)) return -ENOMEM; diff --git a/drivers/usb/early/xhci-dbc.c b/drivers/usb/early/xhci-dbc.c index d2652dccc699..c9cfb100ecdc 100644 --- a/drivers/usb/early/xhci-dbc.c +++ b/drivers/usb/early/xhci-dbc.c @@ -94,7 +94,7 @@ static void * __init xdbc_get_page(dma_addr_t *dma_addr) { void *virt; - virt = memblock_alloc_nopanic(PAGE_SIZE, PAGE_SIZE); + virt = memblock_alloc(PAGE_SIZE, PAGE_SIZE); if (!virt) return NULL; diff --git a/include/linux/memblock.h b/include/linux/memblock.h index c077227e6d53..db69ad97aa2e 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -335,9 +335,6 @@ static inline phys_addr_t memblock_phys_alloc(phys_addr_t size, void *memblock_alloc_try_nid_raw(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, phys_addr_t max_addr, int nid); -void *memblock_alloc_try_nid_nopanic(phys_addr_t size, phys_addr_t align, - phys_addr_t min_addr, phys_addr_t max_addr, - int nid); void *memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, phys_addr_t max_addr, int nid); @@ -364,36 +361,12 @@ static inline void * __init memblock_alloc_from(phys_addr_t size, MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE); } -static inline void * __init memblock_alloc_nopanic(phys_addr_t size, - phys_addr_t align) -{ - return memblock_alloc_try_nid_nopanic(size, align, MEMBLOCK_LOW_LIMIT, - MEMBLOCK_ALLOC_ACCESSIBLE, - NUMA_NO_NODE); -} - static inline void * __init memblock_alloc_low(phys_addr_t size, phys_addr_t align) { return memblock_alloc_try_nid(size, align, MEMBLOCK_LOW_LIMIT, ARCH_LOW_ADDRESS_LIMIT, NUMA_NO_NODE); } -static inline void * __init memblock_alloc_low_nopanic(phys_addr_t size, - phys_addr_t align) -{ - return memblock_alloc_try_nid_nopanic(size, align, MEMBLOCK_LOW_LIMIT, - ARCH_LOW_ADDRESS_LIMIT, - NUMA_NO_NODE); -} - -static inline void * __init memblock_alloc_from_nopanic(phys_addr_t size, - phys_addr_t align, - phys_addr_t min_addr) -{ - return memblock_alloc_try_nid_nopanic(size, align, min_addr, - MEMBLOCK_ALLOC_ACCESSIBLE, - NUMA_NO_NODE); -} static inline void * __init memblock_alloc_node(phys_addr_t size, phys_addr_t align, int nid) @@ -402,14 +375,6 @@ static inline void * __init memblock_alloc_node(phys_addr_t size, MEMBLOCK_ALLOC_ACCESSIBLE, nid); } -static inline void * __init memblock_alloc_node_nopanic(phys_addr_t size, - int nid) -{ - return memblock_alloc_try_nid_nopanic(size, SMP_CACHE_BYTES, - MEMBLOCK_LOW_LIMIT, - MEMBLOCK_ALLOC_ACCESSIBLE, nid); -} - static inline void __init memblock_free_early(phys_addr_t base, phys_addr_t size) { diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 56ac77a80b1f..53012db1e53c 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -256,7 +256,7 @@ swiotlb_init(int verbose) bytes = io_tlb_nslabs << IO_TLB_SHIFT; /* Get IO TLB memory from the low pages */ - vstart = memblock_alloc_low_nopanic(PAGE_ALIGN(bytes), PAGE_SIZE); + vstart = memblock_alloc_low(PAGE_ALIGN(bytes), PAGE_SIZE); if (vstart && !swiotlb_init_with_tbl(vstart, io_tlb_nslabs, verbose)) return; diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 8eee85bb2687..6b7654b8001f 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -1143,14 +1143,7 @@ void __init setup_log_buf(int early) if (!new_log_buf_len) return; - if (early) { - new_log_buf = - memblock_alloc(new_log_buf_len, LOG_ALIGN); - } else { - new_log_buf = memblock_alloc_nopanic(new_log_buf_len, - LOG_ALIGN); - } - + new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN); if (unlikely(!new_log_buf)) { pr_err("log_buf_len: %lu bytes not available\n", new_log_buf_len); diff --git a/mm/memblock.c b/mm/memblock.c index a838c50ca9a8..0ab30d0185bc 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1433,41 +1433,6 @@ void * __init memblock_alloc_try_nid_raw( return ptr; } -/** - * memblock_alloc_try_nid_nopanic - allocate boot memory block - * @size: size of memory block to be allocated in bytes - * @align: alignment of the region and block's size - * @min_addr: the lower bound of the memory region from where the allocation - * is preferred (phys address) - * @max_addr: the upper bound of the memory region from where the allocation - * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to - * allocate only from memory limited by memblock.current_limit value - * @nid: nid of the free area to find, %NUMA_NO_NODE for any node - * - * Public function, provides additional debug information (including caller - * info), if enabled. This function zeroes the allocated memory. - * - * Return: - * Virtual address of allocated memory block on success, NULL on failure. - */ -void * __init memblock_alloc_try_nid_nopanic( - phys_addr_t size, phys_addr_t align, - phys_addr_t min_addr, phys_addr_t max_addr, - int nid) -{ - void *ptr; - - memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pF\n", - __func__, (u64)size, (u64)align, nid, &min_addr, - &max_addr, (void *)_RET_IP_); - - ptr = memblock_alloc_internal(size, align, - min_addr, max_addr, nid); - if (ptr) - memset(ptr, 0, size); - return ptr; -} - /** * memblock_alloc_try_nid - allocate boot memory block * @size: size of memory block to be allocated in bytes diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3eb01dedfb50..03fcf73d47da 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6445,8 +6445,8 @@ static void __ref setup_usemap(struct pglist_data *pgdat, zone->pageblock_flags = NULL; if (usemapsize) { zone->pageblock_flags = - memblock_alloc_node_nopanic(usemapsize, - pgdat->node_id); + memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, + pgdat->node_id); if (!zone->pageblock_flags) panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", usemapsize, zone->name, pgdat->node_id); @@ -6679,7 +6679,8 @@ static void __ref alloc_node_mem_map(struct pglist_data *pgdat) end = pgdat_end_pfn(pgdat); end = ALIGN(end, MAX_ORDER_NR_PAGES); size = (end - start) * sizeof(struct page); - map = memblock_alloc_node_nopanic(size, pgdat->node_id); + map = memblock_alloc_node(size, SMP_CACHE_BYTES, + pgdat->node_id); if (!map) panic("Failed to allocate %ld bytes for node %d memory map\n", size, pgdat->node_id); @@ -7959,8 +7960,7 @@ void *__init alloc_large_system_hash(const char *tablename, size = bucketsize << log2qty; if (flags & HASH_EARLY) { if (flags & HASH_ZERO) - table = memblock_alloc_nopanic(size, - SMP_CACHE_BYTES); + table = memblock_alloc(size, SMP_CACHE_BYTES); else table = memblock_alloc_raw(size, SMP_CACHE_BYTES); diff --git a/mm/page_ext.c b/mm/page_ext.c index ab4244920e0f..d8f1aca4ad43 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -161,7 +161,7 @@ static int __init alloc_node_page_ext(int nid) table_size = get_entry_size() * nr_pages; - base = memblock_alloc_try_nid_nopanic( + base = memblock_alloc_try_nid( table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid); if (!base) diff --git a/mm/percpu.c b/mm/percpu.c index 3f9fb3086a9b..2e6fc8d552c9 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -1905,7 +1905,7 @@ struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups, __alignof__(ai->groups[0].cpu_map[0])); ai_size = base_size + nr_units * sizeof(ai->groups[0].cpu_map[0]); - ptr = memblock_alloc_nopanic(PFN_ALIGN(ai_size), PAGE_SIZE); + ptr = memblock_alloc(PFN_ALIGN(ai_size), PAGE_SIZE); if (!ptr) return NULL; ai = ptr; @@ -2496,7 +2496,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, size_sum = ai->static_size + ai->reserved_size + ai->dyn_size; areas_size = PFN_ALIGN(ai->nr_groups * sizeof(void *)); - areas = memblock_alloc_nopanic(areas_size, SMP_CACHE_BYTES); + areas = memblock_alloc(areas_size, SMP_CACHE_BYTES); if (!areas) { rc = -ENOMEM; goto out_free; @@ -2729,8 +2729,7 @@ EXPORT_SYMBOL(__per_cpu_offset); static void * __init pcpu_dfl_fc_alloc(unsigned int cpu, size_t size, size_t align) { - return memblock_alloc_from_nopanic( - size, align, __pa(MAX_DMA_ADDRESS)); + return memblock_alloc_from(size, align, __pa(MAX_DMA_ADDRESS)); } static void __init pcpu_dfl_fc_free(void *ptr, size_t size) @@ -2778,9 +2777,7 @@ void __init setup_per_cpu_areas(void) void *fc; ai = pcpu_alloc_alloc_info(1, 1); - fc = memblock_alloc_from_nopanic(unit_size, - PAGE_SIZE, - __pa(MAX_DMA_ADDRESS)); + fc = memblock_alloc_from(unit_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); if (!ai || !fc) panic("Failed to allocate memory for percpu areas."); /* kmemleak tracks the percpu allocations separately */ diff --git a/mm/sparse.c b/mm/sparse.c index 7397fb4e78b4..69904aa6165b 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -330,9 +330,7 @@ sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, limit = goal + (1UL << PA_SECTION_SHIFT); nid = early_pfn_to_nid(goal >> PAGE_SHIFT); again: - p = memblock_alloc_try_nid_nopanic(size, - SMP_CACHE_BYTES, goal, limit, - nid); + p = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, goal, limit, nid); if (!p && limit) { limit = 0; goto again; @@ -386,7 +384,7 @@ static unsigned long * __init sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, unsigned long size) { - return memblock_alloc_node_nopanic(size, pgdat->node_id); + return memblock_alloc_node(size, SMP_CACHE_BYTES, pgdat->node_id); } static void __init check_usemap_section_nr(int nid, unsigned long *usemap) From fe145124dbe53c86bf32b941b2f2f88f891d985d Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:46 -0700 Subject: [PATCH 28/38] memblock: remove memblock_{set,clear}_region_flags The memblock API provides dedicated helpers to set or clear a flag on a memory region, e.g. memblock_{mark,clear}_hotplug(). The memblock_{set,clear}_region_flags() functions are used only by the memblock internal function that adjusts the region flags. Drop these functions and use open-coded implementation instead. Link: http://lkml.kernel.org/r/1549455025-17706-2-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Reviewed-by: Andrew Morton Cc: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/memblock.h | 12 ------------ mm/memblock.c | 9 ++++++--- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/include/linux/memblock.h b/include/linux/memblock.h index db69ad97aa2e..294d5d80e150 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -273,18 +273,6 @@ void __next_mem_pfn_range(int *idx, int nid, unsigned long *out_start_pfn, for_each_mem_range_rev(i, &memblock.memory, &memblock.reserved, \ nid, flags, p_start, p_end, p_nid) -static inline void memblock_set_region_flags(struct memblock_region *r, - enum memblock_flags flags) -{ - r->flags |= flags; -} - -static inline void memblock_clear_region_flags(struct memblock_region *r, - enum memblock_flags flags) -{ - r->flags &= ~flags; -} - #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP int memblock_set_node(phys_addr_t base, phys_addr_t size, struct memblock_type *type, int nid); diff --git a/mm/memblock.c b/mm/memblock.c index 0ab30d0185bc..068e147695ee 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -858,11 +858,14 @@ static int __init_memblock memblock_setclr_flag(phys_addr_t base, if (ret) return ret; - for (i = start_rgn; i < end_rgn; i++) + for (i = start_rgn; i < end_rgn; i++) { + struct memblock_region *r = &type->regions[i]; + if (set) - memblock_set_region_flags(&type->regions[i], flag); + r->flags |= flag; else - memblock_clear_region_flags(&type->regions[i], flag); + r->flags &= ~flag; + } memblock_merge_regions(type); return 0; From c9a688a3e918c4eb4f3916ff99a6dae8995af41b Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:50 -0700 Subject: [PATCH 29/38] memblock: split checks whether a region should be skipped to a helper function __next_mem_range() and __next_mem_range_rev() duplicate the code that checks whether a region should be skipped because of node or flags incompatibility. Split this code into a helper function. Link: http://lkml.kernel.org/r/1549455025-17706-3-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Reviewed-by: Andrew Morton Cc: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memblock.c | 53 ++++++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/mm/memblock.c b/mm/memblock.c index 068e147695ee..dfe90bc210d9 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -965,6 +965,29 @@ void __init_memblock __next_reserved_mem_region(u64 *idx, *idx = ULLONG_MAX; } +static bool should_skip_region(struct memblock_region *m, int nid, int flags) +{ + int m_nid = memblock_get_region_node(m); + + /* only memory regions are associated with nodes, check it */ + if (nid != NUMA_NO_NODE && nid != m_nid) + return true; + + /* skip hotpluggable memory regions if needed */ + if (movable_node_is_enabled() && memblock_is_hotpluggable(m)) + return true; + + /* if we want mirror memory skip non-mirror memory regions */ + if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m)) + return true; + + /* skip nomap memory unless we were asked for it explicitly */ + if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m)) + return true; + + return false; +} + /** * __next__mem_range - next function for for_each_free_mem_range() etc. * @idx: pointer to u64 loop variable @@ -1012,20 +1035,7 @@ void __init_memblock __next_mem_range(u64 *idx, int nid, phys_addr_t m_end = m->base + m->size; int m_nid = memblock_get_region_node(m); - /* only memory regions are associated with nodes, check it */ - if (nid != NUMA_NO_NODE && nid != m_nid) - continue; - - /* skip hotpluggable memory regions if needed */ - if (movable_node_is_enabled() && memblock_is_hotpluggable(m)) - continue; - - /* if we want mirror memory skip non-mirror memory regions */ - if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m)) - continue; - - /* skip nomap memory unless we were asked for it explicitly */ - if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m)) + if (should_skip_region(m, nid, flags)) continue; if (!type_b) { @@ -1129,20 +1139,7 @@ void __init_memblock __next_mem_range_rev(u64 *idx, int nid, phys_addr_t m_end = m->base + m->size; int m_nid = memblock_get_region_node(m); - /* only memory regions are associated with nodes, check it */ - if (nid != NUMA_NO_NODE && nid != m_nid) - continue; - - /* skip hotpluggable memory regions if needed */ - if (movable_node_is_enabled() && memblock_is_hotpluggable(m)) - continue; - - /* if we want mirror memory skip non-mirror memory regions */ - if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m)) - continue; - - /* skip nomap memory unless we were asked for it explicitly */ - if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m)) + if (should_skip_region(m, nid, flags)) continue; if (!type_b) { From a2974133b7e0a31c71fabe86aad42a61db4f01ed Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:54 -0700 Subject: [PATCH 30/38] mm: memblock: update comments and kernel-doc * Remove comments mentioning bootmem * Extend "DOC: memblock overview" * Add kernel-doc comments for several more functions [akpm@linux-foundation.org: fix copy-n-paste error] Link: http://lkml.kernel.org/r/1549626347-25461-1-git-send-email-rppt@linux.ibm.com Signed-off-by: Mike Rapoport Reviewed-by: Andrew Morton Cc: Jonathan Corbet Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memblock.c | 60 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/mm/memblock.c b/mm/memblock.c index dfe90bc210d9..e7665cf914b1 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -76,8 +76,19 @@ * :c:func:`memblock_set_node`. The :c:func:`memblock_add_node` * performs such an assignment directly. * - * Once memblock is setup the memory can be allocated using either - * memblock or bootmem APIs. + * Once memblock is setup the memory can be allocated using one of the + * API variants: + * + * * :c:func:`memblock_phys_alloc*` - these functions return the + * **physical** address of the allocated memory + * * :c:func:`memblock_alloc*` - these functions return the **virtual** + * address of the allocated memory. + * + * Note, that both API variants use implict assumptions about allowed + * memory ranges and the fallback methods. Consult the documentation + * of :c:func:`memblock_alloc_internal` and + * :c:func:`memblock_alloc_range_nid` functions for more elaboarte + * description. * * As the system boot progresses, the architecture specific * :c:func:`mem_init` function frees all the memory to the buddy page @@ -435,17 +446,7 @@ static int __init_memblock memblock_double_array(struct memblock_type *type, else in_slab = &memblock_reserved_in_slab; - /* Try to find some space for it. - * - * WARNING: We assume that either slab_is_available() and we use it or - * we use MEMBLOCK for allocations. That means that this is unsafe to - * use when bootmem is currently active (unless bootmem itself is - * implemented on top of MEMBLOCK which isn't the case yet) - * - * This should however not be an issue for now, as we currently only - * call into MEMBLOCK while it's still active, or much later when slab - * is active for memory hotplug operations - */ + /* Try to find some space for it */ if (use_slab) { new_array = kmalloc(new_size, GFP_KERNEL); addr = new_array ? __pa(new_array) : 0; @@ -989,7 +990,7 @@ static bool should_skip_region(struct memblock_region *m, int nid, int flags) } /** - * __next__mem_range - next function for for_each_free_mem_range() etc. + * __next_mem_range - next function for for_each_free_mem_range() etc. * @idx: pointer to u64 loop variable * @nid: node selector, %NUMA_NO_NODE for all nodes * @flags: pick from blocks based on memory attributes @@ -1335,6 +1336,18 @@ done: return found; } +/** + * memblock_phys_alloc_range - allocate a memory block inside specified range + * @size: size of memory block to be allocated in bytes + * @align: alignment of the region and block's size + * @start: the lower bound of the memory region to allocate (physical address) + * @end: the upper bound of the memory region to allocate (physical address) + * + * Allocate @size bytes in the between @start and @end. + * + * Return: physical address of the allocated memory block on success, + * %0 on failure. + */ phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, phys_addr_t align, phys_addr_t start, @@ -1343,6 +1356,19 @@ phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE); } +/** + * memblock_phys_alloc_try_nid - allocate a memory block from specified MUMA node + * @size: size of memory block to be allocated in bytes + * @align: alignment of the region and block's size + * @nid: nid of the free area to find, %NUMA_NO_NODE for any node + * + * Allocates memory block from the specified NUMA node. If the node + * has no available memory, attempts to allocated from any node in the + * system. + * + * Return: physical address of the allocated memory block on success, + * %0 on failure. + */ phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) { return memblock_alloc_range_nid(size, align, 0, @@ -1469,13 +1495,13 @@ void * __init memblock_alloc_try_nid( } /** - * __memblock_free_late - free bootmem block pages directly to buddy allocator + * __memblock_free_late - free pages directly to buddy allocator * @base: phys starting address of the boot memory block * @size: size of the boot memory block in bytes * - * This is only useful when the bootmem allocator has already been torn + * This is only useful when the memblock allocator has already been torn * down, but we are still initializing the system. Pages are released directly - * to the buddy allocator, no bootmem metadata is updated because it is gone. + * to the buddy allocator. */ void __init __memblock_free_late(phys_addr_t base, phys_addr_t size) { From 5c01a25a210366362a40dc63f550e72688a60c48 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 11 Mar 2019 23:30:58 -0700 Subject: [PATCH 31/38] of: fix kmemleak crash caused by imbalance in early memory reservation Marc Gonzalez reported the following kmemleak crash: Unable to handle kernel paging request at virtual address ffffffc021e00000 Mem abort info: ESR = 0x96000006 Exception class = DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x00000006 CM = 0, WnR = 0 swapper pgtable: 4k pages, 39-bit VAs, pgdp = (____ptrval____) [ffffffc021e00000] pgd=000000017e3ba803, pud=000000017e3ba803, pmd=0000000000000000 Internal error: Oops: 96000006 [#1] PREEMPT SMP Modules linked in: CPU: 6 PID: 523 Comm: kmemleak Tainted: G S W 5.0.0-rc1 #13 Hardware name: Qualcomm Technologies, Inc. MSM8998 v1 MTP (DT) pstate: 80000085 (Nzcv daIf -PAN -UAO) pc : scan_block+0x70/0x190 lr : scan_block+0x6c/0x190 Process kmemleak (pid: 523, stack limit = 0x(____ptrval____)) Call trace: scan_block+0x70/0x190 scan_gray_list+0x108/0x1c0 kmemleak_scan+0x33c/0x7c0 kmemleak_scan_thread+0x98/0xf0 kthread+0x11c/0x120 ret_from_fork+0x10/0x1c Code: f9000fb4 d503201f 97ffffd2 35000580 (f9400260) The crash happens when a no-map area is allocated in early_init_dt_alloc_reserved_memory_arch(). The allocated region is registered with kmemleak, but it is then removed from memblock using memblock_remove() that is not kmemleak-aware. Replacing memblock_phys_alloc_range() with memblock_find_in_range() makes sure that the allocated memory is not added to kmemleak and then memblock_remove()'ing this memory is safe. As a bonus, since memblock_find_in_range() ensures the allocation in the specified range, the bounds check can be removed. [rppt@linux.ibm.com: of: fix parameters order for call to memblock_find_in_range()] Link: http://lkml.kernel.org/r/20190221112619.GC32004@rapoport-lnx Link: http://lkml.kernel.org/r/20190213181921.GB15270@rapoport-lnx Fixes: 3f0c820664483 ("drivers: of: add initialization code for dynamic reserved memory") Signed-off-by: Mike Rapoport Acked-by: Marek Szyprowski Acked-by: Prateek Patel Tested-by: Marc Gonzalez Cc: Rob Herring Cc: Frank Rowand Cc: Catalin Marinas Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/of/of_reserved_mem.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c index 8c07d7da5256..6a36bc0b3d64 100644 --- a/drivers/of/of_reserved_mem.c +++ b/drivers/of/of_reserved_mem.c @@ -34,22 +34,15 @@ static int __init early_init_dt_alloc_reserved_memory_arch(phys_addr_t size, end = !end ? MEMBLOCK_ALLOC_ANYWHERE : end; align = !align ? SMP_CACHE_BYTES : align; - base = memblock_phys_alloc_range(size, align, 0, end); + base = memblock_find_in_range(start, end, size, align); if (!base) return -ENOMEM; - /* - * Check if the allocated region fits in to start..end window - */ - if (base < start) { - memblock_free(base, size); - return -ENOMEM; - } - *res_base = base; if (nomap) return memblock_remove(base, size); - return 0; + + return memblock_reserve(base, size); } /** From ee9c5e67557f9663b27946ba1d3813fb1924b1fe Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 11 Mar 2019 23:31:02 -0700 Subject: [PATCH 32/38] openvswitch: convert to kvmalloc Patch series "generic radix trees; drop flex arrays". This patch (of 7): There was no real need for this code to be using flexarrays, it's just implementing a hash table - ideally it would be using rhashtables, but that conversion would be significantly more complicated. Link: http://lkml.kernel.org/r/20181217131929.11727-2-kent.overstreet@gmail.com Signed-off-by: Kent Overstreet Reviewed-by: Matthew Wilcox Cc: Pravin B Shelar Cc: Alexey Dobriyan Cc: Al Viro Cc: Dave Hansen Cc: Eric Paris Cc: Marcelo Ricardo Leitner Cc: Neil Horman Cc: Paul Moore Cc: Shaohua Li Cc: Stephen Smalley Cc: Vlad Yasevich Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- net/openvswitch/flow.h | 1 - net/openvswitch/flow_netlink.h | 1 - net/openvswitch/flow_table.c | 51 ++++++++-------------------------- net/openvswitch/flow_table.h | 3 +- 4 files changed, 13 insertions(+), 43 deletions(-) diff --git a/net/openvswitch/flow.h b/net/openvswitch/flow.h index ba01fc4270bd..5b8e5bd7457b 100644 --- a/net/openvswitch/flow.h +++ b/net/openvswitch/flow.h @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include diff --git a/net/openvswitch/flow_netlink.h b/net/openvswitch/flow_netlink.h index 6657606b2b47..66f9553758a5 100644 --- a/net/openvswitch/flow_netlink.h +++ b/net/openvswitch/flow_netlink.h @@ -30,7 +30,6 @@ #include #include #include -#include #include #include diff --git a/net/openvswitch/flow_table.c b/net/openvswitch/flow_table.c index 80ea2a71852e..cfb0098c9a01 100644 --- a/net/openvswitch/flow_table.c +++ b/net/openvswitch/flow_table.c @@ -111,29 +111,6 @@ int ovs_flow_tbl_count(const struct flow_table *table) return table->count; } -static struct flex_array *alloc_buckets(unsigned int n_buckets) -{ - struct flex_array *buckets; - int i, err; - - buckets = flex_array_alloc(sizeof(struct hlist_head), - n_buckets, GFP_KERNEL); - if (!buckets) - return NULL; - - err = flex_array_prealloc(buckets, 0, n_buckets, GFP_KERNEL); - if (err) { - flex_array_free(buckets); - return NULL; - } - - for (i = 0; i < n_buckets; i++) - INIT_HLIST_HEAD((struct hlist_head *) - flex_array_get(buckets, i)); - - return buckets; -} - static void flow_free(struct sw_flow *flow) { int cpu; @@ -168,31 +145,30 @@ void ovs_flow_free(struct sw_flow *flow, bool deferred) flow_free(flow); } -static void free_buckets(struct flex_array *buckets) -{ - flex_array_free(buckets); -} - - static void __table_instance_destroy(struct table_instance *ti) { - free_buckets(ti->buckets); + kvfree(ti->buckets); kfree(ti); } static struct table_instance *table_instance_alloc(int new_size) { struct table_instance *ti = kmalloc(sizeof(*ti), GFP_KERNEL); + int i; if (!ti) return NULL; - ti->buckets = alloc_buckets(new_size); - + ti->buckets = kvmalloc_array(new_size, sizeof(struct hlist_head), + GFP_KERNEL); if (!ti->buckets) { kfree(ti); return NULL; } + + for (i = 0; i < new_size; i++) + INIT_HLIST_HEAD(&ti->buckets[i]); + ti->n_buckets = new_size; ti->node_ver = 0; ti->keep_flows = false; @@ -249,7 +225,7 @@ static void table_instance_destroy(struct table_instance *ti, for (i = 0; i < ti->n_buckets; i++) { struct sw_flow *flow; - struct hlist_head *head = flex_array_get(ti->buckets, i); + struct hlist_head *head = &ti->buckets[i]; struct hlist_node *n; int ver = ti->node_ver; int ufid_ver = ufid_ti->node_ver; @@ -294,7 +270,7 @@ struct sw_flow *ovs_flow_tbl_dump_next(struct table_instance *ti, ver = ti->node_ver; while (*bucket < ti->n_buckets) { i = 0; - head = flex_array_get(ti->buckets, *bucket); + head = &ti->buckets[*bucket]; hlist_for_each_entry_rcu(flow, head, flow_table.node[ver]) { if (i < *last) { i++; @@ -313,8 +289,7 @@ struct sw_flow *ovs_flow_tbl_dump_next(struct table_instance *ti, static struct hlist_head *find_bucket(struct table_instance *ti, u32 hash) { hash = jhash_1word(hash, ti->hash_seed); - return flex_array_get(ti->buckets, - (hash & (ti->n_buckets - 1))); + return &ti->buckets[hash & (ti->n_buckets - 1)]; } static void table_instance_insert(struct table_instance *ti, @@ -347,9 +322,7 @@ static void flow_table_copy_flows(struct table_instance *old, /* Insert in new table. */ for (i = 0; i < old->n_buckets; i++) { struct sw_flow *flow; - struct hlist_head *head; - - head = flex_array_get(old->buckets, i); + struct hlist_head *head = &old->buckets[i]; if (ufid) hlist_for_each_entry(flow, head, diff --git a/net/openvswitch/flow_table.h b/net/openvswitch/flow_table.h index 2dd9900f533d..de5ec6cf5174 100644 --- a/net/openvswitch/flow_table.h +++ b/net/openvswitch/flow_table.h @@ -29,7 +29,6 @@ #include #include #include -#include #include #include @@ -37,7 +36,7 @@ #include "flow.h" struct table_instance { - struct flex_array *buckets; + struct hlist_head *buckets; unsigned int n_buckets; struct rcu_head rcu; int node_ver; From b330e6a49dc3e9145de5c986b29bbbb884351e92 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 11 Mar 2019 23:31:06 -0700 Subject: [PATCH 33/38] md: convert to kvmalloc The code really just wants a big flat buffer, so just do that. Link: http://lkml.kernel.org/r/20181217131929.11727-3-kent.overstreet@gmail.com Signed-off-by: Kent Overstreet Reviewed-by: Matthew Wilcox Cc: Shaohua Li Cc: Alexey Dobriyan Cc: Al Viro Cc: Dave Hansen Cc: Eric Paris Cc: Marcelo Ricardo Leitner Cc: Neil Horman Cc: Paul Moore Cc: Pravin B Shelar Cc: Stephen Smalley Cc: Vlad Yasevich Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/md/raid5-ppl.c | 6 +-- drivers/md/raid5.c | 85 +++++++++++++++++++----------------------- drivers/md/raid5.h | 9 +++-- 3 files changed, 45 insertions(+), 55 deletions(-) diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c index 3a7c36326589..0b096ddc9c1e 100644 --- a/drivers/md/raid5-ppl.c +++ b/drivers/md/raid5-ppl.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include "md.h" @@ -165,7 +164,7 @@ ops_run_partial_parity(struct stripe_head *sh, struct raid5_percpu *percpu, struct dma_async_tx_descriptor *tx) { int disks = sh->disks; - struct page **srcs = flex_array_get(percpu->scribble, 0); + struct page **srcs = percpu->scribble; int count = 0, pd_idx = sh->pd_idx, i; struct async_submit_ctl submit; @@ -196,8 +195,7 @@ ops_run_partial_parity(struct stripe_head *sh, struct raid5_percpu *percpu, } init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, tx, - NULL, sh, flex_array_get(percpu->scribble, 0) - + sizeof(struct page *) * (sh->disks + 2)); + NULL, sh, (void *) (srcs + sh->disks + 2)); if (count == 1) tx = async_memcpy(sh->ppl_page, srcs[0], 0, 0, PAGE_SIZE, diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index cecea901ab8c..77ffd09be486 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -54,7 +54,6 @@ #include #include #include -#include #include #include @@ -1394,22 +1393,16 @@ static void ops_complete_compute(void *stripe_head_ref) } /* return a pointer to the address conversion region of the scribble buffer */ -static addr_conv_t *to_addr_conv(struct stripe_head *sh, - struct raid5_percpu *percpu, int i) +static struct page **to_addr_page(struct raid5_percpu *percpu, int i) { - void *addr; - - addr = flex_array_get(percpu->scribble, i); - return addr + sizeof(struct page *) * (sh->disks + 2); + return percpu->scribble + i * percpu->scribble_obj_size; } /* return a pointer to the address conversion region of the scribble buffer */ -static struct page **to_addr_page(struct raid5_percpu *percpu, int i) +static addr_conv_t *to_addr_conv(struct stripe_head *sh, + struct raid5_percpu *percpu, int i) { - void *addr; - - addr = flex_array_get(percpu->scribble, i); - return addr; + return (void *) (to_addr_page(percpu, i) + sh->disks + 2); } static struct dma_async_tx_descriptor * @@ -2238,21 +2231,23 @@ static int grow_stripes(struct r5conf *conf, int num) * calculate over all devices (not just the data blocks), using zeros in place * of the P and Q blocks. */ -static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags) +static int scribble_alloc(struct raid5_percpu *percpu, + int num, int cnt, gfp_t flags) { - struct flex_array *ret; - size_t len; + size_t obj_size = + sizeof(struct page *) * (num+2) + + sizeof(addr_conv_t) * (num+2); + void *scribble; - len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); - ret = flex_array_alloc(len, cnt, flags); - if (!ret) - return NULL; - /* always prealloc all elements, so no locking is required */ - if (flex_array_prealloc(ret, 0, cnt, flags)) { - flex_array_free(ret); - return NULL; - } - return ret; + scribble = kvmalloc_array(cnt, obj_size, flags); + if (!scribble) + return -ENOMEM; + + kvfree(percpu->scribble); + + percpu->scribble = scribble; + percpu->scribble_obj_size = obj_size; + return 0; } static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) @@ -2270,23 +2265,18 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) return 0; mddev_suspend(conf->mddev); get_online_cpus(); + for_each_present_cpu(cpu) { struct raid5_percpu *percpu; - struct flex_array *scribble; percpu = per_cpu_ptr(conf->percpu, cpu); - scribble = scribble_alloc(new_disks, - new_sectors / STRIPE_SECTORS, - GFP_NOIO); - - if (scribble) { - flex_array_free(percpu->scribble); - percpu->scribble = scribble; - } else { - err = -ENOMEM; + err = scribble_alloc(percpu, new_disks, + new_sectors / STRIPE_SECTORS, + GFP_NOIO); + if (err) break; - } } + put_online_cpus(); mddev_resume(conf->mddev); if (!err) { @@ -6742,25 +6732,26 @@ raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks) static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) { safe_put_page(percpu->spare_page); - if (percpu->scribble) - flex_array_free(percpu->scribble); percpu->spare_page = NULL; + kvfree(percpu->scribble); percpu->scribble = NULL; } static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) { - if (conf->level == 6 && !percpu->spare_page) + if (conf->level == 6 && !percpu->spare_page) { percpu->spare_page = alloc_page(GFP_KERNEL); - if (!percpu->scribble) - percpu->scribble = scribble_alloc(max(conf->raid_disks, - conf->previous_raid_disks), - max(conf->chunk_sectors, - conf->prev_chunk_sectors) - / STRIPE_SECTORS, - GFP_KERNEL); + if (!percpu->spare_page) + return -ENOMEM; + } - if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) { + if (scribble_alloc(percpu, + max(conf->raid_disks, + conf->previous_raid_disks), + max(conf->chunk_sectors, + conf->prev_chunk_sectors) + / STRIPE_SECTORS, + GFP_KERNEL)) { free_scratch_buffer(conf, percpu); return -ENOMEM; } diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h index 8474c224127b..cf991f13403e 100644 --- a/drivers/md/raid5.h +++ b/drivers/md/raid5.h @@ -638,10 +638,11 @@ struct r5conf { /* per cpu variables */ struct raid5_percpu { struct page *spare_page; /* Used when checking P/Q in raid6 */ - struct flex_array *scribble; /* space for constructing buffer - * lists and performing address - * conversions - */ + void *scribble; /* space for constructing buffer + * lists and performing address + * conversions + */ + int scribble_obj_size; } __percpu *percpu; int scribble_disks; int scribble_sectors; From acdf52d97f824019888422842757013b37441dd1 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 11 Mar 2019 23:31:10 -0700 Subject: [PATCH 34/38] selinux: convert to kvmalloc The flex arrays were being used for constant sized arrays, so there's no benefit to using flex_arrays over something simpler. Link: http://lkml.kernel.org/r/20181217131929.11727-4-kent.overstreet@gmail.com Signed-off-by: Kent Overstreet Cc: Paul Moore Cc: Stephen Smalley Cc: Eric Paris Cc: Alexey Dobriyan Cc: Al Viro Cc: Dave Hansen Cc: Marcelo Ricardo Leitner Cc: Matthew Wilcox Cc: Neil Horman Cc: Pravin B Shelar Cc: Shaohua Li Cc: Vlad Yasevich Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- security/selinux/ss/avtab.c | 40 +++++----- security/selinux/ss/avtab.h | 4 +- security/selinux/ss/conditional.c | 6 +- security/selinux/ss/policydb.c | 122 ++++++++---------------------- security/selinux/ss/policydb.h | 12 +-- security/selinux/ss/services.c | 22 ++---- 6 files changed, 62 insertions(+), 144 deletions(-) diff --git a/security/selinux/ss/avtab.c b/security/selinux/ss/avtab.c index c0417cf17fee..8c5800750fa8 100644 --- a/security/selinux/ss/avtab.c +++ b/security/selinux/ss/avtab.c @@ -93,12 +93,10 @@ avtab_insert_node(struct avtab *h, int hvalue, newnode->next = prev->next; prev->next = newnode; } else { - newnode->next = flex_array_get_ptr(h->htable, hvalue); - if (flex_array_put_ptr(h->htable, hvalue, newnode, - GFP_KERNEL|__GFP_ZERO)) { - kmem_cache_free(avtab_node_cachep, newnode); - return NULL; - } + struct avtab_node **n = &h->htable[hvalue]; + + newnode->next = *n; + *n = newnode; } h->nel++; @@ -111,11 +109,11 @@ static int avtab_insert(struct avtab *h, struct avtab_key *key, struct avtab_dat struct avtab_node *prev, *cur, *newnode; u16 specified = key->specified & ~(AVTAB_ENABLED|AVTAB_ENABLED_OLD); - if (!h || !h->htable) + if (!h) return -EINVAL; hvalue = avtab_hash(key, h->mask); - for (prev = NULL, cur = flex_array_get_ptr(h->htable, hvalue); + for (prev = NULL, cur = h->htable[hvalue]; cur; prev = cur, cur = cur->next) { if (key->source_type == cur->key.source_type && @@ -156,10 +154,10 @@ avtab_insert_nonunique(struct avtab *h, struct avtab_key *key, struct avtab_datu struct avtab_node *prev, *cur; u16 specified = key->specified & ~(AVTAB_ENABLED|AVTAB_ENABLED_OLD); - if (!h || !h->htable) + if (!h) return NULL; hvalue = avtab_hash(key, h->mask); - for (prev = NULL, cur = flex_array_get_ptr(h->htable, hvalue); + for (prev = NULL, cur = h->htable[hvalue]; cur; prev = cur, cur = cur->next) { if (key->source_type == cur->key.source_type && @@ -186,11 +184,11 @@ struct avtab_datum *avtab_search(struct avtab *h, struct avtab_key *key) struct avtab_node *cur; u16 specified = key->specified & ~(AVTAB_ENABLED|AVTAB_ENABLED_OLD); - if (!h || !h->htable) + if (!h) return NULL; hvalue = avtab_hash(key, h->mask); - for (cur = flex_array_get_ptr(h->htable, hvalue); cur; + for (cur = h->htable[hvalue]; cur; cur = cur->next) { if (key->source_type == cur->key.source_type && key->target_type == cur->key.target_type && @@ -222,11 +220,11 @@ avtab_search_node(struct avtab *h, struct avtab_key *key) struct avtab_node *cur; u16 specified = key->specified & ~(AVTAB_ENABLED|AVTAB_ENABLED_OLD); - if (!h || !h->htable) + if (!h) return NULL; hvalue = avtab_hash(key, h->mask); - for (cur = flex_array_get_ptr(h->htable, hvalue); cur; + for (cur = h->htable[hvalue]; cur; cur = cur->next) { if (key->source_type == cur->key.source_type && key->target_type == cur->key.target_type && @@ -281,11 +279,11 @@ void avtab_destroy(struct avtab *h) int i; struct avtab_node *cur, *temp; - if (!h || !h->htable) + if (!h) return; for (i = 0; i < h->nslot; i++) { - cur = flex_array_get_ptr(h->htable, i); + cur = h->htable[i]; while (cur) { temp = cur; cur = cur->next; @@ -295,7 +293,7 @@ void avtab_destroy(struct avtab *h) kmem_cache_free(avtab_node_cachep, temp); } } - flex_array_free(h->htable); + kvfree(h->htable); h->htable = NULL; h->nslot = 0; h->mask = 0; @@ -303,6 +301,7 @@ void avtab_destroy(struct avtab *h) int avtab_init(struct avtab *h) { + kvfree(h->htable); h->htable = NULL; h->nel = 0; return 0; @@ -329,8 +328,7 @@ int avtab_alloc(struct avtab *h, u32 nrules) nslot = MAX_AVTAB_HASH_BUCKETS; mask = nslot - 1; - h->htable = flex_array_alloc(sizeof(struct avtab_node *), nslot, - GFP_KERNEL | __GFP_ZERO); + h->htable = kvcalloc(nslot, sizeof(void *), GFP_KERNEL); if (!h->htable) return -ENOMEM; @@ -353,7 +351,7 @@ void avtab_hash_eval(struct avtab *h, char *tag) max_chain_len = 0; chain2_len_sum = 0; for (i = 0; i < h->nslot; i++) { - cur = flex_array_get_ptr(h->htable, i); + cur = h->htable[i]; if (cur) { slots_used++; chain_len = 0; @@ -646,7 +644,7 @@ int avtab_write(struct policydb *p, struct avtab *a, void *fp) return rc; for (i = 0; i < a->nslot; i++) { - for (cur = flex_array_get_ptr(a->htable, i); cur; + for (cur = a->htable[i]; cur; cur = cur->next) { rc = avtab_write_item(p, cur, fp); if (rc) diff --git a/security/selinux/ss/avtab.h b/security/selinux/ss/avtab.h index 0d652fad5319..de16673b2314 100644 --- a/security/selinux/ss/avtab.h +++ b/security/selinux/ss/avtab.h @@ -24,7 +24,6 @@ #define _SS_AVTAB_H_ #include "security.h" -#include struct avtab_key { u16 source_type; /* source type */ @@ -84,11 +83,10 @@ struct avtab_node { }; struct avtab { - struct flex_array *htable; + struct avtab_node **htable; u32 nel; /* number of elements */ u32 nslot; /* number of hash slots */ u32 mask; /* mask to compute hash func */ - }; int avtab_init(struct avtab *); diff --git a/security/selinux/ss/conditional.c b/security/selinux/ss/conditional.c index f49e522e932d..3bbb60345209 100644 --- a/security/selinux/ss/conditional.c +++ b/security/selinux/ss/conditional.c @@ -195,7 +195,6 @@ int cond_index_bool(void *key, void *datum, void *datap) { struct policydb *p; struct cond_bool_datum *booldatum; - struct flex_array *fa; booldatum = datum; p = datap; @@ -203,10 +202,7 @@ int cond_index_bool(void *key, void *datum, void *datap) if (!booldatum->value || booldatum->value > p->p_bools.nprim) return -EINVAL; - fa = p->sym_val_to_name[SYM_BOOLS]; - if (flex_array_put_ptr(fa, booldatum->value - 1, key, - GFP_KERNEL | __GFP_ZERO)) - BUG(); + p->sym_val_to_name[SYM_BOOLS][booldatum->value - 1] = key; p->bool_val_to_struct[booldatum->value - 1] = booldatum; return 0; diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index c1c31e33657a..6b576e588725 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -36,7 +36,6 @@ #include #include #include -#include #include "security.h" #include "policydb.h" @@ -341,17 +340,14 @@ static int common_index(void *key, void *datum, void *datap) { struct policydb *p; struct common_datum *comdatum; - struct flex_array *fa; comdatum = datum; p = datap; if (!comdatum->value || comdatum->value > p->p_commons.nprim) return -EINVAL; - fa = p->sym_val_to_name[SYM_COMMONS]; - if (flex_array_put_ptr(fa, comdatum->value - 1, key, - GFP_KERNEL | __GFP_ZERO)) - BUG(); + p->sym_val_to_name[SYM_COMMONS][comdatum->value - 1] = key; + return 0; } @@ -359,16 +355,13 @@ static int class_index(void *key, void *datum, void *datap) { struct policydb *p; struct class_datum *cladatum; - struct flex_array *fa; cladatum = datum; p = datap; if (!cladatum->value || cladatum->value > p->p_classes.nprim) return -EINVAL; - fa = p->sym_val_to_name[SYM_CLASSES]; - if (flex_array_put_ptr(fa, cladatum->value - 1, key, - GFP_KERNEL | __GFP_ZERO)) - BUG(); + + p->sym_val_to_name[SYM_CLASSES][cladatum->value - 1] = key; p->class_val_to_struct[cladatum->value - 1] = cladatum; return 0; } @@ -377,7 +370,6 @@ static int role_index(void *key, void *datum, void *datap) { struct policydb *p; struct role_datum *role; - struct flex_array *fa; role = datum; p = datap; @@ -386,10 +378,7 @@ static int role_index(void *key, void *datum, void *datap) || role->bounds > p->p_roles.nprim) return -EINVAL; - fa = p->sym_val_to_name[SYM_ROLES]; - if (flex_array_put_ptr(fa, role->value - 1, key, - GFP_KERNEL | __GFP_ZERO)) - BUG(); + p->sym_val_to_name[SYM_ROLES][role->value - 1] = key; p->role_val_to_struct[role->value - 1] = role; return 0; } @@ -398,7 +387,6 @@ static int type_index(void *key, void *datum, void *datap) { struct policydb *p; struct type_datum *typdatum; - struct flex_array *fa; typdatum = datum; p = datap; @@ -408,15 +396,8 @@ static int type_index(void *key, void *datum, void *datap) || typdatum->value > p->p_types.nprim || typdatum->bounds > p->p_types.nprim) return -EINVAL; - fa = p->sym_val_to_name[SYM_TYPES]; - if (flex_array_put_ptr(fa, typdatum->value - 1, key, - GFP_KERNEL | __GFP_ZERO)) - BUG(); - - fa = p->type_val_to_struct_array; - if (flex_array_put_ptr(fa, typdatum->value - 1, typdatum, - GFP_KERNEL | __GFP_ZERO)) - BUG(); + p->sym_val_to_name[SYM_TYPES][typdatum->value - 1] = key; + p->type_val_to_struct_array[typdatum->value - 1] = typdatum; } return 0; @@ -426,7 +407,6 @@ static int user_index(void *key, void *datum, void *datap) { struct policydb *p; struct user_datum *usrdatum; - struct flex_array *fa; usrdatum = datum; p = datap; @@ -435,10 +415,7 @@ static int user_index(void *key, void *datum, void *datap) || usrdatum->bounds > p->p_users.nprim) return -EINVAL; - fa = p->sym_val_to_name[SYM_USERS]; - if (flex_array_put_ptr(fa, usrdatum->value - 1, key, - GFP_KERNEL | __GFP_ZERO)) - BUG(); + p->sym_val_to_name[SYM_USERS][usrdatum->value - 1] = key; p->user_val_to_struct[usrdatum->value - 1] = usrdatum; return 0; } @@ -447,7 +424,6 @@ static int sens_index(void *key, void *datum, void *datap) { struct policydb *p; struct level_datum *levdatum; - struct flex_array *fa; levdatum = datum; p = datap; @@ -456,10 +432,8 @@ static int sens_index(void *key, void *datum, void *datap) if (!levdatum->level->sens || levdatum->level->sens > p->p_levels.nprim) return -EINVAL; - fa = p->sym_val_to_name[SYM_LEVELS]; - if (flex_array_put_ptr(fa, levdatum->level->sens - 1, key, - GFP_KERNEL | __GFP_ZERO)) - BUG(); + + p->sym_val_to_name[SYM_LEVELS][levdatum->level->sens - 1] = key; } return 0; @@ -469,7 +443,6 @@ static int cat_index(void *key, void *datum, void *datap) { struct policydb *p; struct cat_datum *catdatum; - struct flex_array *fa; catdatum = datum; p = datap; @@ -477,10 +450,8 @@ static int cat_index(void *key, void *datum, void *datap) if (!catdatum->isalias) { if (!catdatum->value || catdatum->value > p->p_cats.nprim) return -EINVAL; - fa = p->sym_val_to_name[SYM_CATS]; - if (flex_array_put_ptr(fa, catdatum->value - 1, key, - GFP_KERNEL | __GFP_ZERO)) - BUG(); + + p->sym_val_to_name[SYM_CATS][catdatum->value - 1] = key; } return 0; @@ -568,35 +539,23 @@ static int policydb_index(struct policydb *p) if (!p->user_val_to_struct) return -ENOMEM; - /* Yes, I want the sizeof the pointer, not the structure */ - p->type_val_to_struct_array = flex_array_alloc(sizeof(struct type_datum *), - p->p_types.nprim, - GFP_KERNEL | __GFP_ZERO); + p->type_val_to_struct_array = kvcalloc(p->p_types.nprim, + sizeof(*p->type_val_to_struct_array), + GFP_KERNEL); if (!p->type_val_to_struct_array) return -ENOMEM; - rc = flex_array_prealloc(p->type_val_to_struct_array, 0, - p->p_types.nprim, GFP_KERNEL | __GFP_ZERO); - if (rc) - goto out; - rc = cond_init_bool_indexes(p); if (rc) goto out; for (i = 0; i < SYM_NUM; i++) { - p->sym_val_to_name[i] = flex_array_alloc(sizeof(char *), - p->symtab[i].nprim, - GFP_KERNEL | __GFP_ZERO); + p->sym_val_to_name[i] = kvcalloc(p->symtab[i].nprim, + sizeof(char *), + GFP_KERNEL); if (!p->sym_val_to_name[i]) return -ENOMEM; - rc = flex_array_prealloc(p->sym_val_to_name[i], - 0, p->symtab[i].nprim, - GFP_KERNEL | __GFP_ZERO); - if (rc) - goto out; - rc = hashtab_map(p->symtab[i].table, index_f[i], p); if (rc) goto out; @@ -810,16 +769,13 @@ void policydb_destroy(struct policydb *p) hashtab_destroy(p->symtab[i].table); } - for (i = 0; i < SYM_NUM; i++) { - if (p->sym_val_to_name[i]) - flex_array_free(p->sym_val_to_name[i]); - } + for (i = 0; i < SYM_NUM; i++) + kvfree(p->sym_val_to_name[i]); kfree(p->class_val_to_struct); kfree(p->role_val_to_struct); kfree(p->user_val_to_struct); - if (p->type_val_to_struct_array) - flex_array_free(p->type_val_to_struct_array); + kvfree(p->type_val_to_struct_array); avtab_destroy(&p->te_avtab); @@ -872,17 +828,9 @@ void policydb_destroy(struct policydb *p) hashtab_map(p->range_tr, range_tr_destroy, NULL); hashtab_destroy(p->range_tr); - if (p->type_attr_map_array) { - for (i = 0; i < p->p_types.nprim; i++) { - struct ebitmap *e; - - e = flex_array_get(p->type_attr_map_array, i); - if (!e) - continue; - ebitmap_destroy(e); - } - flex_array_free(p->type_attr_map_array); - } + for (i = 0; i < p->p_types.nprim; i++) + ebitmap_destroy(&p->type_attr_map_array[i]); + kvfree(p->type_attr_map_array); ebitmap_destroy(&p->filename_trans_ttypes); ebitmap_destroy(&p->policycaps); @@ -1770,8 +1718,7 @@ static int type_bounds_sanity_check(void *key, void *datum, void *datap) return -EINVAL; } - upper = flex_array_get_ptr(p->type_val_to_struct_array, - upper->bounds - 1); + upper = p->type_val_to_struct_array[upper->bounds - 1]; BUG_ON(!upper); if (upper->attribute) { @@ -2543,23 +2490,15 @@ int policydb_read(struct policydb *p, void *fp) if (rc) goto bad; - rc = -ENOMEM; - p->type_attr_map_array = flex_array_alloc(sizeof(struct ebitmap), - p->p_types.nprim, - GFP_KERNEL | __GFP_ZERO); + p->type_attr_map_array = kvcalloc(p->p_types.nprim, + sizeof(*p->type_attr_map_array), + GFP_KERNEL); if (!p->type_attr_map_array) goto bad; - /* preallocate so we don't have to worry about the put ever failing */ - rc = flex_array_prealloc(p->type_attr_map_array, 0, p->p_types.nprim, - GFP_KERNEL | __GFP_ZERO); - if (rc) - goto bad; - for (i = 0; i < p->p_types.nprim; i++) { - struct ebitmap *e = flex_array_get(p->type_attr_map_array, i); + struct ebitmap *e = &p->type_attr_map_array[i]; - BUG_ON(!e); ebitmap_init(e); if (p->policyvers >= POLICYDB_VERSION_AVTAB) { rc = ebitmap_read(e, fp); @@ -3554,9 +3493,8 @@ int policydb_write(struct policydb *p, void *fp) return rc; for (i = 0; i < p->p_types.nprim; i++) { - struct ebitmap *e = flex_array_get(p->type_attr_map_array, i); + struct ebitmap *e = &p->type_attr_map_array[i]; - BUG_ON(!e); rc = ebitmap_write(e, fp); if (rc) return rc; diff --git a/security/selinux/ss/policydb.h b/security/selinux/ss/policydb.h index 215f8f30ac5a..27039149ff0a 100644 --- a/security/selinux/ss/policydb.h +++ b/security/selinux/ss/policydb.h @@ -24,8 +24,6 @@ #ifndef _SS_POLICYDB_H_ #define _SS_POLICYDB_H_ -#include - #include "symtab.h" #include "avtab.h" #include "sidtab.h" @@ -251,13 +249,13 @@ struct policydb { #define p_cats symtab[SYM_CATS] /* symbol names indexed by (value - 1) */ - struct flex_array *sym_val_to_name[SYM_NUM]; + char **sym_val_to_name[SYM_NUM]; /* class, role, and user attributes indexed by (value - 1) */ struct class_datum **class_val_to_struct; struct role_datum **role_val_to_struct; struct user_datum **user_val_to_struct; - struct flex_array *type_val_to_struct_array; + struct type_datum **type_val_to_struct_array; /* type enforcement access vectors and transitions */ struct avtab te_avtab; @@ -294,7 +292,7 @@ struct policydb { struct hashtab *range_tr; /* type -> attribute reverse mapping */ - struct flex_array *type_attr_map_array; + struct ebitmap *type_attr_map_array; struct ebitmap policycaps; @@ -369,9 +367,7 @@ static inline int put_entry(const void *buf, size_t bytes, int num, struct polic static inline char *sym_name(struct policydb *p, unsigned int sym_num, unsigned int element_nr) { - struct flex_array *fa = p->sym_val_to_name[sym_num]; - - return flex_array_get_ptr(fa, element_nr); + return p->sym_val_to_name[sym_num][element_nr]; } extern u16 string_to_security_class(struct policydb *p, const char *name); diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index 1269e2be3c2d..ec62918521b1 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -49,7 +49,6 @@ #include #include #include -#include #include #include @@ -545,15 +544,13 @@ static void type_attribute_bounds_av(struct policydb *policydb, struct type_datum *target; u32 masked = 0; - source = flex_array_get_ptr(policydb->type_val_to_struct_array, - scontext->type - 1); + source = policydb->type_val_to_struct_array[scontext->type - 1]; BUG_ON(!source); if (!source->bounds) return; - target = flex_array_get_ptr(policydb->type_val_to_struct_array, - tcontext->type - 1); + target = policydb->type_val_to_struct_array[tcontext->type - 1]; BUG_ON(!target); memset(&lo_avd, 0, sizeof(lo_avd)); @@ -653,11 +650,9 @@ static void context_struct_compute_av(struct policydb *policydb, */ avkey.target_class = tclass; avkey.specified = AVTAB_AV | AVTAB_XPERMS; - sattr = flex_array_get(policydb->type_attr_map_array, - scontext->type - 1); + sattr = &policydb->type_attr_map_array[scontext->type - 1]; BUG_ON(!sattr); - tattr = flex_array_get(policydb->type_attr_map_array, - tcontext->type - 1); + tattr = &policydb->type_attr_map_array[tcontext->type - 1]; BUG_ON(!tattr); ebitmap_for_each_positive_bit(sattr, snode, i) { ebitmap_for_each_positive_bit(tattr, tnode, j) { @@ -900,8 +895,7 @@ int security_bounded_transition(struct selinux_state *state, index = new_context->type; while (true) { - type = flex_array_get_ptr(policydb->type_val_to_struct_array, - index - 1); + type = policydb->type_val_to_struct_array[index - 1]; BUG_ON(!type); /* not bounded anymore */ @@ -1064,11 +1058,9 @@ void security_compute_xperms_decision(struct selinux_state *state, avkey.target_class = tclass; avkey.specified = AVTAB_XPERMS; - sattr = flex_array_get(policydb->type_attr_map_array, - scontext->type - 1); + sattr = &policydb->type_attr_map_array[scontext->type - 1]; BUG_ON(!sattr); - tattr = flex_array_get(policydb->type_attr_map_array, - tcontext->type - 1); + tattr = &policydb->type_attr_map_array[tcontext->type - 1]; BUG_ON(!tattr); ebitmap_for_each_positive_bit(sattr, snode, i) { ebitmap_for_each_positive_bit(tattr, tnode, j) { From ba20ba2e3743bac786dff777954c11930256075e Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 11 Mar 2019 23:31:14 -0700 Subject: [PATCH 35/38] generic radix trees Very simple radix tree implementation that supports storing arbitrary size entries, up to PAGE_SIZE - upcoming patches will convert existing flex_array users to genradixes. The new genradix code has a much simpler API and implementation, and doesn't have a hard limit on the number of elements like flex_array does. Link: http://lkml.kernel.org/r/20181217131929.11727-5-kent.overstreet@gmail.com Signed-off-by: Kent Overstreet Cc: Alexey Dobriyan Cc: Al Viro Cc: Dave Hansen Cc: Eric Paris Cc: Marcelo Ricardo Leitner Cc: Matthew Wilcox Cc: Neil Horman Cc: Paul Moore Cc: Pravin B Shelar Cc: Shaohua Li Cc: Stephen Smalley Cc: Vlad Yasevich Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/core-api/generic-radix-tree.rst | 12 + Documentation/core-api/index.rst | 1 + include/linux/generic-radix-tree.h | 231 ++++++++++++++++++ lib/Makefile | 3 +- lib/generic-radix-tree.c | 217 ++++++++++++++++ 5 files changed, 463 insertions(+), 1 deletion(-) create mode 100644 Documentation/core-api/generic-radix-tree.rst create mode 100644 include/linux/generic-radix-tree.h create mode 100644 lib/generic-radix-tree.c diff --git a/Documentation/core-api/generic-radix-tree.rst b/Documentation/core-api/generic-radix-tree.rst new file mode 100644 index 000000000000..ed42839ae42f --- /dev/null +++ b/Documentation/core-api/generic-radix-tree.rst @@ -0,0 +1,12 @@ +================================= +Generic radix trees/sparse arrays +================================= + +.. kernel-doc:: include/linux/generic-radix-tree.h + :doc: Generic radix trees/sparse arrays + +generic radix tree functions +---------------------------- + +.. kernel-doc:: include/linux/generic-radix-tree.h + :functions: diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst index 3adee82be311..6870baffef82 100644 --- a/Documentation/core-api/index.rst +++ b/Documentation/core-api/index.rst @@ -28,6 +28,7 @@ Core utilities errseq printk-formats circular-buffers + generic-radix-tree memory-allocation mm-api gfp_mask-from-fs-io diff --git a/include/linux/generic-radix-tree.h b/include/linux/generic-radix-tree.h new file mode 100644 index 000000000000..3a91130a4fbd --- /dev/null +++ b/include/linux/generic-radix-tree.h @@ -0,0 +1,231 @@ +#ifndef _LINUX_GENERIC_RADIX_TREE_H +#define _LINUX_GENERIC_RADIX_TREE_H + +/** + * DOC: Generic radix trees/sparse arrays: + * + * Very simple and minimalistic, supporting arbitrary size entries up to + * PAGE_SIZE. + * + * A genradix is defined with the type it will store, like so: + * + * static GENRADIX(struct foo) foo_genradix; + * + * The main operations are: + * + * - genradix_init(radix) - initialize an empty genradix + * + * - genradix_free(radix) - free all memory owned by the genradix and + * reinitialize it + * + * - genradix_ptr(radix, idx) - gets a pointer to the entry at idx, returning + * NULL if that entry does not exist + * + * - genradix_ptr_alloc(radix, idx, gfp) - gets a pointer to an entry, + * allocating it if necessary + * + * - genradix_for_each(radix, iter, p) - iterate over each entry in a genradix + * + * The radix tree allocates one page of entries at a time, so entries may exist + * that were never explicitly allocated - they will be initialized to all + * zeroes. + * + * Internally, a genradix is just a radix tree of pages, and indexing works in + * terms of byte offsets. The wrappers in this header file use sizeof on the + * type the radix contains to calculate a byte offset from the index - see + * __idx_to_offset. + */ + +#include +#include +#include +#include + +struct genradix_root; + +struct __genradix { + struct genradix_root __rcu *root; +}; + +/* + * NOTE: currently, sizeof(_type) must not be larger than PAGE_SIZE: + */ + +#define __GENRADIX_INITIALIZER \ + { \ + .tree = { \ + .root = NULL, \ + } \ + } + +/* + * We use a 0 size array to stash the type we're storing without taking any + * space at runtime - then the various accessor macros can use typeof() to get + * to it for casts/sizeof - we also force the alignment so that storing a type + * with a ridiculous alignment doesn't blow up the alignment or size of the + * genradix. + */ + +#define GENRADIX(_type) \ +struct { \ + struct __genradix tree; \ + _type type[0] __aligned(1); \ +} + +#define DEFINE_GENRADIX(_name, _type) \ + GENRADIX(_type) _name = __GENRADIX_INITIALIZER + +/** + * genradix_init - initialize a genradix + * @_radix: genradix to initialize + * + * Does not fail + */ +#define genradix_init(_radix) \ +do { \ + *(_radix) = (typeof(*_radix)) __GENRADIX_INITIALIZER; \ +} while (0) + +void __genradix_free(struct __genradix *); + +/** + * genradix_free: free all memory owned by a genradix + * @_radix: the genradix to free + * + * After freeing, @_radix will be reinitialized and empty + */ +#define genradix_free(_radix) __genradix_free(&(_radix)->tree) + +static inline size_t __idx_to_offset(size_t idx, size_t obj_size) +{ + if (__builtin_constant_p(obj_size)) + BUILD_BUG_ON(obj_size > PAGE_SIZE); + else + BUG_ON(obj_size > PAGE_SIZE); + + if (!is_power_of_2(obj_size)) { + size_t objs_per_page = PAGE_SIZE / obj_size; + + return (idx / objs_per_page) * PAGE_SIZE + + (idx % objs_per_page) * obj_size; + } else { + return idx * obj_size; + } +} + +#define __genradix_cast(_radix) (typeof((_radix)->type[0]) *) +#define __genradix_obj_size(_radix) sizeof((_radix)->type[0]) +#define __genradix_idx_to_offset(_radix, _idx) \ + __idx_to_offset(_idx, __genradix_obj_size(_radix)) + +void *__genradix_ptr(struct __genradix *, size_t); + +/** + * genradix_ptr - get a pointer to a genradix entry + * @_radix: genradix to access + * @_idx: index to fetch + * + * Returns a pointer to entry at @_idx, or NULL if that entry does not exist. + */ +#define genradix_ptr(_radix, _idx) \ + (__genradix_cast(_radix) \ + __genradix_ptr(&(_radix)->tree, \ + __genradix_idx_to_offset(_radix, _idx))) + +void *__genradix_ptr_alloc(struct __genradix *, size_t, gfp_t); + +/** + * genradix_ptr_alloc - get a pointer to a genradix entry, allocating it + * if necessary + * @_radix: genradix to access + * @_idx: index to fetch + * @_gfp: gfp mask + * + * Returns a pointer to entry at @_idx, or NULL on allocation failure + */ +#define genradix_ptr_alloc(_radix, _idx, _gfp) \ + (__genradix_cast(_radix) \ + __genradix_ptr_alloc(&(_radix)->tree, \ + __genradix_idx_to_offset(_radix, _idx), \ + _gfp)) + +struct genradix_iter { + size_t offset; + size_t pos; +}; + +/** + * genradix_iter_init - initialize a genradix_iter + * @_radix: genradix that will be iterated over + * @_idx: index to start iterating from + */ +#define genradix_iter_init(_radix, _idx) \ + ((struct genradix_iter) { \ + .pos = (_idx), \ + .offset = __genradix_idx_to_offset((_radix), (_idx)),\ + }) + +void *__genradix_iter_peek(struct genradix_iter *, struct __genradix *, size_t); + +/** + * genradix_iter_peek - get first entry at or above iterator's current + * position + * @_iter: a genradix_iter + * @_radix: genradix being iterated over + * + * If no more entries exist at or above @_iter's current position, returns NULL + */ +#define genradix_iter_peek(_iter, _radix) \ + (__genradix_cast(_radix) \ + __genradix_iter_peek(_iter, &(_radix)->tree, \ + PAGE_SIZE / __genradix_obj_size(_radix))) + +static inline void __genradix_iter_advance(struct genradix_iter *iter, + size_t obj_size) +{ + iter->offset += obj_size; + + if (!is_power_of_2(obj_size) && + (iter->offset & (PAGE_SIZE - 1)) + obj_size > PAGE_SIZE) + iter->offset = round_up(iter->offset, PAGE_SIZE); + + iter->pos++; +} + +#define genradix_iter_advance(_iter, _radix) \ + __genradix_iter_advance(_iter, __genradix_obj_size(_radix)) + +#define genradix_for_each_from(_radix, _iter, _p, _start) \ + for (_iter = genradix_iter_init(_radix, _start); \ + (_p = genradix_iter_peek(&_iter, _radix)) != NULL; \ + genradix_iter_advance(&_iter, _radix)) + +/** + * genradix_for_each - iterate over entry in a genradix + * @_radix: genradix to iterate over + * @_iter: a genradix_iter to track current position + * @_p: pointer to genradix entry type + * + * On every iteration, @_p will point to the current entry, and @_iter.pos + * will be the current entry's index. + */ +#define genradix_for_each(_radix, _iter, _p) \ + genradix_for_each_from(_radix, _iter, _p, 0) + +int __genradix_prealloc(struct __genradix *, size_t, gfp_t); + +/** + * genradix_prealloc - preallocate entries in a generic radix tree + * @_radix: genradix to preallocate + * @_nr: number of entries to preallocate + * @_gfp: gfp mask + * + * Returns 0 on success, -ENOMEM on failure + */ +#define genradix_prealloc(_radix, _nr, _gfp) \ + __genradix_prealloc(&(_radix)->tree, \ + __genradix_idx_to_offset(_radix, _nr + 1),\ + _gfp) + + +#endif /* _LINUX_GENERIC_RADIX_TREE_H */ diff --git a/lib/Makefile b/lib/Makefile index 647517940b29..b798b41d01ae 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -38,7 +38,8 @@ obj-y += bcd.o div64.o sort.o parser.o debug_locks.o random32.o \ gcd.o lcm.o list_sort.o uuid.o flex_array.o iov_iter.o clz_ctz.o \ bsearch.o find_bit.o llist.o memweight.o kfifo.o \ percpu-refcount.o rhashtable.o reciprocal_div.o \ - once.o refcount.o usercopy.o errseq.o bucket_locks.o + once.o refcount.o usercopy.o errseq.o bucket_locks.o \ + generic-radix-tree.o obj-$(CONFIG_STRING_SELFTEST) += test_string.o obj-y += string_helpers.o obj-$(CONFIG_TEST_STRING_HELPERS) += test-string_helpers.o diff --git a/lib/generic-radix-tree.c b/lib/generic-radix-tree.c new file mode 100644 index 000000000000..a7bafc413730 --- /dev/null +++ b/lib/generic-radix-tree.c @@ -0,0 +1,217 @@ + +#include +#include +#include + +#define GENRADIX_ARY (PAGE_SIZE / sizeof(struct genradix_node *)) +#define GENRADIX_ARY_SHIFT ilog2(GENRADIX_ARY) + +struct genradix_node { + union { + /* Interior node: */ + struct genradix_node *children[GENRADIX_ARY]; + + /* Leaf: */ + u8 data[PAGE_SIZE]; + }; +}; + +static inline int genradix_depth_shift(unsigned depth) +{ + return PAGE_SHIFT + GENRADIX_ARY_SHIFT * depth; +} + +/* + * Returns size (of data, in bytes) that a tree of a given depth holds: + */ +static inline size_t genradix_depth_size(unsigned depth) +{ + return 1UL << genradix_depth_shift(depth); +} + +/* depth that's needed for a genradix that can address up to ULONG_MAX: */ +#define GENRADIX_MAX_DEPTH \ + DIV_ROUND_UP(BITS_PER_LONG - PAGE_SHIFT, GENRADIX_ARY_SHIFT) + +#define GENRADIX_DEPTH_MASK \ + ((unsigned long) (roundup_pow_of_two(GENRADIX_MAX_DEPTH + 1) - 1)) + +unsigned genradix_root_to_depth(struct genradix_root *r) +{ + return (unsigned long) r & GENRADIX_DEPTH_MASK; +} + +struct genradix_node *genradix_root_to_node(struct genradix_root *r) +{ + return (void *) ((unsigned long) r & ~GENRADIX_DEPTH_MASK); +} + +/* + * Returns pointer to the specified byte @offset within @radix, or NULL if not + * allocated + */ +void *__genradix_ptr(struct __genradix *radix, size_t offset) +{ + struct genradix_root *r = READ_ONCE(radix->root); + struct genradix_node *n = genradix_root_to_node(r); + unsigned level = genradix_root_to_depth(r); + + if (ilog2(offset) >= genradix_depth_shift(level)) + return NULL; + + while (1) { + if (!n) + return NULL; + if (!level) + break; + + level--; + + n = n->children[offset >> genradix_depth_shift(level)]; + offset &= genradix_depth_size(level) - 1; + } + + return &n->data[offset]; +} +EXPORT_SYMBOL(__genradix_ptr); + +/* + * Returns pointer to the specified byte @offset within @radix, allocating it if + * necessary - newly allocated slots are always zeroed out: + */ +void *__genradix_ptr_alloc(struct __genradix *radix, size_t offset, + gfp_t gfp_mask) +{ + struct genradix_root *v = READ_ONCE(radix->root); + struct genradix_node *n, *new_node = NULL; + unsigned level; + + /* Increase tree depth if necessary: */ + while (1) { + struct genradix_root *r = v, *new_root; + + n = genradix_root_to_node(r); + level = genradix_root_to_depth(r); + + if (n && ilog2(offset) < genradix_depth_shift(level)) + break; + + if (!new_node) { + new_node = (void *) + __get_free_page(gfp_mask|__GFP_ZERO); + if (!new_node) + return NULL; + } + + new_node->children[0] = n; + new_root = ((struct genradix_root *) + ((unsigned long) new_node | (n ? level + 1 : 0))); + + if ((v = cmpxchg_release(&radix->root, r, new_root)) == r) { + v = new_root; + new_node = NULL; + } + } + + while (level--) { + struct genradix_node **p = + &n->children[offset >> genradix_depth_shift(level)]; + offset &= genradix_depth_size(level) - 1; + + n = READ_ONCE(*p); + if (!n) { + if (!new_node) { + new_node = (void *) + __get_free_page(gfp_mask|__GFP_ZERO); + if (!new_node) + return NULL; + } + + if (!(n = cmpxchg_release(p, NULL, new_node))) + swap(n, new_node); + } + } + + if (new_node) + free_page((unsigned long) new_node); + + return &n->data[offset]; +} +EXPORT_SYMBOL(__genradix_ptr_alloc); + +void *__genradix_iter_peek(struct genradix_iter *iter, + struct __genradix *radix, + size_t objs_per_page) +{ + struct genradix_root *r; + struct genradix_node *n; + unsigned level, i; +restart: + r = READ_ONCE(radix->root); + if (!r) + return NULL; + + n = genradix_root_to_node(r); + level = genradix_root_to_depth(r); + + if (ilog2(iter->offset) >= genradix_depth_shift(level)) + return NULL; + + while (level) { + level--; + + i = (iter->offset >> genradix_depth_shift(level)) & + (GENRADIX_ARY - 1); + + while (!n->children[i]) { + i++; + iter->offset = round_down(iter->offset + + genradix_depth_size(level), + genradix_depth_size(level)); + iter->pos = (iter->offset >> PAGE_SHIFT) * + objs_per_page; + if (i == GENRADIX_ARY) + goto restart; + } + + n = n->children[i]; + } + + return &n->data[iter->offset & (PAGE_SIZE - 1)]; +} +EXPORT_SYMBOL(__genradix_iter_peek); + +static void genradix_free_recurse(struct genradix_node *n, unsigned level) +{ + if (level) { + unsigned i; + + for (i = 0; i < GENRADIX_ARY; i++) + if (n->children[i]) + genradix_free_recurse(n->children[i], level - 1); + } + + free_page((unsigned long) n); +} + +int __genradix_prealloc(struct __genradix *radix, size_t size, + gfp_t gfp_mask) +{ + size_t offset; + + for (offset = 0; offset < size; offset += PAGE_SIZE) + if (!__genradix_ptr_alloc(radix, offset, gfp_mask)) + return -ENOMEM; + + return 0; +} +EXPORT_SYMBOL(__genradix_prealloc); + +void __genradix_free(struct __genradix *radix) +{ + struct genradix_root *r = xchg(&radix->root, NULL); + + genradix_free_recurse(genradix_root_to_node(r), + genradix_root_to_depth(r)); +} +EXPORT_SYMBOL(__genradix_free); From 94f8f3b02e1ee0418b5cc9352626cdc2b6bd4299 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 11 Mar 2019 23:31:18 -0700 Subject: [PATCH 36/38] proc: commit to genradix The new generic radix trees have a simpler API and implementation, and no limitations on number of elements, so all flex_array users are being converted Link: http://lkml.kernel.org/r/20181217131929.11727-6-kent.overstreet@gmail.com Signed-off-by: Kent Overstreet Reviewed-by: Alexey Dobriyan Cc: Al Viro Cc: Dave Hansen Cc: Eric Paris Cc: Marcelo Ricardo Leitner Cc: Matthew Wilcox Cc: Neil Horman Cc: Paul Moore Cc: Pravin B Shelar Cc: Shaohua Li Cc: Stephen Smalley Cc: Vlad Yasevich Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/base.c | 43 +++++++++++++++---------------------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/fs/proc/base.c b/fs/proc/base.c index 1b548d060d28..f5ebdd87afb2 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -59,6 +59,7 @@ #include #include #include +#include #include #include #include @@ -92,7 +93,6 @@ #include #include #include -#include #include #include #include "internal.h" @@ -2142,11 +2142,12 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) struct task_struct *task; struct mm_struct *mm; unsigned long nr_files, pos, i; - struct flex_array *fa = NULL; - struct map_files_info info; + GENRADIX(struct map_files_info) fa; struct map_files_info *p; int ret; + genradix_init(&fa); + ret = -ENOENT; task = get_proc_task(file_inode(file)); if (!task) @@ -2178,35 +2179,22 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) */ for (vma = mm->mmap, pos = 2; vma; vma = vma->vm_next) { - if (vma->vm_file && ++pos > ctx->pos) - nr_files++; - } + if (!vma->vm_file) + continue; + if (++pos <= ctx->pos) + continue; - if (nr_files) { - fa = flex_array_alloc(sizeof(info), nr_files, - GFP_KERNEL); - if (!fa || flex_array_prealloc(fa, 0, nr_files, - GFP_KERNEL)) { + p = genradix_ptr_alloc(&fa, nr_files++, GFP_KERNEL); + if (!p) { ret = -ENOMEM; - if (fa) - flex_array_free(fa); up_read(&mm->mmap_sem); mmput(mm); goto out_put_task; } - for (i = 0, vma = mm->mmap, pos = 2; vma; - vma = vma->vm_next) { - if (!vma->vm_file) - continue; - if (++pos <= ctx->pos) - continue; - info.start = vma->vm_start; - info.end = vma->vm_end; - info.mode = vma->vm_file->f_mode; - if (flex_array_put(fa, i++, &info, GFP_KERNEL)) - BUG(); - } + p->start = vma->vm_start; + p->end = vma->vm_end; + p->mode = vma->vm_file->f_mode; } up_read(&mm->mmap_sem); mmput(mm); @@ -2215,7 +2203,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) char buf[4 * sizeof(long) + 2]; /* max: %lx-%lx\0 */ unsigned int len; - p = flex_array_get(fa, i); + p = genradix_ptr(&fa, i); len = snprintf(buf, sizeof(buf), "%lx-%lx", p->start, p->end); if (!proc_fill_cache(file, ctx, buf, len, @@ -2225,12 +2213,11 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) break; ctx->pos++; } - if (fa) - flex_array_free(fa); out_put_task: put_task_struct(task); out: + genradix_free(&fa); return ret; } From 2075e50caf5ea28be3cba0d01b3058bb5c3b0168 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 11 Mar 2019 23:31:22 -0700 Subject: [PATCH 37/38] sctp: convert to genradix This also makes sctp_stream_alloc_(out|in) saner, in that they no longer allocate new flex_arrays/genradixes, they just preallocate more elements. This code does however have a suspicious lack of locking. Link: http://lkml.kernel.org/r/20181217131929.11727-7-kent.overstreet@gmail.com Signed-off-by: Kent Overstreet Cc: Vlad Yasevich Cc: Neil Horman Cc: Marcelo Ricardo Leitner Cc: Alexey Dobriyan Cc: Al Viro Cc: Dave Hansen Cc: Eric Paris Cc: Matthew Wilcox Cc: Paul Moore Cc: Pravin B Shelar Cc: Shaohua Li Cc: Stephen Smalley Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/net/sctp/structs.h | 15 +++-- net/sctp/stream.c | 125 ++++++----------------------------- net/sctp/stream_interleave.c | 2 +- 3 files changed, 28 insertions(+), 114 deletions(-) diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h index 58e4b23cecf4..140fd836a396 100644 --- a/include/net/sctp/structs.h +++ b/include/net/sctp/structs.h @@ -48,6 +48,7 @@ #define __sctp_structs_h__ #include +#include #include #include /* linux/in.h needs this!! */ #include /* We get struct sockaddr_in. */ @@ -57,7 +58,6 @@ #include /* This gets us atomic counters. */ #include /* We need sk_buff_head. */ #include /* We need tq_struct. */ -#include /* We need flex_array. */ #include /* We need sctp* header structs. */ #include /* We need auth specific structs */ #include /* For inet_skb_parm */ @@ -1449,8 +1449,9 @@ struct sctp_stream_in { }; struct sctp_stream { - struct flex_array *out; - struct flex_array *in; + GENRADIX(struct sctp_stream_out) out; + GENRADIX(struct sctp_stream_in) in; + __u16 outcnt; __u16 incnt; /* Current stream being sent, if any */ @@ -1473,17 +1474,17 @@ struct sctp_stream { }; static inline struct sctp_stream_out *sctp_stream_out( - const struct sctp_stream *stream, + struct sctp_stream *stream, __u16 sid) { - return flex_array_get(stream->out, sid); + return genradix_ptr(&stream->out, sid); } static inline struct sctp_stream_in *sctp_stream_in( - const struct sctp_stream *stream, + struct sctp_stream *stream, __u16 sid) { - return flex_array_get(stream->in, sid); + return genradix_ptr(&stream->in, sid); } #define SCTP_SO(s, i) sctp_stream_out((s), (i)) diff --git a/net/sctp/stream.c b/net/sctp/stream.c index 3b47457862cc..b6bb68adac6e 100644 --- a/net/sctp/stream.c +++ b/net/sctp/stream.c @@ -37,66 +37,6 @@ #include #include -static struct flex_array *fa_alloc(size_t elem_size, size_t elem_count, - gfp_t gfp) -{ - struct flex_array *result; - int err; - - result = flex_array_alloc(elem_size, elem_count, gfp); - if (result) { - err = flex_array_prealloc(result, 0, elem_count, gfp); - if (err) { - flex_array_free(result); - result = NULL; - } - } - - return result; -} - -static void fa_free(struct flex_array *fa) -{ - if (fa) - flex_array_free(fa); -} - -static void fa_copy(struct flex_array *fa, struct flex_array *from, - size_t index, size_t count) -{ - void *elem; - - while (count--) { - elem = flex_array_get(from, index); - flex_array_put(fa, index, elem, 0); - index++; - } -} - -static void fa_zero(struct flex_array *fa, size_t index, size_t count) -{ - void *elem; - - while (count--) { - elem = flex_array_get(fa, index); - memset(elem, 0, fa->element_size); - index++; - } -} - -static size_t fa_index(struct flex_array *fa, void *elem, size_t count) -{ - size_t index = 0; - - while (count--) { - if (elem == flex_array_get(fa, index)) - break; - index++; - } - - return index; -} - /* Migrates chunks from stream queues to new stream queues if needed, * but not across associations. Also, removes those chunks to streams * higher than the new max. @@ -153,53 +93,32 @@ static void sctp_stream_outq_migrate(struct sctp_stream *stream, static int sctp_stream_alloc_out(struct sctp_stream *stream, __u16 outcnt, gfp_t gfp) { - struct flex_array *out; - size_t elem_size = sizeof(struct sctp_stream_out); + int ret; - out = fa_alloc(elem_size, outcnt, gfp); - if (!out) - return -ENOMEM; + if (outcnt <= stream->outcnt) + return 0; - if (stream->out) { - fa_copy(out, stream->out, 0, min(outcnt, stream->outcnt)); - if (stream->out_curr) { - size_t index = fa_index(stream->out, stream->out_curr, - stream->outcnt); - - BUG_ON(index == stream->outcnt); - stream->out_curr = flex_array_get(out, index); - } - fa_free(stream->out); - } - - if (outcnt > stream->outcnt) - fa_zero(out, stream->outcnt, (outcnt - stream->outcnt)); - - stream->out = out; + ret = genradix_prealloc(&stream->out, outcnt, gfp); + if (ret) + return ret; + stream->outcnt = outcnt; return 0; } static int sctp_stream_alloc_in(struct sctp_stream *stream, __u16 incnt, gfp_t gfp) { - struct flex_array *in; - size_t elem_size = sizeof(struct sctp_stream_in); + int ret; - in = fa_alloc(elem_size, incnt, gfp); - if (!in) - return -ENOMEM; + if (incnt <= stream->incnt) + return 0; - if (stream->in) { - fa_copy(in, stream->in, 0, min(incnt, stream->incnt)); - fa_free(stream->in); - } - - if (incnt > stream->incnt) - fa_zero(in, stream->incnt, (incnt - stream->incnt)); - - stream->in = in; + ret = genradix_prealloc(&stream->in, incnt, gfp); + if (ret) + return ret; + stream->incnt = incnt; return 0; } @@ -226,7 +145,6 @@ int sctp_stream_init(struct sctp_stream *stream, __u16 outcnt, __u16 incnt, if (ret) goto out; - stream->outcnt = outcnt; for (i = 0; i < stream->outcnt; i++) SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN; @@ -238,14 +156,11 @@ in: ret = sctp_stream_alloc_in(stream, incnt, gfp); if (ret) { sched->free(stream); - fa_free(stream->out); - stream->out = NULL; + genradix_free(&stream->out); stream->outcnt = 0; goto out; } - stream->incnt = incnt; - out: return ret; } @@ -270,8 +185,8 @@ void sctp_stream_free(struct sctp_stream *stream) sched->free(stream); for (i = 0; i < stream->outcnt; i++) kfree(SCTP_SO(stream, i)->ext); - fa_free(stream->out); - fa_free(stream->in); + genradix_free(&stream->out); + genradix_free(&stream->in); } void sctp_stream_clear(struct sctp_stream *stream) @@ -302,8 +217,8 @@ void sctp_stream_update(struct sctp_stream *stream, struct sctp_stream *new) sched->sched_all(stream); - new->out = NULL; - new->in = NULL; + new->out.tree.root = NULL; + new->in.tree.root = NULL; new->outcnt = 0; new->incnt = 0; } @@ -555,8 +470,6 @@ int sctp_send_add_streams(struct sctp_association *asoc, goto out; } - stream->outcnt = outcnt; - asoc->strreset_outstanding = !!out + !!in; out: diff --git a/net/sctp/stream_interleave.c b/net/sctp/stream_interleave.c index a6bf21579466..102c6fefe38c 100644 --- a/net/sctp/stream_interleave.c +++ b/net/sctp/stream_interleave.c @@ -101,7 +101,7 @@ static void sctp_chunk_assign_mid(struct sctp_chunk *chunk) static bool sctp_validate_data(struct sctp_chunk *chunk) { - const struct sctp_stream *stream; + struct sctp_stream *stream; __u16 sid, ssn; if (chunk->chunk_hdr->type != SCTP_CID_DATA) From 586187d7de71b4da7956ba588ae42253b9ff6482 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 11 Mar 2019 23:31:26 -0700 Subject: [PATCH 38/38] Drop flex_arrays All existing users have been converted to generic radix trees Link: http://lkml.kernel.org/r/20181217131929.11727-8-kent.overstreet@gmail.com Signed-off-by: Kent Overstreet Acked-by: Dave Hansen Cc: Alexey Dobriyan Cc: Al Viro Cc: Eric Paris Cc: Marcelo Ricardo Leitner Cc: Matthew Wilcox Cc: Neil Horman Cc: Paul Moore Cc: Pravin B Shelar Cc: Shaohua Li Cc: Stephen Smalley Cc: Vlad Yasevich Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/core-api/flexible-arrays.rst | 130 ------- Documentation/flexible-arrays.txt | 123 ------- include/linux/flex_array.h | 149 -------- include/linux/poison.h | 3 - lib/Makefile | 2 +- lib/flex_array.c | 398 --------------------- tools/include/linux/poison.h | 3 - 7 files changed, 1 insertion(+), 807 deletions(-) delete mode 100644 Documentation/core-api/flexible-arrays.rst delete mode 100644 Documentation/flexible-arrays.txt delete mode 100644 include/linux/flex_array.h delete mode 100644 lib/flex_array.c diff --git a/Documentation/core-api/flexible-arrays.rst b/Documentation/core-api/flexible-arrays.rst deleted file mode 100644 index b6b85a1b518e..000000000000 --- a/Documentation/core-api/flexible-arrays.rst +++ /dev/null @@ -1,130 +0,0 @@ - -=================================== -Using flexible arrays in the kernel -=================================== - -Large contiguous memory allocations can be unreliable in the Linux kernel. -Kernel programmers will sometimes respond to this problem by allocating -pages with :c:func:`vmalloc()`. This solution not ideal, though. On 32-bit -systems, memory from vmalloc() must be mapped into a relatively small address -space; it's easy to run out. On SMP systems, the page table changes required -by vmalloc() allocations can require expensive cross-processor interrupts on -all CPUs. And, on all systems, use of space in the vmalloc() range increases -pressure on the translation lookaside buffer (TLB), reducing the performance -of the system. - -In many cases, the need for memory from vmalloc() can be eliminated by piecing -together an array from smaller parts; the flexible array library exists to make -this task easier. - -A flexible array holds an arbitrary (within limits) number of fixed-sized -objects, accessed via an integer index. Sparse arrays are handled -reasonably well. Only single-page allocations are made, so memory -allocation failures should be relatively rare. The down sides are that the -arrays cannot be indexed directly, individual object size cannot exceed the -system page size, and putting data into a flexible array requires a copy -operation. It's also worth noting that flexible arrays do no internal -locking at all; if concurrent access to an array is possible, then the -caller must arrange for appropriate mutual exclusion. - -The creation of a flexible array is done with :c:func:`flex_array_alloc()`:: - - #include - - struct flex_array *flex_array_alloc(int element_size, - unsigned int total, - gfp_t flags); - -The individual object size is provided by ``element_size``, while total is the -maximum number of objects which can be stored in the array. The flags -argument is passed directly to the internal memory allocation calls. With -the current code, using flags to ask for high memory is likely to lead to -notably unpleasant side effects. - -It is also possible to define flexible arrays at compile time with:: - - DEFINE_FLEX_ARRAY(name, element_size, total); - -This macro will result in a definition of an array with the given name; the -element size and total will be checked for validity at compile time. - -Storing data into a flexible array is accomplished with a call to -:c:func:`flex_array_put()`:: - - int flex_array_put(struct flex_array *array, unsigned int element_nr, - void *src, gfp_t flags); - -This call will copy the data from src into the array, in the position -indicated by ``element_nr`` (which must be less than the maximum specified when -the array was created). If any memory allocations must be performed, flags -will be used. The return value is zero on success, a negative error code -otherwise. - -There might possibly be a need to store data into a flexible array while -running in some sort of atomic context; in this situation, sleeping in the -memory allocator would be a bad thing. That can be avoided by using -``GFP_ATOMIC`` for the flags value, but, often, there is a better way. The -trick is to ensure that any needed memory allocations are done before -entering atomic context, using :c:func:`flex_array_prealloc()`:: - - int flex_array_prealloc(struct flex_array *array, unsigned int start, - unsigned int nr_elements, gfp_t flags); - -This function will ensure that memory for the elements indexed in the range -defined by ``start`` and ``nr_elements`` has been allocated. Thereafter, a -``flex_array_put()`` call on an element in that range is guaranteed not to -block. - -Getting data back out of the array is done with :c:func:`flex_array_get()`:: - - void *flex_array_get(struct flex_array *fa, unsigned int element_nr); - -The return value is a pointer to the data element, or NULL if that -particular element has never been allocated. - -Note that it is possible to get back a valid pointer for an element which -has never been stored in the array. Memory for array elements is allocated -one page at a time; a single allocation could provide memory for several -adjacent elements. Flexible array elements are normally initialized to the -value ``FLEX_ARRAY_FREE`` (defined as 0x6c in ), so errors -involving that number probably result from use of unstored array entries. -Note that, if array elements are allocated with ``__GFP_ZERO``, they will be -initialized to zero and this poisoning will not happen. - -Individual elements in the array can be cleared with -:c:func:`flex_array_clear()`:: - - int flex_array_clear(struct flex_array *array, unsigned int element_nr); - -This function will set the given element to ``FLEX_ARRAY_FREE`` and return -zero. If storage for the indicated element is not allocated for the array, -``flex_array_clear()`` will return ``-EINVAL`` instead. Note that clearing an -element does not release the storage associated with it; to reduce the -allocated size of an array, call :c:func:`flex_array_shrink()`:: - - int flex_array_shrink(struct flex_array *array); - -The return value will be the number of pages of memory actually freed. -This function works by scanning the array for pages containing nothing but -``FLEX_ARRAY_FREE`` bytes, so (1) it can be expensive, and (2) it will not work -if the array's pages are allocated with ``__GFP_ZERO``. - -It is possible to remove all elements of an array with a call to -:c:func:`flex_array_free_parts()`:: - - void flex_array_free_parts(struct flex_array *array); - -This call frees all elements, but leaves the array itself in place. -Freeing the entire array is done with :c:func:`flex_array_free()`:: - - void flex_array_free(struct flex_array *array); - -As of this writing, there are no users of flexible arrays in the mainline -kernel. The functions described here are also not exported to modules; -that will probably be fixed when somebody comes up with a need for it. - - -Flexible array functions ------------------------- - -.. kernel-doc:: include/linux/flex_array.h diff --git a/Documentation/flexible-arrays.txt b/Documentation/flexible-arrays.txt deleted file mode 100644 index a0f2989dd804..000000000000 --- a/Documentation/flexible-arrays.txt +++ /dev/null @@ -1,123 +0,0 @@ -=================================== -Using flexible arrays in the kernel -=================================== - -:Updated: Last updated for 2.6.32 -:Author: Jonathan Corbet - -Large contiguous memory allocations can be unreliable in the Linux kernel. -Kernel programmers will sometimes respond to this problem by allocating -pages with vmalloc(). This solution not ideal, though. On 32-bit systems, -memory from vmalloc() must be mapped into a relatively small address space; -it's easy to run out. On SMP systems, the page table changes required by -vmalloc() allocations can require expensive cross-processor interrupts on -all CPUs. And, on all systems, use of space in the vmalloc() range -increases pressure on the translation lookaside buffer (TLB), reducing the -performance of the system. - -In many cases, the need for memory from vmalloc() can be eliminated by -piecing together an array from smaller parts; the flexible array library -exists to make this task easier. - -A flexible array holds an arbitrary (within limits) number of fixed-sized -objects, accessed via an integer index. Sparse arrays are handled -reasonably well. Only single-page allocations are made, so memory -allocation failures should be relatively rare. The down sides are that the -arrays cannot be indexed directly, individual object size cannot exceed the -system page size, and putting data into a flexible array requires a copy -operation. It's also worth noting that flexible arrays do no internal -locking at all; if concurrent access to an array is possible, then the -caller must arrange for appropriate mutual exclusion. - -The creation of a flexible array is done with:: - - #include - - struct flex_array *flex_array_alloc(int element_size, - unsigned int total, - gfp_t flags); - -The individual object size is provided by element_size, while total is the -maximum number of objects which can be stored in the array. The flags -argument is passed directly to the internal memory allocation calls. With -the current code, using flags to ask for high memory is likely to lead to -notably unpleasant side effects. - -It is also possible to define flexible arrays at compile time with:: - - DEFINE_FLEX_ARRAY(name, element_size, total); - -This macro will result in a definition of an array with the given name; the -element size and total will be checked for validity at compile time. - -Storing data into a flexible array is accomplished with a call to:: - - int flex_array_put(struct flex_array *array, unsigned int element_nr, - void *src, gfp_t flags); - -This call will copy the data from src into the array, in the position -indicated by element_nr (which must be less than the maximum specified when -the array was created). If any memory allocations must be performed, flags -will be used. The return value is zero on success, a negative error code -otherwise. - -There might possibly be a need to store data into a flexible array while -running in some sort of atomic context; in this situation, sleeping in the -memory allocator would be a bad thing. That can be avoided by using -GFP_ATOMIC for the flags value, but, often, there is a better way. The -trick is to ensure that any needed memory allocations are done before -entering atomic context, using:: - - int flex_array_prealloc(struct flex_array *array, unsigned int start, - unsigned int nr_elements, gfp_t flags); - -This function will ensure that memory for the elements indexed in the range -defined by start and nr_elements has been allocated. Thereafter, a -flex_array_put() call on an element in that range is guaranteed not to -block. - -Getting data back out of the array is done with:: - - void *flex_array_get(struct flex_array *fa, unsigned int element_nr); - -The return value is a pointer to the data element, or NULL if that -particular element has never been allocated. - -Note that it is possible to get back a valid pointer for an element which -has never been stored in the array. Memory for array elements is allocated -one page at a time; a single allocation could provide memory for several -adjacent elements. Flexible array elements are normally initialized to the -value FLEX_ARRAY_FREE (defined as 0x6c in ), so errors -involving that number probably result from use of unstored array entries. -Note that, if array elements are allocated with __GFP_ZERO, they will be -initialized to zero and this poisoning will not happen. - -Individual elements in the array can be cleared with:: - - int flex_array_clear(struct flex_array *array, unsigned int element_nr); - -This function will set the given element to FLEX_ARRAY_FREE and return -zero. If storage for the indicated element is not allocated for the array, -flex_array_clear() will return -EINVAL instead. Note that clearing an -element does not release the storage associated with it; to reduce the -allocated size of an array, call:: - - int flex_array_shrink(struct flex_array *array); - -The return value will be the number of pages of memory actually freed. -This function works by scanning the array for pages containing nothing but -FLEX_ARRAY_FREE bytes, so (1) it can be expensive, and (2) it will not work -if the array's pages are allocated with __GFP_ZERO. - -It is possible to remove all elements of an array with a call to:: - - void flex_array_free_parts(struct flex_array *array); - -This call frees all elements, but leaves the array itself in place. -Freeing the entire array is done with:: - - void flex_array_free(struct flex_array *array); - -As of this writing, there are no users of flexible arrays in the mainline -kernel. The functions described here are also not exported to modules; -that will probably be fixed when somebody comes up with a need for it. diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h deleted file mode 100644 index b94fa61b51fb..000000000000 --- a/include/linux/flex_array.h +++ /dev/null @@ -1,149 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _FLEX_ARRAY_H -#define _FLEX_ARRAY_H - -#include -#include -#include - -#define FLEX_ARRAY_PART_SIZE PAGE_SIZE -#define FLEX_ARRAY_BASE_SIZE PAGE_SIZE - -struct flex_array_part; - -/* - * This is meant to replace cases where an array-like - * structure has gotten too big to fit into kmalloc() - * and the developer is getting tempted to use - * vmalloc(). - */ - -struct flex_array { - union { - struct { - int element_size; - int total_nr_elements; - int elems_per_part; - struct reciprocal_value reciprocal_elems; - struct flex_array_part *parts[]; - }; - /* - * This little trick makes sure that - * sizeof(flex_array) == PAGE_SIZE - */ - char padding[FLEX_ARRAY_BASE_SIZE]; - }; -}; - -/* Number of bytes left in base struct flex_array, excluding metadata */ -#define FLEX_ARRAY_BASE_BYTES_LEFT \ - (FLEX_ARRAY_BASE_SIZE - offsetof(struct flex_array, parts)) - -/* Number of pointers in base to struct flex_array_part pages */ -#define FLEX_ARRAY_NR_BASE_PTRS \ - (FLEX_ARRAY_BASE_BYTES_LEFT / sizeof(struct flex_array_part *)) - -/* Number of elements of size that fit in struct flex_array_part */ -#define FLEX_ARRAY_ELEMENTS_PER_PART(size) \ - (FLEX_ARRAY_PART_SIZE / size) - -/* - * Defines a statically allocated flex array and ensures its parameters are - * valid. - */ -#define DEFINE_FLEX_ARRAY(__arrayname, __element_size, __total) \ - struct flex_array __arrayname = { { { \ - .element_size = (__element_size), \ - .total_nr_elements = (__total), \ - } } }; \ - static inline void __arrayname##_invalid_parameter(void) \ - { \ - BUILD_BUG_ON((__total) > FLEX_ARRAY_NR_BASE_PTRS * \ - FLEX_ARRAY_ELEMENTS_PER_PART(__element_size)); \ - } - -/** - * flex_array_alloc() - Creates a flexible array. - * @element_size: individual object size. - * @total: maximum number of objects which can be stored. - * @flags: GFP flags - * - * Return: Returns an object of structure flex_array. - */ -struct flex_array *flex_array_alloc(int element_size, unsigned int total, - gfp_t flags); - -/** - * flex_array_prealloc() - Ensures that memory for the elements indexed in the - * range defined by start and nr_elements has been allocated. - * @fa: array to allocate memory to. - * @start: start address - * @nr_elements: number of elements to be allocated. - * @flags: GFP flags - * - */ -int flex_array_prealloc(struct flex_array *fa, unsigned int start, - unsigned int nr_elements, gfp_t flags); - -/** - * flex_array_free() - Removes all elements of a flexible array. - * @fa: array to be freed. - */ -void flex_array_free(struct flex_array *fa); - -/** - * flex_array_free_parts() - Removes all elements of a flexible array, but - * leaves the array itself in place. - * @fa: array to be emptied. - */ -void flex_array_free_parts(struct flex_array *fa); - -/** - * flex_array_put() - Stores data into a flexible array. - * @fa: array where element is to be stored. - * @element_nr: position to copy, must be less than the maximum specified when - * the array was created. - * @src: data source to be copied into the array. - * @flags: GFP flags - * - * Return: Returns zero on success, a negative error code otherwise. - */ -int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, - gfp_t flags); - -/** - * flex_array_clear() - Clears an individual element in the array, sets the - * given element to FLEX_ARRAY_FREE. - * @element_nr: element position to clear. - * @fa: array to which element to be cleared belongs. - * - * Return: Returns zero on success, -EINVAL otherwise. - */ -int flex_array_clear(struct flex_array *fa, unsigned int element_nr); - -/** - * flex_array_get() - Retrieves data into a flexible array. - * - * @element_nr: Element position to retrieve data from. - * @fa: array from which data is to be retrieved. - * - * Return: Returns a pointer to the data element, or NULL if that - * particular element has never been allocated. - */ -void *flex_array_get(struct flex_array *fa, unsigned int element_nr); - -/** - * flex_array_shrink() - Reduces the allocated size of an array. - * @fa: array to shrink. - * - * Return: Returns number of pages of memory actually freed. - * - */ -int flex_array_shrink(struct flex_array *fa); - -#define flex_array_put_ptr(fa, nr, src, gfp) \ - flex_array_put(fa, nr, (void *)&(src), gfp) - -void *flex_array_get_ptr(struct flex_array *fa, unsigned int element_nr); - -#endif /* _FLEX_ARRAY_H */ diff --git a/include/linux/poison.h b/include/linux/poison.h index 5046bad0c1c5..d6d980a681c7 100644 --- a/include/linux/poison.h +++ b/include/linux/poison.h @@ -83,9 +83,6 @@ #define MUTEX_DEBUG_FREE 0x22 #define MUTEX_POISON_WW_CTX ((void *) 0x500 + POISON_POINTER_DELTA) -/********** lib/flex_array.c **********/ -#define FLEX_ARRAY_FREE 0x6c /* for use-after-free poisoning */ - /********** security/ **********/ #define KEY_DESTROY 0xbd diff --git a/lib/Makefile b/lib/Makefile index b798b41d01ae..4e066120a0d6 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -35,7 +35,7 @@ obj-y += lockref.o obj-y += bcd.o div64.o sort.o parser.o debug_locks.o random32.o \ bust_spinlocks.o kasprintf.o bitmap.o scatterlist.o \ - gcd.o lcm.o list_sort.o uuid.o flex_array.o iov_iter.o clz_ctz.o \ + gcd.o lcm.o list_sort.o uuid.o iov_iter.o clz_ctz.o \ bsearch.o find_bit.o llist.o memweight.o kfifo.o \ percpu-refcount.o rhashtable.o reciprocal_div.o \ once.o refcount.o usercopy.o errseq.o bucket_locks.o \ diff --git a/lib/flex_array.c b/lib/flex_array.c deleted file mode 100644 index 2eed22fa507c..000000000000 --- a/lib/flex_array.c +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Flexible array managed in PAGE_SIZE parts - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * - * Copyright IBM Corporation, 2009 - * - * Author: Dave Hansen - */ - -#include -#include -#include -#include -#include - -struct flex_array_part { - char elements[FLEX_ARRAY_PART_SIZE]; -}; - -/* - * If a user requests an allocation which is small - * enough, we may simply use the space in the - * flex_array->parts[] array to store the user - * data. - */ -static inline int elements_fit_in_base(struct flex_array *fa) -{ - int data_size = fa->element_size * fa->total_nr_elements; - if (data_size <= FLEX_ARRAY_BASE_BYTES_LEFT) - return 1; - return 0; -} - -/** - * flex_array_alloc - allocate a new flexible array - * @element_size: the size of individual elements in the array - * @total: total number of elements that this should hold - * @flags: page allocation flags to use for base array - * - * Note: all locking must be provided by the caller. - * - * @total is used to size internal structures. If the user ever - * accesses any array indexes >=@total, it will produce errors. - * - * The maximum number of elements is defined as: the number of - * elements that can be stored in a page times the number of - * page pointers that we can fit in the base structure or (using - * integer math): - * - * (PAGE_SIZE/element_size) * (PAGE_SIZE-8)/sizeof(void *) - * - * Here's a table showing example capacities. Note that the maximum - * index that the get/put() functions is just nr_objects-1. This - * basically means that you get 4MB of storage on 32-bit and 2MB on - * 64-bit. - * - * - * Element size | Objects | Objects | - * PAGE_SIZE=4k | 32-bit | 64-bit | - * ---------------------------------| - * 1 bytes | 4177920 | 2088960 | - * 2 bytes | 2088960 | 1044480 | - * 3 bytes | 1392300 | 696150 | - * 4 bytes | 1044480 | 522240 | - * 32 bytes | 130560 | 65408 | - * 33 bytes | 126480 | 63240 | - * 2048 bytes | 2040 | 1020 | - * 2049 bytes | 1020 | 510 | - * void * | 1044480 | 261120 | - * - * Since 64-bit pointers are twice the size, we lose half the - * capacity in the base structure. Also note that no effort is made - * to efficiently pack objects across page boundaries. - */ -struct flex_array *flex_array_alloc(int element_size, unsigned int total, - gfp_t flags) -{ - struct flex_array *ret; - int elems_per_part = 0; - int max_size = 0; - struct reciprocal_value reciprocal_elems = { 0 }; - - if (element_size) { - elems_per_part = FLEX_ARRAY_ELEMENTS_PER_PART(element_size); - reciprocal_elems = reciprocal_value(elems_per_part); - max_size = FLEX_ARRAY_NR_BASE_PTRS * elems_per_part; - } - - /* max_size will end up 0 if element_size > PAGE_SIZE */ - if (total > max_size) - return NULL; - ret = kzalloc(sizeof(struct flex_array), flags); - if (!ret) - return NULL; - ret->element_size = element_size; - ret->total_nr_elements = total; - ret->elems_per_part = elems_per_part; - ret->reciprocal_elems = reciprocal_elems; - if (elements_fit_in_base(ret) && !(flags & __GFP_ZERO)) - memset(&ret->parts[0], FLEX_ARRAY_FREE, - FLEX_ARRAY_BASE_BYTES_LEFT); - return ret; -} -EXPORT_SYMBOL(flex_array_alloc); - -static int fa_element_to_part_nr(struct flex_array *fa, - unsigned int element_nr) -{ - /* - * if element_size == 0 we don't get here, so we never touch - * the zeroed fa->reciprocal_elems, which would yield invalid - * results - */ - return reciprocal_divide(element_nr, fa->reciprocal_elems); -} - -/** - * flex_array_free_parts - just free the second-level pages - * @fa: the flex array from which to free parts - * - * This is to be used in cases where the base 'struct flex_array' - * has been statically allocated and should not be free. - */ -void flex_array_free_parts(struct flex_array *fa) -{ - int part_nr; - - if (elements_fit_in_base(fa)) - return; - for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++) - kfree(fa->parts[part_nr]); -} -EXPORT_SYMBOL(flex_array_free_parts); - -void flex_array_free(struct flex_array *fa) -{ - flex_array_free_parts(fa); - kfree(fa); -} -EXPORT_SYMBOL(flex_array_free); - -static unsigned int index_inside_part(struct flex_array *fa, - unsigned int element_nr, - unsigned int part_nr) -{ - unsigned int part_offset; - - part_offset = element_nr - part_nr * fa->elems_per_part; - return part_offset * fa->element_size; -} - -static struct flex_array_part * -__fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags) -{ - struct flex_array_part *part = fa->parts[part_nr]; - if (!part) { - part = kmalloc(sizeof(struct flex_array_part), flags); - if (!part) - return NULL; - if (!(flags & __GFP_ZERO)) - memset(part, FLEX_ARRAY_FREE, - sizeof(struct flex_array_part)); - fa->parts[part_nr] = part; - } - return part; -} - -/** - * flex_array_put - copy data into the array at @element_nr - * @fa: the flex array to copy data into - * @element_nr: index of the position in which to insert - * the new element. - * @src: address of data to copy into the array - * @flags: page allocation flags to use for array expansion - * - * - * Note that this *copies* the contents of @src into - * the array. If you are trying to store an array of - * pointers, make sure to pass in &ptr instead of ptr. - * You may instead wish to use the flex_array_put_ptr() - * helper function. - * - * Locking must be provided by the caller. - */ -int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, - gfp_t flags) -{ - int part_nr = 0; - struct flex_array_part *part; - void *dst; - - if (element_nr >= fa->total_nr_elements) - return -ENOSPC; - if (!fa->element_size) - return 0; - if (elements_fit_in_base(fa)) - part = (struct flex_array_part *)&fa->parts[0]; - else { - part_nr = fa_element_to_part_nr(fa, element_nr); - part = __fa_get_part(fa, part_nr, flags); - if (!part) - return -ENOMEM; - } - dst = &part->elements[index_inside_part(fa, element_nr, part_nr)]; - memcpy(dst, src, fa->element_size); - return 0; -} -EXPORT_SYMBOL(flex_array_put); - -/** - * flex_array_clear - clear element in array at @element_nr - * @fa: the flex array of the element. - * @element_nr: index of the position to clear. - * - * Locking must be provided by the caller. - */ -int flex_array_clear(struct flex_array *fa, unsigned int element_nr) -{ - int part_nr = 0; - struct flex_array_part *part; - void *dst; - - if (element_nr >= fa->total_nr_elements) - return -ENOSPC; - if (!fa->element_size) - return 0; - if (elements_fit_in_base(fa)) - part = (struct flex_array_part *)&fa->parts[0]; - else { - part_nr = fa_element_to_part_nr(fa, element_nr); - part = fa->parts[part_nr]; - if (!part) - return -EINVAL; - } - dst = &part->elements[index_inside_part(fa, element_nr, part_nr)]; - memset(dst, FLEX_ARRAY_FREE, fa->element_size); - return 0; -} -EXPORT_SYMBOL(flex_array_clear); - -/** - * flex_array_prealloc - guarantee that array space exists - * @fa: the flex array for which to preallocate parts - * @start: index of first array element for which space is allocated - * @nr_elements: number of elements for which space is allocated - * @flags: page allocation flags - * - * This will guarantee that no future calls to flex_array_put() - * will allocate memory. It can be used if you are expecting to - * be holding a lock or in some atomic context while writing - * data into the array. - * - * Locking must be provided by the caller. - */ -int flex_array_prealloc(struct flex_array *fa, unsigned int start, - unsigned int nr_elements, gfp_t flags) -{ - int start_part; - int end_part; - int part_nr; - unsigned int end; - struct flex_array_part *part; - - if (!start && !nr_elements) - return 0; - if (start >= fa->total_nr_elements) - return -ENOSPC; - if (!nr_elements) - return 0; - - end = start + nr_elements - 1; - - if (end >= fa->total_nr_elements) - return -ENOSPC; - if (!fa->element_size) - return 0; - if (elements_fit_in_base(fa)) - return 0; - start_part = fa_element_to_part_nr(fa, start); - end_part = fa_element_to_part_nr(fa, end); - for (part_nr = start_part; part_nr <= end_part; part_nr++) { - part = __fa_get_part(fa, part_nr, flags); - if (!part) - return -ENOMEM; - } - return 0; -} -EXPORT_SYMBOL(flex_array_prealloc); - -/** - * flex_array_get - pull data back out of the array - * @fa: the flex array from which to extract data - * @element_nr: index of the element to fetch from the array - * - * Returns a pointer to the data at index @element_nr. Note - * that this is a copy of the data that was passed in. If you - * are using this to store pointers, you'll get back &ptr. You - * may instead wish to use the flex_array_get_ptr helper. - * - * Locking must be provided by the caller. - */ -void *flex_array_get(struct flex_array *fa, unsigned int element_nr) -{ - int part_nr = 0; - struct flex_array_part *part; - - if (!fa->element_size) - return NULL; - if (element_nr >= fa->total_nr_elements) - return NULL; - if (elements_fit_in_base(fa)) - part = (struct flex_array_part *)&fa->parts[0]; - else { - part_nr = fa_element_to_part_nr(fa, element_nr); - part = fa->parts[part_nr]; - if (!part) - return NULL; - } - return &part->elements[index_inside_part(fa, element_nr, part_nr)]; -} -EXPORT_SYMBOL(flex_array_get); - -/** - * flex_array_get_ptr - pull a ptr back out of the array - * @fa: the flex array from which to extract data - * @element_nr: index of the element to fetch from the array - * - * Returns the pointer placed in the flex array at element_nr using - * flex_array_put_ptr(). This function should not be called if the - * element in question was not set using the _put_ptr() helper. - */ -void *flex_array_get_ptr(struct flex_array *fa, unsigned int element_nr) -{ - void **tmp; - - tmp = flex_array_get(fa, element_nr); - if (!tmp) - return NULL; - - return *tmp; -} -EXPORT_SYMBOL(flex_array_get_ptr); - -static int part_is_free(struct flex_array_part *part) -{ - int i; - - for (i = 0; i < sizeof(struct flex_array_part); i++) - if (part->elements[i] != FLEX_ARRAY_FREE) - return 0; - return 1; -} - -/** - * flex_array_shrink - free unused second-level pages - * @fa: the flex array to shrink - * - * Frees all second-level pages that consist solely of unused - * elements. Returns the number of pages freed. - * - * Locking must be provided by the caller. - */ -int flex_array_shrink(struct flex_array *fa) -{ - struct flex_array_part *part; - int part_nr; - int ret = 0; - - if (!fa->total_nr_elements || !fa->element_size) - return 0; - if (elements_fit_in_base(fa)) - return ret; - for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++) { - part = fa->parts[part_nr]; - if (!part) - continue; - if (part_is_free(part)) { - fa->parts[part_nr] = NULL; - kfree(part); - ret++; - } - } - return ret; -} -EXPORT_SYMBOL(flex_array_shrink); diff --git a/tools/include/linux/poison.h b/tools/include/linux/poison.h index 9fdcd3eaac3b..d29725769107 100644 --- a/tools/include/linux/poison.h +++ b/tools/include/linux/poison.h @@ -87,9 +87,6 @@ #define MUTEX_DEBUG_INIT 0x11 #define MUTEX_DEBUG_FREE 0x22 -/********** lib/flex_array.c **********/ -#define FLEX_ARRAY_FREE 0x6c /* for use-after-free poisoning */ - /********** security/ **********/ #define KEY_DESTROY 0xbd