2008-01-30 15:30:37 +03:00
|
|
|
/*
|
2005-04-17 02:20:36 +04:00
|
|
|
* Generic VM initialization for x86-64 NUMA setups.
|
|
|
|
* Copyright 2002,2003 Andi Kleen, SuSE Labs.
|
2008-01-30 15:30:37 +03:00
|
|
|
*/
|
2005-04-17 02:20:36 +04:00
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/bootmem.h>
|
2010-08-26 00:39:17 +04:00
|
|
|
#include <linux/memblock.h>
|
2005-04-17 02:20:36 +04:00
|
|
|
#include <linux/mmzone.h>
|
|
|
|
#include <linux/ctype.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/nodemask.h>
|
2008-01-30 15:33:11 +03:00
|
|
|
#include <linux/sched.h>
|
2005-04-17 02:20:36 +04:00
|
|
|
|
|
|
|
#include <asm/e820.h>
|
|
|
|
#include <asm/proto.h>
|
|
|
|
#include <asm/dma.h>
|
|
|
|
#include <asm/numa.h>
|
|
|
|
#include <asm/acpi.h>
|
2010-09-17 20:03:43 +04:00
|
|
|
#include <asm/amd_nb.h>
|
2005-04-17 02:20:36 +04:00
|
|
|
|
2005-09-07 02:17:45 +04:00
|
|
|
struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
|
2008-01-30 15:30:37 +03:00
|
|
|
EXPORT_SYMBOL(node_data);
|
|
|
|
|
2006-03-25 18:31:46 +03:00
|
|
|
struct memnode memnode;
|
2005-04-17 02:20:36 +04:00
|
|
|
|
2008-01-30 15:33:25 +03:00
|
|
|
s16 apicid_to_node[MAX_LOCAL_APIC] __cpuinitdata = {
|
2008-01-30 15:30:37 +03:00
|
|
|
[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
|
2005-09-12 20:49:24 +04:00
|
|
|
};
|
2008-01-30 15:30:37 +03:00
|
|
|
|
2005-04-17 02:20:36 +04:00
|
|
|
int numa_off __initdata;
|
2008-05-12 17:43:36 +04:00
|
|
|
static unsigned long __initdata nodemap_addr;
|
|
|
|
static unsigned long __initdata nodemap_size;
|
2005-04-17 02:20:36 +04:00
|
|
|
|
2009-01-27 06:56:47 +03:00
|
|
|
/*
|
|
|
|
* Map cpu index to node index
|
|
|
|
*/
|
|
|
|
DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
|
|
|
|
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
|
|
|
|
|
2005-11-05 19:25:54 +03:00
|
|
|
/*
|
|
|
|
* Given a shift value, try to populate memnodemap[]
|
|
|
|
* Returns :
|
|
|
|
* 1 if OK
|
|
|
|
* 0 if memnodmap[] too small (of shift too small)
|
|
|
|
* -1 if node overlap or lost ram (shift too big)
|
|
|
|
*/
|
2008-01-30 15:30:37 +03:00
|
|
|
static int __init populate_memnodemap(const struct bootnode *nodes,
|
2008-03-25 20:14:35 +03:00
|
|
|
int numnodes, int shift, int *nodeids)
|
2005-04-17 02:20:36 +04:00
|
|
|
{
|
2005-11-05 19:25:54 +03:00
|
|
|
unsigned long addr, end;
|
2008-01-30 15:30:37 +03:00
|
|
|
int i, res = -1;
|
2005-07-29 08:15:38 +04:00
|
|
|
|
2008-01-30 15:33:25 +03:00
|
|
|
memset(memnodemap, 0xff, sizeof(s16)*memnodemapsize);
|
2005-07-29 08:15:38 +04:00
|
|
|
for (i = 0; i < numnodes; i++) {
|
2005-11-05 19:25:54 +03:00
|
|
|
addr = nodes[i].start;
|
|
|
|
end = nodes[i].end;
|
|
|
|
if (addr >= end)
|
2005-07-29 08:15:38 +04:00
|
|
|
continue;
|
2007-02-13 15:26:19 +03:00
|
|
|
if ((end >> shift) >= memnodemapsize)
|
2005-11-05 19:25:54 +03:00
|
|
|
return 0;
|
|
|
|
do {
|
2008-01-30 15:33:25 +03:00
|
|
|
if (memnodemap[addr >> shift] != NUMA_NO_NODE)
|
2005-07-29 08:15:38 +04:00
|
|
|
return -1;
|
2008-03-25 20:14:35 +03:00
|
|
|
|
|
|
|
if (!nodeids)
|
|
|
|
memnodemap[addr >> shift] = i;
|
|
|
|
else
|
|
|
|
memnodemap[addr >> shift] = nodeids[i];
|
|
|
|
|
2007-02-13 15:26:19 +03:00
|
|
|
addr += (1UL << shift);
|
2005-11-05 19:25:54 +03:00
|
|
|
} while (addr < end);
|
|
|
|
res = 1;
|
2008-01-30 15:30:37 +03:00
|
|
|
}
|
2005-11-05 19:25:54 +03:00
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2007-02-13 15:26:19 +03:00
|
|
|
static int __init allocate_cachealigned_memnodemap(void)
|
|
|
|
{
|
2008-02-01 19:49:41 +03:00
|
|
|
unsigned long addr;
|
2007-02-13 15:26:19 +03:00
|
|
|
|
|
|
|
memnodemap = memnode.embedded_map;
|
2008-01-30 15:33:15 +03:00
|
|
|
if (memnodemapsize <= ARRAY_SIZE(memnode.embedded_map))
|
2007-02-13 15:26:19 +03:00
|
|
|
return 0;
|
|
|
|
|
2008-02-01 19:49:41 +03:00
|
|
|
addr = 0x8000;
|
2008-07-25 18:48:58 +04:00
|
|
|
nodemap_size = roundup(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES);
|
2010-08-26 00:39:17 +04:00
|
|
|
nodemap_addr = memblock_find_in_range(addr, max_pfn<<PAGE_SHIFT,
|
2008-02-01 19:49:41 +03:00
|
|
|
nodemap_size, L1_CACHE_BYTES);
|
2010-08-26 00:39:17 +04:00
|
|
|
if (nodemap_addr == MEMBLOCK_ERROR) {
|
2007-02-13 15:26:19 +03:00
|
|
|
printk(KERN_ERR
|
|
|
|
"NUMA: Unable to allocate Memory to Node hash map\n");
|
|
|
|
nodemap_addr = nodemap_size = 0;
|
|
|
|
return -1;
|
|
|
|
}
|
2008-02-01 19:49:41 +03:00
|
|
|
memnodemap = phys_to_virt(nodemap_addr);
|
2010-08-26 00:39:17 +04:00
|
|
|
memblock_x86_reserve_range(nodemap_addr, nodemap_addr + nodemap_size, "MEMNODEMAP");
|
2007-02-13 15:26:19 +03:00
|
|
|
|
|
|
|
printk(KERN_DEBUG "NUMA: Allocated memnodemap from %lx - %lx\n",
|
|
|
|
nodemap_addr, nodemap_addr + nodemap_size);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The LSB of all start and end addresses in the node map is the value of the
|
|
|
|
* maximum possible shift.
|
|
|
|
*/
|
2008-01-30 15:30:37 +03:00
|
|
|
static int __init extract_lsb_from_nodes(const struct bootnode *nodes,
|
|
|
|
int numnodes)
|
2005-11-05 19:25:54 +03:00
|
|
|
{
|
2007-02-13 15:26:20 +03:00
|
|
|
int i, nodes_used = 0;
|
2007-02-13 15:26:19 +03:00
|
|
|
unsigned long start, end;
|
|
|
|
unsigned long bitfield = 0, memtop = 0;
|
|
|
|
|
|
|
|
for (i = 0; i < numnodes; i++) {
|
|
|
|
start = nodes[i].start;
|
|
|
|
end = nodes[i].end;
|
|
|
|
if (start >= end)
|
|
|
|
continue;
|
2007-02-13 15:26:20 +03:00
|
|
|
bitfield |= start;
|
|
|
|
nodes_used++;
|
2007-02-13 15:26:19 +03:00
|
|
|
if (end > memtop)
|
|
|
|
memtop = end;
|
|
|
|
}
|
2007-02-13 15:26:20 +03:00
|
|
|
if (nodes_used <= 1)
|
|
|
|
i = 63;
|
|
|
|
else
|
|
|
|
i = find_first_bit(&bitfield, sizeof(unsigned long)*8);
|
2007-02-13 15:26:19 +03:00
|
|
|
memnodemapsize = (memtop >> i)+1;
|
|
|
|
return i;
|
|
|
|
}
|
2005-11-05 19:25:54 +03:00
|
|
|
|
2008-03-25 20:14:35 +03:00
|
|
|
int __init compute_hash_shift(struct bootnode *nodes, int numnodes,
|
|
|
|
int *nodeids)
|
2007-02-13 15:26:19 +03:00
|
|
|
{
|
|
|
|
int shift;
|
2005-11-05 19:25:54 +03:00
|
|
|
|
2007-02-13 15:26:19 +03:00
|
|
|
shift = extract_lsb_from_nodes(nodes, numnodes);
|
|
|
|
if (allocate_cachealigned_memnodemap())
|
|
|
|
return -1;
|
2006-01-12 00:44:33 +03:00
|
|
|
printk(KERN_DEBUG "NUMA: Using %d for the hash shift.\n",
|
2005-11-05 19:25:54 +03:00
|
|
|
shift);
|
|
|
|
|
2008-03-25 20:14:35 +03:00
|
|
|
if (populate_memnodemap(nodes, numnodes, shift, nodeids) != 1) {
|
2008-01-30 15:30:37 +03:00
|
|
|
printk(KERN_INFO "Your memory is not aligned you need to "
|
|
|
|
"rebuild your kernel with a bigger NODEMAPSIZE "
|
|
|
|
"shift=%d\n", shift);
|
2005-11-05 19:25:54 +03:00
|
|
|
return -1;
|
|
|
|
}
|
2005-07-29 08:15:38 +04:00
|
|
|
return shift;
|
2005-04-17 02:20:36 +04:00
|
|
|
}
|
|
|
|
|
mm: clean up for early_pfn_to_nid()
What's happening is that the assertion in mm/page_alloc.c:move_freepages()
is triggering:
BUG_ON(page_zone(start_page) != page_zone(end_page));
Once I knew this is what was happening, I added some annotations:
if (unlikely(page_zone(start_page) != page_zone(end_page))) {
printk(KERN_ERR "move_freepages: Bogus zones: "
"start_page[%p] end_page[%p] zone[%p]\n",
start_page, end_page, zone);
printk(KERN_ERR "move_freepages: "
"start_zone[%p] end_zone[%p]\n",
page_zone(start_page), page_zone(end_page));
printk(KERN_ERR "move_freepages: "
"start_pfn[0x%lx] end_pfn[0x%lx]\n",
page_to_pfn(start_page), page_to_pfn(end_page));
printk(KERN_ERR "move_freepages: "
"start_nid[%d] end_nid[%d]\n",
page_to_nid(start_page), page_to_nid(end_page));
...
And here's what I got:
move_freepages: Bogus zones: start_page[2207d0000] end_page[2207dffc0] zone[fffff8103effcb00]
move_freepages: start_zone[fffff8103effcb00] end_zone[fffff8003fffeb00]
move_freepages: start_pfn[0x81f600] end_pfn[0x81f7ff]
move_freepages: start_nid[1] end_nid[0]
My memory layout on this box is:
[ 0.000000] Zone PFN ranges:
[ 0.000000] Normal 0x00000000 -> 0x0081ff5d
[ 0.000000] Movable zone start PFN for each node
[ 0.000000] early_node_map[8] active PFN ranges
[ 0.000000] 0: 0x00000000 -> 0x00020000
[ 0.000000] 1: 0x00800000 -> 0x0081f7ff
[ 0.000000] 1: 0x0081f800 -> 0x0081fe50
[ 0.000000] 1: 0x0081fed1 -> 0x0081fed8
[ 0.000000] 1: 0x0081feda -> 0x0081fedb
[ 0.000000] 1: 0x0081fedd -> 0x0081fee5
[ 0.000000] 1: 0x0081fee7 -> 0x0081ff51
[ 0.000000] 1: 0x0081ff59 -> 0x0081ff5d
So it's a block move in that 0x81f600-->0x81f7ff region which triggers
the problem.
This patch:
Declaration of early_pfn_to_nid() is scattered over per-arch include
files, and it seems it's complicated to know when the declaration is used.
I think it makes fix-for-memmap-init not easy.
This patch moves all declaration to include/linux/mm.h
After this,
if !CONFIG_NODES_POPULATES_NODE_MAP && !CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
-> Use static definition in include/linux/mm.h
else if !CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
-> Use generic definition in mm/page_alloc.c
else
-> per-arch back end function will be called.
Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Tested-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Reported-by: David Miller <davem@davemlloft.net>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: <stable@kernel.org> [2.6.25.x, 2.6.26.x, 2.6.27.x, 2.6.28.x]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-02-19 01:48:32 +03:00
|
|
|
int __meminit __early_pfn_to_nid(unsigned long pfn)
|
2005-06-23 11:08:07 +04:00
|
|
|
{
|
|
|
|
return phys_to_nid(pfn << PAGE_SHIFT);
|
|
|
|
}
|
|
|
|
|
2008-01-30 15:30:37 +03:00
|
|
|
static void * __init early_node_mem(int nodeid, unsigned long start,
|
2008-02-01 19:49:41 +03:00
|
|
|
unsigned long end, unsigned long size,
|
|
|
|
unsigned long align)
|
2006-04-07 21:49:21 +04:00
|
|
|
{
|
2010-02-10 12:20:18 +03:00
|
|
|
unsigned long mem;
|
2008-01-30 15:30:37 +03:00
|
|
|
|
2010-02-10 12:20:18 +03:00
|
|
|
/*
|
|
|
|
* put it on high as possible
|
|
|
|
* something will go with NODE_DATA
|
|
|
|
*/
|
|
|
|
if (start < (MAX_DMA_PFN<<PAGE_SHIFT))
|
|
|
|
start = MAX_DMA_PFN<<PAGE_SHIFT;
|
|
|
|
if (start < (MAX_DMA32_PFN<<PAGE_SHIFT) &&
|
|
|
|
end > (MAX_DMA32_PFN<<PAGE_SHIFT))
|
|
|
|
start = MAX_DMA32_PFN<<PAGE_SHIFT;
|
2010-08-26 00:39:17 +04:00
|
|
|
mem = memblock_x86_find_in_range_node(nodeid, start, end, size, align);
|
|
|
|
if (mem != MEMBLOCK_ERROR)
|
2006-04-07 21:49:21 +04:00
|
|
|
return __va(mem);
|
2008-02-01 19:49:42 +03:00
|
|
|
|
2010-02-10 12:20:18 +03:00
|
|
|
/* extend the search scope */
|
|
|
|
end = max_pfn_mapped << PAGE_SHIFT;
|
2010-10-28 20:50:17 +04:00
|
|
|
start = MAX_DMA_PFN << PAGE_SHIFT;
|
|
|
|
mem = memblock_find_in_range(start, end, size, align);
|
2010-08-26 00:39:17 +04:00
|
|
|
if (mem != MEMBLOCK_ERROR)
|
2006-04-07 21:49:21 +04:00
|
|
|
return __va(mem);
|
2008-02-01 19:49:42 +03:00
|
|
|
|
2010-02-10 12:20:15 +03:00
|
|
|
printk(KERN_ERR "Cannot find %lu bytes in node %d\n",
|
2008-01-30 15:30:37 +03:00
|
|
|
size, nodeid);
|
2010-02-10 12:20:15 +03:00
|
|
|
|
|
|
|
return NULL;
|
2006-04-07 21:49:21 +04:00
|
|
|
}
|
|
|
|
|
2005-04-17 02:20:36 +04:00
|
|
|
/* Initialize bootmem allocator for a node */
|
2009-05-16 00:59:37 +04:00
|
|
|
void __init
|
|
|
|
setup_node_bootmem(int nodeid, unsigned long start, unsigned long end)
|
2008-01-30 15:30:37 +03:00
|
|
|
{
|
2010-02-10 12:20:20 +03:00
|
|
|
unsigned long start_pfn, last_pfn, nodedata_phys;
|
2009-05-16 00:59:37 +04:00
|
|
|
const int pgdat_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
|
2008-03-18 22:52:37 +03:00
|
|
|
int nid;
|
2005-04-17 02:20:36 +04:00
|
|
|
|
2009-04-23 01:19:27 +04:00
|
|
|
if (!end)
|
|
|
|
return;
|
|
|
|
|
2009-05-16 00:59:37 +04:00
|
|
|
/*
|
|
|
|
* Don't confuse VM with a node that doesn't have the
|
|
|
|
* minimum amount of memory:
|
|
|
|
*/
|
|
|
|
if (end && (end - start) < NODE_MIN_SIZE)
|
|
|
|
return;
|
|
|
|
|
2008-07-25 18:48:58 +04:00
|
|
|
start = roundup(start, ZONE_ALIGN);
|
2005-04-17 02:20:36 +04:00
|
|
|
|
2010-02-10 12:20:20 +03:00
|
|
|
printk(KERN_INFO "Initmem setup node %d %016lx-%016lx\n", nodeid,
|
2008-01-30 15:30:37 +03:00
|
|
|
start, end);
|
2005-04-17 02:20:36 +04:00
|
|
|
|
|
|
|
start_pfn = start >> PAGE_SHIFT;
|
2008-05-12 17:43:36 +04:00
|
|
|
last_pfn = end >> PAGE_SHIFT;
|
2005-04-17 02:20:36 +04:00
|
|
|
|
2008-02-01 19:49:41 +03:00
|
|
|
node_data[nodeid] = early_node_mem(nodeid, start, end, pgdat_size,
|
|
|
|
SMP_CACHE_BYTES);
|
2006-04-07 21:49:21 +04:00
|
|
|
if (node_data[nodeid] == NULL)
|
|
|
|
return;
|
|
|
|
nodedata_phys = __pa(node_data[nodeid]);
|
2010-08-26 00:39:17 +04:00
|
|
|
memblock_x86_reserve_range(nodedata_phys, nodedata_phys + pgdat_size, "NODE_DATA");
|
2008-02-04 18:47:56 +03:00
|
|
|
printk(KERN_INFO " NODE_DATA [%016lx - %016lx]\n", nodedata_phys,
|
|
|
|
nodedata_phys + pgdat_size - 1);
|
2010-02-10 12:20:15 +03:00
|
|
|
nid = phys_to_nid(nodedata_phys);
|
|
|
|
if (nid != nodeid)
|
|
|
|
printk(KERN_INFO " NODE_DATA(%d) on node %d\n", nodeid, nid);
|
2005-04-17 02:20:36 +04:00
|
|
|
|
|
|
|
memset(NODE_DATA(nodeid), 0, sizeof(pg_data_t));
|
2010-02-10 12:20:20 +03:00
|
|
|
NODE_DATA(nodeid)->node_id = nodeid;
|
2005-04-17 02:20:36 +04:00
|
|
|
NODE_DATA(nodeid)->node_start_pfn = start_pfn;
|
2008-05-12 17:43:36 +04:00
|
|
|
NODE_DATA(nodeid)->node_spanned_pages = last_pfn - start_pfn;
|
2005-04-17 02:20:36 +04:00
|
|
|
|
|
|
|
node_set_online(nodeid);
|
2008-01-30 15:30:37 +03:00
|
|
|
}
|
2005-04-17 02:20:36 +04:00
|
|
|
|
2008-01-30 15:30:37 +03:00
|
|
|
/*
|
|
|
|
* There are unfortunately some poorly designed mainboards around that
|
|
|
|
* only connect memory to a single CPU. This breaks the 1:1 cpu->node
|
|
|
|
* mapping. To avoid this fill in the mapping for all possible CPUs,
|
|
|
|
* as the number of CPUs is not known yet. We round robin the existing
|
|
|
|
* nodes.
|
|
|
|
*/
|
2005-04-17 02:20:36 +04:00
|
|
|
void __init numa_init_array(void)
|
|
|
|
{
|
|
|
|
int rr, i;
|
2008-01-30 15:30:37 +03:00
|
|
|
|
2005-09-30 22:59:22 +04:00
|
|
|
rr = first_node(node_online_map);
|
2008-12-17 04:34:01 +03:00
|
|
|
for (i = 0; i < nr_cpu_ids; i++) {
|
2008-01-30 15:33:33 +03:00
|
|
|
if (early_cpu_to_node(i) != NUMA_NO_NODE)
|
2005-04-17 02:20:36 +04:00
|
|
|
continue;
|
2008-01-30 15:30:37 +03:00
|
|
|
numa_set_node(i, rr);
|
2005-04-17 02:20:36 +04:00
|
|
|
rr = next_node(rr, node_online_map);
|
|
|
|
if (rr == MAX_NUMNODES)
|
|
|
|
rr = first_node(node_online_map);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_NUMA_EMU
|
2007-02-13 15:26:22 +03:00
|
|
|
/* Numa emulation */
|
2009-09-26 02:20:09 +04:00
|
|
|
static struct bootnode nodes[MAX_NUMNODES] __initdata;
|
|
|
|
static struct bootnode physnodes[MAX_NUMNODES] __initdata;
|
2008-05-12 17:43:36 +04:00
|
|
|
static char *cmdline __initdata;
|
2005-04-17 02:20:36 +04:00
|
|
|
|
2009-09-26 02:20:09 +04:00
|
|
|
static int __init setup_physnodes(unsigned long start, unsigned long end,
|
|
|
|
int acpi, int k8)
|
|
|
|
{
|
|
|
|
int nr_nodes = 0;
|
|
|
|
int ret = 0;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
#ifdef CONFIG_ACPI_NUMA
|
|
|
|
if (acpi)
|
|
|
|
nr_nodes = acpi_get_nodes(physnodes);
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_K8_NUMA
|
|
|
|
if (k8)
|
|
|
|
nr_nodes = k8_get_nodes(physnodes);
|
|
|
|
#endif
|
|
|
|
/*
|
|
|
|
* Basic sanity checking on the physical node map: there may be errors
|
|
|
|
* if the SRAT or K8 incorrectly reported the topology or the mem=
|
|
|
|
* kernel parameter is used.
|
|
|
|
*/
|
|
|
|
for (i = 0; i < nr_nodes; i++) {
|
|
|
|
if (physnodes[i].start == physnodes[i].end)
|
|
|
|
continue;
|
|
|
|
if (physnodes[i].start > end) {
|
|
|
|
physnodes[i].end = physnodes[i].start;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (physnodes[i].end < start) {
|
|
|
|
physnodes[i].start = physnodes[i].end;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (physnodes[i].start < start)
|
|
|
|
physnodes[i].start = start;
|
|
|
|
if (physnodes[i].end > end)
|
|
|
|
physnodes[i].end = end;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Remove all nodes that have no memory or were truncated because of the
|
|
|
|
* limited address range.
|
|
|
|
*/
|
|
|
|
for (i = 0; i < nr_nodes; i++) {
|
|
|
|
if (physnodes[i].start == physnodes[i].end)
|
|
|
|
continue;
|
|
|
|
physnodes[ret].start = physnodes[i].start;
|
|
|
|
physnodes[ret].end = physnodes[i].end;
|
|
|
|
ret++;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If no physical topology was detected, a single node is faked to cover
|
|
|
|
* the entire address space.
|
|
|
|
*/
|
|
|
|
if (!ret) {
|
|
|
|
physnodes[ret].start = start;
|
|
|
|
physnodes[ret].end = end;
|
|
|
|
ret = 1;
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2007-02-13 15:26:22 +03:00
|
|
|
/*
|
2008-01-30 15:30:37 +03:00
|
|
|
* Setups up nid to range from addr to addr + size. If the end
|
|
|
|
* boundary is greater than max_addr, then max_addr is used instead.
|
|
|
|
* The return value is 0 if there is additional memory left for
|
|
|
|
* allocation past addr and -1 otherwise. addr is adjusted to be at
|
|
|
|
* the end of the node.
|
2007-02-13 15:26:22 +03:00
|
|
|
*/
|
2009-09-26 02:20:09 +04:00
|
|
|
static int __init setup_node_range(int nid, u64 *addr, u64 size, u64 max_addr)
|
2007-02-13 15:26:22 +03:00
|
|
|
{
|
2007-05-02 21:27:09 +04:00
|
|
|
int ret = 0;
|
|
|
|
nodes[nid].start = *addr;
|
|
|
|
*addr += size;
|
|
|
|
if (*addr >= max_addr) {
|
|
|
|
*addr = max_addr;
|
|
|
|
ret = -1;
|
|
|
|
}
|
|
|
|
nodes[nid].end = *addr;
|
2007-05-02 21:27:20 +04:00
|
|
|
node_set(nid, node_possible_map);
|
2007-05-02 21:27:09 +04:00
|
|
|
printk(KERN_INFO "Faking node %d at %016Lx-%016Lx (%LuMB)\n", nid,
|
|
|
|
nodes[nid].start, nodes[nid].end,
|
|
|
|
(nodes[nid].end - nodes[nid].start) >> 20);
|
|
|
|
return ret;
|
2007-02-13 15:26:22 +03:00
|
|
|
}
|
|
|
|
|
2009-09-26 02:20:09 +04:00
|
|
|
/*
|
|
|
|
* Sets up nr_nodes fake nodes interleaved over physical nodes ranging from addr
|
|
|
|
* to max_addr. The return value is the number of nodes allocated.
|
|
|
|
*/
|
|
|
|
static int __init split_nodes_interleave(u64 addr, u64 max_addr,
|
|
|
|
int nr_phys_nodes, int nr_nodes)
|
|
|
|
{
|
|
|
|
nodemask_t physnode_mask = NODE_MASK_NONE;
|
|
|
|
u64 size;
|
|
|
|
int big;
|
|
|
|
int ret = 0;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (nr_nodes <= 0)
|
|
|
|
return -1;
|
|
|
|
if (nr_nodes > MAX_NUMNODES) {
|
|
|
|
pr_info("numa=fake=%d too large, reducing to %d\n",
|
|
|
|
nr_nodes, MAX_NUMNODES);
|
|
|
|
nr_nodes = MAX_NUMNODES;
|
|
|
|
}
|
|
|
|
|
2010-08-26 00:39:17 +04:00
|
|
|
size = (max_addr - addr - memblock_x86_hole_size(addr, max_addr)) / nr_nodes;
|
2009-09-26 02:20:09 +04:00
|
|
|
/*
|
|
|
|
* Calculate the number of big nodes that can be allocated as a result
|
|
|
|
* of consolidating the remainder.
|
|
|
|
*/
|
2010-02-16 00:43:25 +03:00
|
|
|
big = ((size & ~FAKE_NODE_MIN_HASH_MASK) * nr_nodes) /
|
2009-09-26 02:20:09 +04:00
|
|
|
FAKE_NODE_MIN_SIZE;
|
|
|
|
|
|
|
|
size &= FAKE_NODE_MIN_HASH_MASK;
|
|
|
|
if (!size) {
|
|
|
|
pr_err("Not enough memory for each node. "
|
|
|
|
"NUMA emulation disabled.\n");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < nr_phys_nodes; i++)
|
|
|
|
if (physnodes[i].start != physnodes[i].end)
|
|
|
|
node_set(i, physnode_mask);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Continue to fill physical nodes with fake nodes until there is no
|
|
|
|
* memory left on any of them.
|
|
|
|
*/
|
|
|
|
while (nodes_weight(physnode_mask)) {
|
|
|
|
for_each_node_mask(i, physnode_mask) {
|
|
|
|
u64 end = physnodes[i].start + size;
|
|
|
|
u64 dma32_end = PFN_PHYS(MAX_DMA32_PFN);
|
|
|
|
|
|
|
|
if (ret < big)
|
|
|
|
end += FAKE_NODE_MIN_SIZE;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Continue to add memory to this fake node if its
|
|
|
|
* non-reserved memory is less than the per-node size.
|
|
|
|
*/
|
|
|
|
while (end - physnodes[i].start -
|
2010-08-26 00:39:17 +04:00
|
|
|
memblock_x86_hole_size(physnodes[i].start, end) < size) {
|
2009-09-26 02:20:09 +04:00
|
|
|
end += FAKE_NODE_MIN_SIZE;
|
|
|
|
if (end > physnodes[i].end) {
|
|
|
|
end = physnodes[i].end;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there won't be at least FAKE_NODE_MIN_SIZE of
|
|
|
|
* non-reserved memory in ZONE_DMA32 for the next node,
|
|
|
|
* this one must extend to the boundary.
|
|
|
|
*/
|
|
|
|
if (end < dma32_end && dma32_end - end -
|
2010-08-26 00:39:17 +04:00
|
|
|
memblock_x86_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE)
|
2009-09-26 02:20:09 +04:00
|
|
|
end = dma32_end;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there won't be enough non-reserved memory for the
|
|
|
|
* next node, this one must extend to the end of the
|
|
|
|
* physical node.
|
|
|
|
*/
|
|
|
|
if (physnodes[i].end - end -
|
2010-08-26 00:39:17 +04:00
|
|
|
memblock_x86_hole_size(end, physnodes[i].end) < size)
|
2009-09-26 02:20:09 +04:00
|
|
|
end = physnodes[i].end;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Avoid allocating more nodes than requested, which can
|
|
|
|
* happen as a result of rounding down each node's size
|
|
|
|
* to FAKE_NODE_MIN_SIZE.
|
|
|
|
*/
|
|
|
|
if (nodes_weight(physnode_mask) + ret >= nr_nodes)
|
|
|
|
end = physnodes[i].end;
|
|
|
|
|
|
|
|
if (setup_node_range(ret++, &physnodes[i].start,
|
|
|
|
end - physnodes[i].start,
|
|
|
|
physnodes[i].end) < 0)
|
|
|
|
node_clear(i, physnode_mask);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2010-02-16 00:43:30 +03:00
|
|
|
/*
|
|
|
|
* Returns the end address of a node so that there is at least `size' amount of
|
|
|
|
* non-reserved memory or `max_addr' is reached.
|
|
|
|
*/
|
|
|
|
static u64 __init find_end_of_node(u64 start, u64 max_addr, u64 size)
|
|
|
|
{
|
|
|
|
u64 end = start + size;
|
|
|
|
|
2010-08-26 00:39:17 +04:00
|
|
|
while (end - start - memblock_x86_hole_size(start, end) < size) {
|
2010-02-16 00:43:30 +03:00
|
|
|
end += FAKE_NODE_MIN_SIZE;
|
|
|
|
if (end > max_addr) {
|
|
|
|
end = max_addr;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return end;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Sets up fake nodes of `size' interleaved over physical nodes ranging from
|
|
|
|
* `addr' to `max_addr'. The return value is the number of nodes allocated.
|
|
|
|
*/
|
|
|
|
static int __init split_nodes_size_interleave(u64 addr, u64 max_addr, u64 size)
|
|
|
|
{
|
|
|
|
nodemask_t physnode_mask = NODE_MASK_NONE;
|
|
|
|
u64 min_size;
|
|
|
|
int ret = 0;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!size)
|
|
|
|
return -1;
|
|
|
|
/*
|
|
|
|
* The limit on emulated nodes is MAX_NUMNODES, so the size per node is
|
|
|
|
* increased accordingly if the requested size is too small. This
|
|
|
|
* creates a uniform distribution of node sizes across the entire
|
|
|
|
* machine (but not necessarily over physical nodes).
|
|
|
|
*/
|
2010-08-26 00:39:17 +04:00
|
|
|
min_size = (max_addr - addr - memblock_x86_hole_size(addr, max_addr)) /
|
2010-02-16 00:43:30 +03:00
|
|
|
MAX_NUMNODES;
|
|
|
|
min_size = max(min_size, FAKE_NODE_MIN_SIZE);
|
|
|
|
if ((min_size & FAKE_NODE_MIN_HASH_MASK) < min_size)
|
|
|
|
min_size = (min_size + FAKE_NODE_MIN_SIZE) &
|
|
|
|
FAKE_NODE_MIN_HASH_MASK;
|
|
|
|
if (size < min_size) {
|
|
|
|
pr_err("Fake node size %LuMB too small, increasing to %LuMB\n",
|
|
|
|
size >> 20, min_size >> 20);
|
|
|
|
size = min_size;
|
|
|
|
}
|
|
|
|
size &= FAKE_NODE_MIN_HASH_MASK;
|
|
|
|
|
|
|
|
for (i = 0; i < MAX_NUMNODES; i++)
|
|
|
|
if (physnodes[i].start != physnodes[i].end)
|
|
|
|
node_set(i, physnode_mask);
|
|
|
|
/*
|
|
|
|
* Fill physical nodes with fake nodes of size until there is no memory
|
|
|
|
* left on any of them.
|
|
|
|
*/
|
|
|
|
while (nodes_weight(physnode_mask)) {
|
|
|
|
for_each_node_mask(i, physnode_mask) {
|
|
|
|
u64 dma32_end = MAX_DMA32_PFN << PAGE_SHIFT;
|
|
|
|
u64 end;
|
|
|
|
|
|
|
|
end = find_end_of_node(physnodes[i].start,
|
|
|
|
physnodes[i].end, size);
|
|
|
|
/*
|
|
|
|
* If there won't be at least FAKE_NODE_MIN_SIZE of
|
|
|
|
* non-reserved memory in ZONE_DMA32 for the next node,
|
|
|
|
* this one must extend to the boundary.
|
|
|
|
*/
|
|
|
|
if (end < dma32_end && dma32_end - end -
|
2010-08-26 00:39:17 +04:00
|
|
|
memblock_x86_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE)
|
2010-02-16 00:43:30 +03:00
|
|
|
end = dma32_end;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If there won't be enough non-reserved memory for the
|
|
|
|
* next node, this one must extend to the end of the
|
|
|
|
* physical node.
|
|
|
|
*/
|
|
|
|
if (physnodes[i].end - end -
|
2010-08-26 00:39:17 +04:00
|
|
|
memblock_x86_hole_size(end, physnodes[i].end) < size)
|
2010-02-16 00:43:30 +03:00
|
|
|
end = physnodes[i].end;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Setup the fake node that will be allocated as bootmem
|
|
|
|
* later. If setup_node_range() returns non-zero, there
|
|
|
|
* is no more memory available on this physical node.
|
|
|
|
*/
|
|
|
|
if (setup_node_range(ret++, &physnodes[i].start,
|
|
|
|
end - physnodes[i].start,
|
|
|
|
physnodes[i].end) < 0)
|
|
|
|
node_clear(i, physnode_mask);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2007-05-02 21:27:09 +04:00
|
|
|
/*
|
2008-05-12 17:43:36 +04:00
|
|
|
* Sets up the system RAM area from start_pfn to last_pfn according to the
|
2007-05-02 21:27:09 +04:00
|
|
|
* numa=fake command-line option.
|
|
|
|
*/
|
2009-09-26 02:20:09 +04:00
|
|
|
static int __init numa_emulation(unsigned long start_pfn,
|
|
|
|
unsigned long last_pfn, int acpi, int k8)
|
2007-05-02 21:27:09 +04:00
|
|
|
{
|
2010-02-16 00:43:33 +03:00
|
|
|
u64 addr = start_pfn << PAGE_SHIFT;
|
2008-05-12 17:43:36 +04:00
|
|
|
u64 max_addr = last_pfn << PAGE_SHIFT;
|
2009-09-26 02:20:09 +04:00
|
|
|
int num_phys_nodes;
|
2010-02-16 00:43:33 +03:00
|
|
|
int num_nodes;
|
|
|
|
int i;
|
2007-05-02 21:27:09 +04:00
|
|
|
|
2009-09-26 02:20:09 +04:00
|
|
|
num_phys_nodes = setup_physnodes(addr, max_addr, acpi, k8);
|
2010-02-16 00:43:30 +03:00
|
|
|
/*
|
|
|
|
* If the numa=fake command-line contains a 'M' or 'G', it represents
|
2010-02-16 00:43:33 +03:00
|
|
|
* the fixed node size. Otherwise, if it is just a single number N,
|
|
|
|
* split the system RAM into N fake nodes.
|
2010-02-16 00:43:30 +03:00
|
|
|
*/
|
|
|
|
if (strchr(cmdline, 'M') || strchr(cmdline, 'G')) {
|
2010-02-16 00:43:33 +03:00
|
|
|
u64 size;
|
|
|
|
|
2010-02-16 00:43:30 +03:00
|
|
|
size = memparse(cmdline, &cmdline);
|
|
|
|
num_nodes = split_nodes_size_interleave(addr, max_addr, size);
|
2010-02-16 00:43:33 +03:00
|
|
|
} else {
|
|
|
|
unsigned long n;
|
2010-02-16 00:43:30 +03:00
|
|
|
|
2010-02-16 00:43:33 +03:00
|
|
|
n = simple_strtoul(cmdline, NULL, 0);
|
|
|
|
num_nodes = split_nodes_interleave(addr, max_addr, num_phys_nodes, n);
|
2007-05-02 21:27:09 +04:00
|
|
|
}
|
|
|
|
|
2010-02-16 00:43:33 +03:00
|
|
|
if (num_nodes < 0)
|
|
|
|
return num_nodes;
|
2008-03-25 20:14:35 +03:00
|
|
|
memnode_shift = compute_hash_shift(nodes, num_nodes, NULL);
|
2007-05-02 21:27:09 +04:00
|
|
|
if (memnode_shift < 0) {
|
|
|
|
memnode_shift = 0;
|
|
|
|
printk(KERN_ERR "No NUMA hash function found. NUMA emulation "
|
|
|
|
"disabled.\n");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-09-26 02:20:09 +04:00
|
|
|
* We need to vacate all active ranges that may have been registered for
|
|
|
|
* the e820 memory map.
|
2007-05-02 21:27:09 +04:00
|
|
|
*/
|
|
|
|
remove_all_active_ranges();
|
2007-05-02 21:27:20 +04:00
|
|
|
for_each_node_mask(i, node_possible_map) {
|
2010-08-26 00:39:17 +04:00
|
|
|
memblock_x86_register_active_regions(i, nodes[i].start >> PAGE_SHIFT,
|
2006-09-27 12:49:52 +04:00
|
|
|
nodes[i].end >> PAGE_SHIFT);
|
2008-01-30 15:30:37 +03:00
|
|
|
setup_node_bootmem(i, nodes[i].start, nodes[i].end);
|
2006-09-27 12:49:52 +04:00
|
|
|
}
|
2007-07-21 19:10:32 +04:00
|
|
|
acpi_fake_nodes(nodes, num_nodes);
|
2008-01-30 15:30:37 +03:00
|
|
|
numa_init_array();
|
|
|
|
return 0;
|
2005-04-17 02:20:36 +04:00
|
|
|
}
|
2007-05-02 21:27:09 +04:00
|
|
|
#endif /* CONFIG_NUMA_EMU */
|
2005-04-17 02:20:36 +04:00
|
|
|
|
2009-09-26 02:20:00 +04:00
|
|
|
void __init initmem_init(unsigned long start_pfn, unsigned long last_pfn,
|
|
|
|
int acpi, int k8)
|
2008-01-30 15:30:37 +03:00
|
|
|
{
|
2005-04-17 02:20:36 +04:00
|
|
|
int i;
|
|
|
|
|
2007-05-02 21:27:20 +04:00
|
|
|
nodes_clear(node_possible_map);
|
2008-02-17 13:02:21 +03:00
|
|
|
nodes_clear(node_online_map);
|
2007-05-02 21:27:20 +04:00
|
|
|
|
2005-04-17 02:20:36 +04:00
|
|
|
#ifdef CONFIG_NUMA_EMU
|
2009-09-26 02:20:09 +04:00
|
|
|
if (cmdline && !numa_emulation(start_pfn, last_pfn, acpi, k8))
|
2008-01-30 15:30:37 +03:00
|
|
|
return;
|
2007-05-02 21:27:20 +04:00
|
|
|
nodes_clear(node_possible_map);
|
2008-02-17 13:02:21 +03:00
|
|
|
nodes_clear(node_online_map);
|
2005-04-17 02:20:36 +04:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_ACPI_NUMA
|
2009-09-26 02:20:04 +04:00
|
|
|
if (!numa_off && acpi && !acpi_scan_nodes(start_pfn << PAGE_SHIFT,
|
|
|
|
last_pfn << PAGE_SHIFT))
|
2008-01-30 15:30:37 +03:00
|
|
|
return;
|
2007-05-02 21:27:20 +04:00
|
|
|
nodes_clear(node_possible_map);
|
2008-02-17 13:02:21 +03:00
|
|
|
nodes_clear(node_online_map);
|
2005-04-17 02:20:36 +04:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_K8_NUMA
|
2009-09-26 02:20:00 +04:00
|
|
|
if (!numa_off && k8 && !k8_scan_nodes())
|
2005-04-17 02:20:36 +04:00
|
|
|
return;
|
2007-05-02 21:27:20 +04:00
|
|
|
nodes_clear(node_possible_map);
|
2008-02-17 13:02:21 +03:00
|
|
|
nodes_clear(node_online_map);
|
2005-04-17 02:20:36 +04:00
|
|
|
#endif
|
|
|
|
printk(KERN_INFO "%s\n",
|
|
|
|
numa_off ? "NUMA turned off" : "No NUMA configuration found");
|
|
|
|
|
2008-01-30 15:30:37 +03:00
|
|
|
printk(KERN_INFO "Faking a node at %016lx-%016lx\n",
|
2005-04-17 02:20:36 +04:00
|
|
|
start_pfn << PAGE_SHIFT,
|
2008-05-12 17:43:36 +04:00
|
|
|
last_pfn << PAGE_SHIFT);
|
2008-01-30 15:30:37 +03:00
|
|
|
/* setup dummy node covering all memory */
|
|
|
|
memnode_shift = 63;
|
2007-02-13 15:26:19 +03:00
|
|
|
memnodemap = memnode.embedded_map;
|
2005-04-17 02:20:36 +04:00
|
|
|
memnodemap[0] = 0;
|
|
|
|
node_set_online(0);
|
2007-05-02 21:27:20 +04:00
|
|
|
node_set(0, node_possible_map);
|
2008-12-17 04:34:01 +03:00
|
|
|
for (i = 0; i < nr_cpu_ids; i++)
|
2005-11-05 19:25:53 +03:00
|
|
|
numa_set_node(i, 0);
|
2010-08-26 00:39:17 +04:00
|
|
|
memblock_x86_register_active_regions(0, start_pfn, last_pfn);
|
2008-05-12 17:43:36 +04:00
|
|
|
setup_node_bootmem(0, start_pfn << PAGE_SHIFT, last_pfn << PAGE_SHIFT);
|
2005-11-05 19:25:53 +03:00
|
|
|
}
|
|
|
|
|
2008-01-30 15:30:37 +03:00
|
|
|
unsigned long __init numa_free_all_bootmem(void)
|
|
|
|
{
|
2005-04-17 02:20:36 +04:00
|
|
|
unsigned long pages = 0;
|
2008-01-30 15:30:37 +03:00
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_online_node(i)
|
2005-04-17 02:20:36 +04:00
|
|
|
pages += free_all_bootmem_node(NODE_DATA(i));
|
2008-01-30 15:30:37 +03:00
|
|
|
|
2010-02-10 12:20:20 +03:00
|
|
|
pages += free_all_memory_core_early(MAX_NUMNODES);
|
|
|
|
|
2005-04-17 02:20:36 +04:00
|
|
|
return pages;
|
2008-01-30 15:30:37 +03:00
|
|
|
}
|
2005-04-17 02:20:36 +04:00
|
|
|
|
2006-09-26 12:52:32 +04:00
|
|
|
static __init int numa_setup(char *opt)
|
2008-01-30 15:30:37 +03:00
|
|
|
{
|
2006-09-26 12:52:32 +04:00
|
|
|
if (!opt)
|
|
|
|
return -EINVAL;
|
2008-01-30 15:30:37 +03:00
|
|
|
if (!strncmp(opt, "off", 3))
|
2005-04-17 02:20:36 +04:00
|
|
|
numa_off = 1;
|
|
|
|
#ifdef CONFIG_NUMA_EMU
|
2007-05-02 21:27:09 +04:00
|
|
|
if (!strncmp(opt, "fake=", 5))
|
|
|
|
cmdline = opt + 5;
|
2005-04-17 02:20:36 +04:00
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_ACPI_NUMA
|
2008-01-30 15:30:37 +03:00
|
|
|
if (!strncmp(opt, "noacpi", 6))
|
|
|
|
acpi_numa = -1;
|
2005-04-17 02:20:36 +04:00
|
|
|
#endif
|
2006-09-26 12:52:32 +04:00
|
|
|
return 0;
|
2008-01-30 15:30:37 +03:00
|
|
|
}
|
2006-09-26 12:52:32 +04:00
|
|
|
early_param("numa", numa_setup);
|
|
|
|
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 23:21:12 +04:00
|
|
|
#ifdef CONFIG_NUMA
|
2009-11-21 11:23:37 +03:00
|
|
|
|
|
|
|
static __init int find_near_online_node(int node)
|
|
|
|
{
|
|
|
|
int n, val;
|
|
|
|
int min_val = INT_MAX;
|
|
|
|
int best_node = -1;
|
|
|
|
|
|
|
|
for_each_online_node(n) {
|
|
|
|
val = node_distance(node, n);
|
|
|
|
|
|
|
|
if (val < min_val) {
|
|
|
|
min_val = val;
|
|
|
|
best_node = n;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return best_node;
|
|
|
|
}
|
|
|
|
|
2006-01-12 00:45:36 +03:00
|
|
|
/*
|
|
|
|
* Setup early cpu_to_node.
|
|
|
|
*
|
|
|
|
* Populate cpu_to_node[] only if x86_cpu_to_apicid[],
|
|
|
|
* and apicid_to_node[] tables have valid entries for a CPU.
|
|
|
|
* This means we skip cpu_to_node[] initialisation for NUMA
|
|
|
|
* emulation and faking node case (when running a kernel compiled
|
|
|
|
* for NUMA on a non NUMA box), which is OK as cpu_to_node[]
|
|
|
|
* is already initialized in a round robin manner at numa_init_array,
|
|
|
|
* prior to this call, and this initialization is good enough
|
|
|
|
* for the fake NUMA cases.
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 23:21:12 +04:00
|
|
|
*
|
|
|
|
* Called before the per_cpu areas are setup.
|
2006-01-12 00:45:36 +03:00
|
|
|
*/
|
|
|
|
void __init init_cpu_to_node(void)
|
|
|
|
{
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 23:21:12 +04:00
|
|
|
int cpu;
|
|
|
|
u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);
|
2008-01-30 15:30:37 +03:00
|
|
|
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 23:21:12 +04:00
|
|
|
BUG_ON(cpu_to_apicid == NULL);
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
2008-02-20 02:35:54 +03:00
|
|
|
int node;
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 23:21:12 +04:00
|
|
|
u16 apicid = cpu_to_apicid[cpu];
|
2008-01-30 15:30:37 +03:00
|
|
|
|
2006-01-12 00:45:36 +03:00
|
|
|
if (apicid == BAD_APICID)
|
|
|
|
continue;
|
2008-02-20 02:35:54 +03:00
|
|
|
node = apicid_to_node[apicid];
|
|
|
|
if (node == NUMA_NO_NODE)
|
2006-01-12 00:45:36 +03:00
|
|
|
continue;
|
2008-02-20 02:35:54 +03:00
|
|
|
if (!node_online(node))
|
2009-11-21 11:23:37 +03:00
|
|
|
node = find_near_online_node(node);
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 23:21:12 +04:00
|
|
|
numa_set_node(cpu, node);
|
2006-01-12 00:45:36 +03:00
|
|
|
}
|
|
|
|
}
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 23:21:12 +04:00
|
|
|
#endif
|
2006-01-12 00:45:36 +03:00
|
|
|
|
2006-01-12 00:46:27 +03:00
|
|
|
|
2009-01-27 06:56:47 +03:00
|
|
|
void __cpuinit numa_set_node(int cpu, int node)
|
|
|
|
{
|
|
|
|
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
|
|
|
|
|
|
|
|
/* early setting, no percpu area yet */
|
|
|
|
if (cpu_to_node_map) {
|
|
|
|
cpu_to_node_map[cpu] = node;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
2009-02-08 17:58:40 +03:00
|
|
|
if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) {
|
2009-01-27 06:56:47 +03:00
|
|
|
printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
|
|
|
|
dump_stack();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
per_cpu(x86_cpu_to_node_map, cpu) = node;
|
|
|
|
|
|
|
|
if (node != NUMA_NO_NODE)
|
2010-05-27 01:44:58 +04:00
|
|
|
set_cpu_numa_node(cpu, node);
|
2009-01-27 06:56:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
void __cpuinit numa_clear_node(int cpu)
|
|
|
|
{
|
|
|
|
numa_set_node(cpu, NUMA_NO_NODE);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef CONFIG_DEBUG_PER_CPU_MAPS
|
|
|
|
|
|
|
|
void __cpuinit numa_add_cpu(int cpu)
|
|
|
|
{
|
2009-03-13 07:19:53 +03:00
|
|
|
cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
|
2009-01-27 06:56:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
void __cpuinit numa_remove_cpu(int cpu)
|
|
|
|
{
|
2009-03-13 07:19:53 +03:00
|
|
|
cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
|
2009-01-27 06:56:47 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
#else /* CONFIG_DEBUG_PER_CPU_MAPS */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* --------- debug versions of the numa functions ---------
|
|
|
|
*/
|
|
|
|
static void __cpuinit numa_set_cpumask(int cpu, int enable)
|
|
|
|
{
|
|
|
|
int node = early_cpu_to_node(cpu);
|
2009-03-13 07:19:57 +03:00
|
|
|
struct cpumask *mask;
|
2009-01-27 06:56:47 +03:00
|
|
|
char buf[64];
|
|
|
|
|
2009-03-13 07:19:53 +03:00
|
|
|
mask = node_to_cpumask_map[node];
|
|
|
|
if (mask == NULL) {
|
|
|
|
printk(KERN_ERR "node_to_cpumask_map[%i] NULL\n", node);
|
2009-01-27 06:56:47 +03:00
|
|
|
dump_stack();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (enable)
|
2009-03-13 07:19:53 +03:00
|
|
|
cpumask_set_cpu(cpu, mask);
|
2009-01-27 06:56:47 +03:00
|
|
|
else
|
2009-03-13 07:19:53 +03:00
|
|
|
cpumask_clear_cpu(cpu, mask);
|
2009-01-27 06:56:47 +03:00
|
|
|
|
|
|
|
cpulist_scnprintf(buf, sizeof(buf), mask);
|
|
|
|
printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n",
|
|
|
|
enable ? "numa_add_cpu" : "numa_remove_cpu", cpu, node, buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
void __cpuinit numa_add_cpu(int cpu)
|
|
|
|
{
|
|
|
|
numa_set_cpumask(cpu, 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
void __cpuinit numa_remove_cpu(int cpu)
|
|
|
|
{
|
|
|
|
numa_set_cpumask(cpu, 0);
|
|
|
|
}
|
|
|
|
|
2010-05-27 01:44:58 +04:00
|
|
|
int __cpu_to_node(int cpu)
|
2009-01-27 06:56:47 +03:00
|
|
|
{
|
|
|
|
if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
|
|
|
|
printk(KERN_WARNING
|
|
|
|
"cpu_to_node(%d): usage too early!\n", cpu);
|
|
|
|
dump_stack();
|
|
|
|
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
|
|
|
|
}
|
|
|
|
return per_cpu(x86_cpu_to_node_map, cpu);
|
|
|
|
}
|
2010-05-27 01:44:58 +04:00
|
|
|
EXPORT_SYMBOL(__cpu_to_node);
|
2009-01-27 06:56:47 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Same function as cpu_to_node() but used if called before the
|
|
|
|
* per_cpu areas are setup.
|
|
|
|
*/
|
|
|
|
int early_cpu_to_node(int cpu)
|
|
|
|
{
|
|
|
|
if (early_per_cpu_ptr(x86_cpu_to_node_map))
|
|
|
|
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
|
|
|
|
|
2009-02-08 17:58:40 +03:00
|
|
|
if (!cpu_possible(cpu)) {
|
2009-01-27 06:56:47 +03:00
|
|
|
printk(KERN_WARNING
|
|
|
|
"early_cpu_to_node(%d): no per_cpu area!\n", cpu);
|
|
|
|
dump_stack();
|
|
|
|
return NUMA_NO_NODE;
|
|
|
|
}
|
|
|
|
return per_cpu(x86_cpu_to_node_map, cpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* --------- end of debug versions of the numa functions ---------
|
|
|
|
*/
|
|
|
|
|
|
|
|
#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
|