2008-07-10 22:16:54 +04:00
|
|
|
#include <linux/threads.h>
|
|
|
|
#include <linux/cpumask.h>
|
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/ctype.h>
|
2008-07-22 09:08:21 +04:00
|
|
|
#include <linux/dmar.h>
|
2011-05-20 03:45:49 +04:00
|
|
|
#include <linux/cpu.h>
|
2008-07-22 09:08:21 +04:00
|
|
|
|
2008-07-10 22:16:54 +04:00
|
|
|
#include <asm/smp.h>
|
2011-05-20 03:45:50 +04:00
|
|
|
#include <asm/x2apic.h>
|
2008-07-10 22:16:54 +04:00
|
|
|
|
2009-04-11 11:25:26 +04:00
|
|
|
static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid);
|
2011-05-20 03:45:48 +04:00
|
|
|
static DEFINE_PER_CPU(cpumask_var_t, cpus_in_cluster);
|
2011-05-20 03:45:49 +04:00
|
|
|
static DEFINE_PER_CPU(cpumask_var_t, ipi_mask);
|
2008-07-10 22:16:54 +04:00
|
|
|
|
2008-10-12 13:44:11 +04:00
|
|
|
static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
|
2008-07-22 09:08:21 +04:00
|
|
|
{
|
2009-02-22 01:23:21 +03:00
|
|
|
return x2apic_enabled();
|
2008-07-22 09:08:21 +04:00
|
|
|
}
|
|
|
|
|
2011-05-20 03:45:48 +04:00
|
|
|
static inline u32 x2apic_cluster(int cpu)
|
|
|
|
{
|
|
|
|
return per_cpu(x86_cpu_to_logical_apicid, cpu) >> 16;
|
|
|
|
}
|
|
|
|
|
2015-11-05 01:57:00 +03:00
|
|
|
static void x2apic_send_IPI(int cpu, int vector)
|
|
|
|
{
|
|
|
|
u32 dest = per_cpu(x86_cpu_to_logical_apicid, cpu);
|
|
|
|
|
|
|
|
x2apic_wrmsr_fence();
|
|
|
|
__x2apic_send_IPI_dest(dest, vector, APIC_DEST_LOGICAL);
|
|
|
|
}
|
|
|
|
|
2011-05-20 03:45:47 +04:00
|
|
|
static void
|
|
|
|
__x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
|
2008-07-10 22:16:54 +04:00
|
|
|
{
|
2011-05-20 03:45:49 +04:00
|
|
|
struct cpumask *cpus_in_cluster_ptr;
|
|
|
|
struct cpumask *ipi_mask_ptr;
|
|
|
|
unsigned int cpu, this_cpu;
|
2009-01-28 17:42:24 +03:00
|
|
|
unsigned long flags;
|
2011-05-20 03:45:49 +04:00
|
|
|
u32 dest;
|
2008-07-10 22:16:54 +04:00
|
|
|
|
2009-03-17 21:16:54 +03:00
|
|
|
x2apic_wrmsr_fence();
|
|
|
|
|
2008-07-10 22:16:54 +04:00
|
|
|
local_irq_save(flags);
|
2011-05-20 03:45:47 +04:00
|
|
|
|
|
|
|
this_cpu = smp_processor_id();
|
2011-05-20 03:45:49 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We are to modify mask, so we need an own copy
|
|
|
|
* and be sure it's manipulated with irq off.
|
|
|
|
*/
|
2014-08-27 04:12:21 +04:00
|
|
|
ipi_mask_ptr = this_cpu_cpumask_var_ptr(ipi_mask);
|
2014-09-09 02:06:54 +04:00
|
|
|
cpumask_copy(ipi_mask_ptr, mask);
|
2011-05-20 03:45:49 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The idea is to send one IPI per cluster.
|
|
|
|
*/
|
|
|
|
for_each_cpu(cpu, ipi_mask_ptr) {
|
|
|
|
unsigned long i;
|
|
|
|
|
|
|
|
cpus_in_cluster_ptr = per_cpu(cpus_in_cluster, cpu);
|
|
|
|
dest = 0;
|
|
|
|
|
|
|
|
/* Collect cpus in cluster. */
|
|
|
|
for_each_cpu_and(i, ipi_mask_ptr, cpus_in_cluster_ptr) {
|
|
|
|
if (apic_dest == APIC_DEST_ALLINC || i != this_cpu)
|
|
|
|
dest |= per_cpu(x86_cpu_to_logical_apicid, i);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!dest)
|
2011-05-20 03:45:47 +04:00
|
|
|
continue;
|
2011-05-20 03:45:49 +04:00
|
|
|
|
|
|
|
__x2apic_send_IPI_dest(dest, vector, apic->dest_logical);
|
|
|
|
/*
|
|
|
|
* Cluster sibling cpus should be discared now so
|
|
|
|
* we would not send IPI them second time.
|
|
|
|
*/
|
|
|
|
cpumask_andnot(ipi_mask_ptr, ipi_mask_ptr, cpus_in_cluster_ptr);
|
2009-01-28 17:42:24 +03:00
|
|
|
}
|
2011-05-20 03:45:47 +04:00
|
|
|
|
2008-07-10 22:16:54 +04:00
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
|
2011-05-20 03:45:47 +04:00
|
|
|
static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
|
|
|
|
{
|
|
|
|
__x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLINC);
|
|
|
|
}
|
|
|
|
|
2009-01-28 17:42:24 +03:00
|
|
|
static void
|
2012-06-05 15:23:15 +04:00
|
|
|
x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
|
2008-07-10 22:16:54 +04:00
|
|
|
{
|
2011-05-20 03:45:47 +04:00
|
|
|
__x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLBUT);
|
2008-12-17 04:33:52 +03:00
|
|
|
}
|
2008-07-10 22:16:54 +04:00
|
|
|
|
2008-12-17 04:33:52 +03:00
|
|
|
static void x2apic_send_IPI_allbutself(int vector)
|
|
|
|
{
|
2011-05-20 03:45:47 +04:00
|
|
|
__x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLBUT);
|
2008-07-10 22:16:54 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
static void x2apic_send_IPI_all(int vector)
|
|
|
|
{
|
2011-05-20 03:45:47 +04:00
|
|
|
__x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC);
|
2008-07-10 22:16:54 +04:00
|
|
|
}
|
|
|
|
|
2012-06-07 17:15:59 +04:00
|
|
|
static int
|
2009-01-28 17:20:18 +03:00
|
|
|
x2apic_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
|
2012-06-07 17:15:59 +04:00
|
|
|
const struct cpumask *andmask,
|
|
|
|
unsigned int *apicid)
|
2008-12-17 04:33:54 +03:00
|
|
|
{
|
2012-05-22 03:58:02 +04:00
|
|
|
u32 dest = 0;
|
|
|
|
u16 cluster;
|
|
|
|
int i;
|
2008-12-17 04:33:54 +03:00
|
|
|
|
2012-05-22 03:58:02 +04:00
|
|
|
for_each_cpu_and(i, cpumask, andmask) {
|
|
|
|
if (!cpumask_test_cpu(i, cpu_online_mask))
|
|
|
|
continue;
|
|
|
|
dest = per_cpu(x86_cpu_to_logical_apicid, i);
|
|
|
|
cluster = x2apic_cluster(i);
|
|
|
|
break;
|
2009-01-28 17:20:18 +03:00
|
|
|
}
|
|
|
|
|
2012-05-22 03:58:02 +04:00
|
|
|
if (!dest)
|
2012-06-07 17:15:59 +04:00
|
|
|
return -EINVAL;
|
2012-05-22 03:58:02 +04:00
|
|
|
|
|
|
|
for_each_cpu_and(i, cpumask, andmask) {
|
|
|
|
if (!cpumask_test_cpu(i, cpu_online_mask))
|
|
|
|
continue;
|
|
|
|
if (cluster != x2apic_cluster(i))
|
|
|
|
continue;
|
|
|
|
dest |= per_cpu(x86_cpu_to_logical_apicid, i);
|
|
|
|
}
|
|
|
|
|
2012-06-07 17:15:59 +04:00
|
|
|
*apicid = dest;
|
|
|
|
|
|
|
|
return 0;
|
2008-12-17 04:33:54 +03:00
|
|
|
}
|
|
|
|
|
2008-07-10 22:16:54 +04:00
|
|
|
static void init_x2apic_ldr(void)
|
2011-05-20 03:45:48 +04:00
|
|
|
{
|
|
|
|
unsigned int this_cpu = smp_processor_id();
|
|
|
|
unsigned int cpu;
|
|
|
|
|
|
|
|
per_cpu(x86_cpu_to_logical_apicid, this_cpu) = apic_read(APIC_LDR);
|
|
|
|
|
2015-03-05 03:19:17 +03:00
|
|
|
cpumask_set_cpu(this_cpu, per_cpu(cpus_in_cluster, this_cpu));
|
2011-05-20 03:45:48 +04:00
|
|
|
for_each_online_cpu(cpu) {
|
|
|
|
if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
|
|
|
|
continue;
|
2015-03-05 03:19:17 +03:00
|
|
|
cpumask_set_cpu(this_cpu, per_cpu(cpus_in_cluster, cpu));
|
|
|
|
cpumask_set_cpu(cpu, per_cpu(cpus_in_cluster, this_cpu));
|
2011-05-20 03:45:48 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* At CPU state changes, update the x2apic cluster sibling info.
|
|
|
|
*/
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-19 02:23:59 +04:00
|
|
|
static int
|
2011-05-20 03:45:48 +04:00
|
|
|
update_clusterinfo(struct notifier_block *nfb, unsigned long action, void *hcpu)
|
|
|
|
{
|
|
|
|
unsigned int this_cpu = (unsigned long)hcpu;
|
|
|
|
unsigned int cpu;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
switch (action) {
|
|
|
|
case CPU_UP_PREPARE:
|
|
|
|
if (!zalloc_cpumask_var(&per_cpu(cpus_in_cluster, this_cpu),
|
|
|
|
GFP_KERNEL)) {
|
|
|
|
err = -ENOMEM;
|
2011-05-20 03:45:49 +04:00
|
|
|
} else if (!zalloc_cpumask_var(&per_cpu(ipi_mask, this_cpu),
|
|
|
|
GFP_KERNEL)) {
|
|
|
|
free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
|
|
|
|
err = -ENOMEM;
|
2011-05-20 03:45:48 +04:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
case CPU_UP_CANCELED:
|
|
|
|
case CPU_UP_CANCELED_FROZEN:
|
|
|
|
case CPU_DEAD:
|
|
|
|
for_each_online_cpu(cpu) {
|
|
|
|
if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
|
|
|
|
continue;
|
2015-03-10 05:13:45 +03:00
|
|
|
cpumask_clear_cpu(this_cpu, per_cpu(cpus_in_cluster, cpu));
|
|
|
|
cpumask_clear_cpu(cpu, per_cpu(cpus_in_cluster, this_cpu));
|
2011-05-20 03:45:48 +04:00
|
|
|
}
|
|
|
|
free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
|
2011-05-20 03:45:49 +04:00
|
|
|
free_cpumask_var(per_cpu(ipi_mask, this_cpu));
|
2011-05-20 03:45:48 +04:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return notifier_from_errno(err);
|
|
|
|
}
|
|
|
|
|
2015-07-20 19:32:53 +03:00
|
|
|
static struct notifier_block x2apic_cpu_notifier = {
|
2011-05-20 03:45:48 +04:00
|
|
|
.notifier_call = update_clusterinfo,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int x2apic_init_cpu_notifier(void)
|
2008-07-10 22:16:54 +04:00
|
|
|
{
|
|
|
|
int cpu = smp_processor_id();
|
|
|
|
|
2011-05-20 03:45:48 +04:00
|
|
|
zalloc_cpumask_var(&per_cpu(cpus_in_cluster, cpu), GFP_KERNEL);
|
2011-05-20 03:45:49 +04:00
|
|
|
zalloc_cpumask_var(&per_cpu(ipi_mask, cpu), GFP_KERNEL);
|
2011-05-20 03:45:48 +04:00
|
|
|
|
2011-05-20 03:45:49 +04:00
|
|
|
BUG_ON(!per_cpu(cpus_in_cluster, cpu) || !per_cpu(ipi_mask, cpu));
|
2011-05-20 03:45:48 +04:00
|
|
|
|
2015-03-05 03:19:17 +03:00
|
|
|
cpumask_set_cpu(cpu, per_cpu(cpus_in_cluster, cpu));
|
2011-05-20 03:45:48 +04:00
|
|
|
register_hotcpu_notifier(&x2apic_cpu_notifier);
|
|
|
|
return 1;
|
2008-07-10 22:16:54 +04:00
|
|
|
}
|
|
|
|
|
2011-05-20 03:45:46 +04:00
|
|
|
static int x2apic_cluster_probe(void)
|
|
|
|
{
|
2011-05-20 03:45:48 +04:00
|
|
|
if (x2apic_mode)
|
|
|
|
return x2apic_init_cpu_notifier();
|
|
|
|
else
|
|
|
|
return 0;
|
2011-05-20 03:45:46 +04:00
|
|
|
}
|
|
|
|
|
2012-06-26 00:38:29 +04:00
|
|
|
static const struct cpumask *x2apic_cluster_target_cpus(void)
|
|
|
|
{
|
|
|
|
return cpu_all_mask;
|
|
|
|
}
|
|
|
|
|
2012-05-22 03:58:02 +04:00
|
|
|
/*
|
|
|
|
* Each x2apic cluster is an allocation domain.
|
|
|
|
*/
|
2012-06-26 00:38:28 +04:00
|
|
|
static void cluster_vector_allocation_domain(int cpu, struct cpumask *retmask,
|
|
|
|
const struct cpumask *mask)
|
2012-05-22 03:58:02 +04:00
|
|
|
{
|
2012-06-26 00:38:29 +04:00
|
|
|
/*
|
|
|
|
* To minimize vector pressure, default case of boot, device bringup
|
|
|
|
* etc will use a single cpu for the interrupt destination.
|
|
|
|
*
|
|
|
|
* On explicit migration requests coming from irqbalance etc,
|
|
|
|
* interrupts will be routed to the x2apic cluster (cluster-id
|
|
|
|
* derived from the first cpu in the mask) members specified
|
|
|
|
* in the mask.
|
|
|
|
*/
|
|
|
|
if (mask == x2apic_cluster_target_cpus())
|
|
|
|
cpumask_copy(retmask, cpumask_of(cpu));
|
|
|
|
else
|
|
|
|
cpumask_and(retmask, mask, per_cpu(cpus_in_cluster, cpu));
|
2012-05-22 03:58:02 +04:00
|
|
|
}
|
|
|
|
|
2011-05-21 04:51:20 +04:00
|
|
|
static struct apic apic_x2apic_cluster = {
|
2009-01-28 04:37:01 +03:00
|
|
|
|
|
|
|
.name = "cluster x2apic",
|
2011-05-20 03:45:46 +04:00
|
|
|
.probe = x2apic_cluster_probe,
|
2009-01-28 04:37:01 +03:00
|
|
|
.acpi_madt_oem_check = x2apic_acpi_madt_oem_check,
|
2012-03-16 23:25:35 +04:00
|
|
|
.apic_id_valid = x2apic_apic_id_valid,
|
2009-01-28 04:37:01 +03:00
|
|
|
.apic_id_registered = x2apic_apic_id_registered,
|
|
|
|
|
2009-01-28 06:02:31 +03:00
|
|
|
.irq_delivery_mode = dest_LowestPrio,
|
2009-01-28 07:13:04 +03:00
|
|
|
.irq_dest_mode = 1, /* logical */
|
2009-01-28 04:37:01 +03:00
|
|
|
|
2012-06-26 00:38:29 +04:00
|
|
|
.target_cpus = x2apic_cluster_target_cpus,
|
2009-01-28 07:08:44 +03:00
|
|
|
.disable_esr = 0,
|
2009-01-28 07:29:25 +03:00
|
|
|
.dest_logical = APIC_DEST_LOGICAL,
|
2009-01-28 04:37:01 +03:00
|
|
|
.check_apicid_used = NULL,
|
|
|
|
|
2012-05-22 03:58:02 +04:00
|
|
|
.vector_allocation_domain = cluster_vector_allocation_domain,
|
2009-01-28 04:37:01 +03:00
|
|
|
.init_apic_ldr = init_x2apic_ldr,
|
|
|
|
|
|
|
|
.ioapic_phys_id_map = NULL,
|
|
|
|
.setup_apic_routing = NULL,
|
2009-01-28 08:50:47 +03:00
|
|
|
.cpu_present_to_apicid = default_cpu_present_to_apicid,
|
2009-01-28 04:37:01 +03:00
|
|
|
.apicid_to_cpu_present = NULL,
|
2009-01-28 14:43:18 +03:00
|
|
|
.check_phys_apicid_present = default_check_phys_apicid_present,
|
2011-05-20 03:45:50 +04:00
|
|
|
.phys_pkg_id = x2apic_phys_pkg_id,
|
2009-01-28 04:37:01 +03:00
|
|
|
|
2011-05-20 03:45:50 +04:00
|
|
|
.get_apic_id = x2apic_get_apic_id,
|
|
|
|
.set_apic_id = x2apic_set_apic_id,
|
2009-01-28 04:37:01 +03:00
|
|
|
.apic_id_mask = 0xFFFFFFFFu,
|
|
|
|
|
|
|
|
.cpu_mask_to_apicid_and = x2apic_cpu_mask_to_apicid_and,
|
|
|
|
|
2015-11-05 01:57:00 +03:00
|
|
|
.send_IPI = x2apic_send_IPI,
|
2009-01-28 04:37:01 +03:00
|
|
|
.send_IPI_mask = x2apic_send_IPI_mask,
|
|
|
|
.send_IPI_mask_allbutself = x2apic_send_IPI_mask_allbutself,
|
|
|
|
.send_IPI_allbutself = x2apic_send_IPI_allbutself,
|
|
|
|
.send_IPI_all = x2apic_send_IPI_all,
|
|
|
|
.send_IPI_self = x2apic_send_IPI_self,
|
|
|
|
|
|
|
|
.inquire_remote_apic = NULL,
|
2009-02-17 10:02:14 +03:00
|
|
|
|
|
|
|
.read = native_apic_msr_read,
|
|
|
|
.write = native_apic_msr_write,
|
2012-05-16 20:03:58 +04:00
|
|
|
.eoi_write = native_apic_msr_eoi_write,
|
2009-02-17 10:02:14 +03:00
|
|
|
.icr_read = native_x2apic_icr_read,
|
|
|
|
.icr_write = native_x2apic_icr_write,
|
|
|
|
.wait_icr_idle = native_x2apic_wait_icr_idle,
|
|
|
|
.safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle,
|
2008-07-10 22:16:54 +04:00
|
|
|
};
|
2011-05-21 04:51:17 +04:00
|
|
|
|
|
|
|
apic_driver(apic_x2apic_cluster);
|