2008-03-03 20:12:53 +03:00
|
|
|
#include <linux/cpumask.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/delay.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/kernel_stat.h>
|
|
|
|
#include <linux/mc146818rtc.h>
|
|
|
|
#include <linux/cache.h>
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
|
|
|
|
#include <asm/smp.h>
|
|
|
|
#include <asm/mtrr.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
#include <asm/mmu_context.h>
|
|
|
|
#include <asm/apic.h>
|
|
|
|
#include <asm/proto.h>
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
#include <mach_apic.h>
|
2008-07-25 09:35:56 +04:00
|
|
|
#include <mach_ipi.h>
|
|
|
|
|
2008-03-03 20:12:53 +03:00
|
|
|
/*
|
|
|
|
* the following functions deal with sending IPIs between CPUs.
|
|
|
|
*
|
|
|
|
* We use 'broadcast', CPU->CPU IPIs and self-IPIs too.
|
|
|
|
*/
|
|
|
|
|
|
|
|
static inline int __prepare_ICR(unsigned int shortcut, int vector)
|
|
|
|
{
|
|
|
|
unsigned int icr = shortcut | APIC_DEST_LOGICAL;
|
|
|
|
|
|
|
|
switch (vector) {
|
|
|
|
default:
|
|
|
|
icr |= APIC_DM_FIXED | vector;
|
|
|
|
break;
|
|
|
|
case NMI_VECTOR:
|
|
|
|
icr |= APIC_DM_NMI;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return icr;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int __prepare_ICR2(unsigned int mask)
|
|
|
|
{
|
|
|
|
return SET_APIC_DEST_FIELD(mask);
|
|
|
|
}
|
|
|
|
|
|
|
|
void __send_IPI_shortcut(unsigned int shortcut, int vector)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Subtle. In the case of the 'never do double writes' workaround
|
|
|
|
* we have to lock out interrupts to be safe. As we don't care
|
|
|
|
* of the value read we use an atomic rmw access to avoid costly
|
|
|
|
* cli/sti. Otherwise we use an even cheaper single atomic write
|
|
|
|
* to the APIC.
|
|
|
|
*/
|
|
|
|
unsigned int cfg;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Wait for idle.
|
|
|
|
*/
|
|
|
|
apic_wait_icr_idle();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* No need to touch the target chip field
|
|
|
|
*/
|
|
|
|
cfg = __prepare_ICR(shortcut, vector);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Send the IPI. The write to APIC_ICR fires this off.
|
|
|
|
*/
|
x86: APIC: remove apic_write_around(); use alternatives
Use alternatives to select the workaround for the 11AP Pentium erratum
for the affected steppings on the fly rather than build time. Remove the
X86_GOOD_APIC configuration option and replace all the calls to
apic_write_around() with plain apic_write(), protecting accesses to the
ESR as appropriate due to the 3AP Pentium erratum. Remove
apic_read_around() and all its invocations altogether as not needed.
Remove apic_write_atomic() and all its implementing backends. The use of
ASM_OUTPUT2() is not strictly needed for input constraints, but I have
used it for readability's sake.
I had the feeling no one else was brave enough to do it, so I went ahead
and here it is. Verified by checking the generated assembly and tested
with both a 32-bit and a 64-bit configuration, also with the 11AP
"feature" forced on and verified with gdb on /proc/kcore to work as
expected (as an 11AP machines are quite hard to get hands on these days).
Some script complained about the use of "volatile", but apic_write() needs
it for the same reason and is effectively a replacement for writel(), so I
have disregarded it.
I am not sure what the policy wrt defconfig files is, they are generated
and there is risk of a conflict resulting from an unrelated change, so I
have left changes to them out. The option will get removed from them at
the next run.
Some testing with machines other than mine will be needed to avoid some
stupid mistake, but despite its volume, the change is not really that
intrusive, so I am fairly confident that because it works for me, it will
everywhere.
Signed-off-by: Maciej W. Rozycki <macro@linux-mips.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-07-16 22:15:30 +04:00
|
|
|
apic_write(APIC_ICR, cfg);
|
2008-03-03 20:12:53 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
void send_IPI_self(int vector)
|
|
|
|
{
|
|
|
|
__send_IPI_shortcut(APIC_DEST_SELF, vector);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This is used to send an IPI with no shorthand notation (the destination is
|
|
|
|
* specified in bits 56 to 63 of the ICR).
|
|
|
|
*/
|
|
|
|
static inline void __send_IPI_dest_field(unsigned long mask, int vector)
|
|
|
|
{
|
|
|
|
unsigned long cfg;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Wait for idle.
|
|
|
|
*/
|
|
|
|
if (unlikely(vector == NMI_VECTOR))
|
|
|
|
safe_apic_wait_icr_idle();
|
|
|
|
else
|
|
|
|
apic_wait_icr_idle();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* prepare target chip field
|
|
|
|
*/
|
|
|
|
cfg = __prepare_ICR2(mask);
|
x86: APIC: remove apic_write_around(); use alternatives
Use alternatives to select the workaround for the 11AP Pentium erratum
for the affected steppings on the fly rather than build time. Remove the
X86_GOOD_APIC configuration option and replace all the calls to
apic_write_around() with plain apic_write(), protecting accesses to the
ESR as appropriate due to the 3AP Pentium erratum. Remove
apic_read_around() and all its invocations altogether as not needed.
Remove apic_write_atomic() and all its implementing backends. The use of
ASM_OUTPUT2() is not strictly needed for input constraints, but I have
used it for readability's sake.
I had the feeling no one else was brave enough to do it, so I went ahead
and here it is. Verified by checking the generated assembly and tested
with both a 32-bit and a 64-bit configuration, also with the 11AP
"feature" forced on and verified with gdb on /proc/kcore to work as
expected (as an 11AP machines are quite hard to get hands on these days).
Some script complained about the use of "volatile", but apic_write() needs
it for the same reason and is effectively a replacement for writel(), so I
have disregarded it.
I am not sure what the policy wrt defconfig files is, they are generated
and there is risk of a conflict resulting from an unrelated change, so I
have left changes to them out. The option will get removed from them at
the next run.
Some testing with machines other than mine will be needed to avoid some
stupid mistake, but despite its volume, the change is not really that
intrusive, so I am fairly confident that because it works for me, it will
everywhere.
Signed-off-by: Maciej W. Rozycki <macro@linux-mips.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-07-16 22:15:30 +04:00
|
|
|
apic_write(APIC_ICR2, cfg);
|
2008-03-03 20:12:53 +03:00
|
|
|
|
|
|
|
/*
|
|
|
|
* program the ICR
|
|
|
|
*/
|
|
|
|
cfg = __prepare_ICR(0, vector);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Send the IPI. The write to APIC_ICR fires this off.
|
|
|
|
*/
|
x86: APIC: remove apic_write_around(); use alternatives
Use alternatives to select the workaround for the 11AP Pentium erratum
for the affected steppings on the fly rather than build time. Remove the
X86_GOOD_APIC configuration option and replace all the calls to
apic_write_around() with plain apic_write(), protecting accesses to the
ESR as appropriate due to the 3AP Pentium erratum. Remove
apic_read_around() and all its invocations altogether as not needed.
Remove apic_write_atomic() and all its implementing backends. The use of
ASM_OUTPUT2() is not strictly needed for input constraints, but I have
used it for readability's sake.
I had the feeling no one else was brave enough to do it, so I went ahead
and here it is. Verified by checking the generated assembly and tested
with both a 32-bit and a 64-bit configuration, also with the 11AP
"feature" forced on and verified with gdb on /proc/kcore to work as
expected (as an 11AP machines are quite hard to get hands on these days).
Some script complained about the use of "volatile", but apic_write() needs
it for the same reason and is effectively a replacement for writel(), so I
have disregarded it.
I am not sure what the policy wrt defconfig files is, they are generated
and there is risk of a conflict resulting from an unrelated change, so I
have left changes to them out. The option will get removed from them at
the next run.
Some testing with machines other than mine will be needed to avoid some
stupid mistake, but despite its volume, the change is not really that
intrusive, so I am fairly confident that because it works for me, it will
everywhere.
Signed-off-by: Maciej W. Rozycki <macro@linux-mips.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-07-16 22:15:30 +04:00
|
|
|
apic_write(APIC_ICR, cfg);
|
2008-03-03 20:12:53 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This is only used on smaller machines.
|
|
|
|
*/
|
2008-12-17 04:33:59 +03:00
|
|
|
void send_IPI_mask_bitmask(const struct cpumask *cpumask, int vector)
|
2008-03-03 20:12:53 +03:00
|
|
|
{
|
2008-12-17 04:33:59 +03:00
|
|
|
unsigned long mask = cpumask_bits(cpumask)[0];
|
2008-03-03 20:12:53 +03:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
2008-12-17 04:33:59 +03:00
|
|
|
WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
|
2008-03-03 20:12:53 +03:00
|
|
|
__send_IPI_dest_field(mask, vector);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
|
2008-12-17 04:33:59 +03:00
|
|
|
void send_IPI_mask_sequence(const struct cpumask *mask, int vector)
|
2008-03-03 20:12:53 +03:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
unsigned int query_cpu;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Hack. The clustered APIC addressing mode doesn't allow us to send
|
|
|
|
* to an arbitrary mask, so I do a unicasts to each CPU instead. This
|
|
|
|
* should be modified to do 1 message per cluster ID - mbligh
|
|
|
|
*/
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
2008-12-17 04:33:59 +03:00
|
|
|
for_each_cpu(query_cpu, mask)
|
2008-12-17 04:33:52 +03:00
|
|
|
__send_IPI_dest_field(cpu_to_logical_apicid(query_cpu), vector);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
|
2008-12-17 04:33:59 +03:00
|
|
|
void send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
|
2008-12-17 04:33:52 +03:00
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
unsigned int query_cpu;
|
|
|
|
unsigned int this_cpu = smp_processor_id();
|
|
|
|
|
|
|
|
/* See Hack comment above */
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
2008-12-17 04:33:59 +03:00
|
|
|
for_each_cpu(query_cpu, mask)
|
2008-12-17 04:33:52 +03:00
|
|
|
if (query_cpu != this_cpu)
|
2008-03-03 20:12:53 +03:00
|
|
|
__send_IPI_dest_field(cpu_to_logical_apicid(query_cpu),
|
|
|
|
vector);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* must come after the send_IPI functions above for inlining */
|
|
|
|
static int convert_apicid_to_cpu(int apic_id)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
if (per_cpu(x86_cpu_to_apicid, i) == apic_id)
|
|
|
|
return i;
|
|
|
|
}
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
int safe_smp_processor_id(void)
|
|
|
|
{
|
|
|
|
int apicid, cpuid;
|
|
|
|
|
|
|
|
if (!boot_cpu_has(X86_FEATURE_APIC))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
apicid = hard_smp_processor_id();
|
|
|
|
if (apicid == BAD_APICID)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
cpuid = convert_apicid_to_cpu(apicid);
|
|
|
|
|
|
|
|
return cpuid >= 0 ? cpuid : 0;
|
|
|
|
}
|
|
|
|
#endif
|