This patch brings the secondary processors up to the point where they
have a per-processor structure pointer in r13, at which point they just
enter a spinloop. If you test it, you should see two additional pieces
of output on your console:
spinning up secondary processor #1: ping = 0xffffffff: pong = 0x1
spinning up secondary processor #2: ping = 0xffffffff: pong = 0x2
spinning up secondary processor #3: ping = 0xffffffff: pong = 0x3
(XEN) CPU #1: Hello World! SP = 1fe60 TOC = 757e78 HID0 = 51118180000000
(XEN) CPU #2: Hello World! SP = 1be60 TOC = 757e78 HID0 = 51118180000000
(XEN) CPU #3: Hello World! SP = 79fe60 TOC = 757e78 HID0 = 51118180000000
I think this patch is a candidate for merging. I will submit a patch
next for smp_processor_id, and then one for making the secondary
processors join the idle domain, but I would like to have any necessary
discussion about this portion first.
Note that this patch also makes arch/powerpc build with -Wshadow, since
otherwise it is way too easy to accidentaly shadow the global parea
variable which is supposed to only be referred to in r13.
Tested on systemsim-gpul with up to four simulated processors, Maple-D
with two processors, JS20 with two processors, and JS21 with four processors.
Signed-off-by: Amos Waterland <apw@xxxxxxxxxx>
---
arch/powerpc/Makefile | 2 -
arch/powerpc/boot_of.c | 53 +++++++++++++++++++++++++++++++++---
arch/powerpc/mpic.c | 2 -
arch/powerpc/ofd_fixup.c | 4 +-
arch/powerpc/powerpc64/exceptions.S | 31 +++++++++++++++++++++
arch/powerpc/powerpc64/ppc970.c | 22 ++++++++------
arch/powerpc/setup.c | 52 ++++++++++++++++++++++++++++++++++-
include/asm-powerpc/config.h | 2 -
include/asm-powerpc/current.h | 3 +-
include/asm-powerpc/processor.h | 2 -
10 files changed, 152 insertions(+), 21 deletions(-)
diff -r 7855f2e80748 xen/arch/powerpc/Makefile
--- a/xen/arch/powerpc/Makefile Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/Makefile Thu Aug 17 22:24:30 2006 -0400
@@ -46,7 +46,7 @@ obj-y += elf32.o
# These are extra warnings like for the arch/ppc directory but may not
# allow the rest of the tree to build.
-PPC_C_WARNINGS += -Wundef -Wmissing-prototypes -Wmissing-declarations
+PPC_C_WARNINGS += -Wundef -Wmissing-prototypes -Wmissing-declarations -Wshadow
CFLAGS += $(PPC_C_WARNINGS)
LINK=0x400000
diff -r 7855f2e80748 xen/arch/powerpc/boot_of.c
--- a/xen/arch/powerpc/boot_of.c Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/boot_of.c Thu Aug 17 22:24:30 2006 -0400
@@ -31,6 +31,9 @@
#include <asm/io.h>
#include "exceptions.h"
#include "of-devtree.h"
+
+/* Secondary processors use this for handshaking with main processor. */
+volatile unsigned int __spin_ack;
static ulong of_vec;
static ulong of_msr;
@@ -955,7 +958,7 @@ static int __init boot_of_cpus(void)
static int __init boot_of_cpus(void)
{
int cpus;
- int cpu;
+ int cpu, bootcpu;
int result;
u32 cpu_clock[2];
@@ -980,10 +983,53 @@ static int __init boot_of_cpus(void)
cpu_khz /= 1000;
of_printf("OF: clock-frequency = %ld KHz\n", cpu_khz);
- /* FIXME: should not depend on the boot CPU bring the first child */
+ /* Look up which CPU we are running on right now. */
+ result = of_getprop(bof_chosen, "cpu", &bootcpu, sizeof (bootcpu));
+ if (result == OF_FAILURE)
+ of_panic("Failed to look up boot cpu\n");
+
cpu = of_getpeer(cpu);
while (cpu > 0) {
- of_start_cpu(cpu, (ulong)spin_start, 0);
+ unsigned int cpuid, ping, pong;
+ unsigned long now, then, timeout;
+
+ if (cpu == bootcpu) {
+ of_printf("skipping boot cpu!\n");
+ continue;
+ }
+
+ result = of_getprop(cpu, "reg", &cpuid, sizeof(cpuid));
+ if (result == OF_FAILURE)
+ of_panic("cpuid lookup failed\n");
+
+ of_printf("spinning up secondary processor #%d: ", cpuid);
+
+ __spin_ack = ~0x0;
+ ping = __spin_ack;
+ pong = __spin_ack;
+ of_printf("ping = 0x%x: ", ping);
+
+ mb();
+ result = of_start_cpu(cpu, (ulong)spin_start, cpuid);
+ if (result == OF_FAILURE)
+ of_panic("start cpu failed\n");
+
+ /* We will give the secondary processor five seconds to reply. */
+ then = mftb();
+ timeout = then + (5 * timebase_freq);
+
+ do {
+ now = mftb();
+ if (now >= timeout) {
+ of_printf("BROKEN: ");
+ break;
+ }
+
+ mb();
+ pong = __spin_ack;
+ } while (pong == ping);
+ of_printf("pong = 0x%x\n", pong);
+
cpu = of_getpeer(cpu);
}
return 1;
@@ -1031,6 +1077,7 @@ multiboot_info_t __init *boot_of_init(
boot_of_rtas();
/* end of OF */
+ of_printf("Quiescing Open Firmware ...\n");
of_call("quiesce", 0, 0, NULL);
return &mbi;
diff -r 7855f2e80748 xen/arch/powerpc/mpic.c
--- a/xen/arch/powerpc/mpic.c Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/mpic.c Thu Aug 17 22:24:30 2006 -0400
@@ -498,7 +498,7 @@ static void mpic_enable_irq(unsigned int
#ifdef CONFIG_MPIC_BROKEN_U3
if (mpic->flags & MPIC_BROKEN_U3) {
- unsigned int src = irq - mpic->irq_offset;
+ src = irq - mpic->irq_offset;
if (mpic_is_ht_interrupt(mpic, src) &&
(irq_desc[irq].status & IRQ_LEVEL))
mpic_ht_end_irq(mpic, src);
diff -r 7855f2e80748 xen/arch/powerpc/ofd_fixup.c
--- a/xen/arch/powerpc/ofd_fixup.c Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/ofd_fixup.c Thu Aug 17 22:24:30 2006 -0400
@@ -497,8 +497,8 @@ int ofd_dom0_fixup(struct domain *d, ulo
r = ofd_prop_add(m, n, "ibm,partition-no", &did, sizeof(did));
ASSERT( r > 0 );
- const char dom0[] = "dom0";
- r = ofd_prop_add(m, n, "ibm,partition-name", dom0, sizeof (dom0));
+ const char dom[] = "dom0";
+ r = ofd_prop_add(m, n, "ibm,partition-name", dom0, sizeof (dom));
ASSERT( r > 0 );
diff -r 7855f2e80748 xen/arch/powerpc/powerpc64/exceptions.S
--- a/xen/arch/powerpc/powerpc64/exceptions.S Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/powerpc64/exceptions.S Thu Aug 17 22:24:30 2006 -0400
@@ -514,6 +514,37 @@ _GLOBAL(sleep)
mtmsrd r3
blr
+/* The primary processor issues a firmware call to spin us up at this
+ * address, passing our CPU number in r3. We only need a function
+ * entry point instead of a descriptor since this is never called from
+ * C code.
+ */
.globl spin_start
spin_start:
+ /* Write our processor number as an acknowledgment that we're alive. */
+ LOADADDR(r14, __spin_ack)
+ stw r3, 0(r14)
+ sync
+ /* Find our index in the array of processor_area struct pointers. */
+ LOADADDR(r14, global_cpu_table)
+ muli r15, r3, 8
+ add r14, r14, r15
+ /* Spin until the pointer for our processor goes valid. */
+1: ld r15, 0(r14)
+ cmpldi r15, 0
+ beq 1b
+ /* Dereference the pointer and load our stack pointer. */
+ isync
+ ld r1, PAREA_stack(r15)
+ li r14, STACK_FRAME_OVERHEAD
+ sub r1, r1, r14
+ /* We must not speculatively execute beyond these loads. */
+ LOADADDR(r14, secondary_cpu_init)
+ ld r2, 8(r14)
+ ld r11, 0(r14)
+ mtctr r11
+ isync
+ /* Jump into C code now. */
+ bctrl
+ nop
b .
diff -r 7855f2e80748 xen/arch/powerpc/powerpc64/ppc970.c
--- a/xen/arch/powerpc/powerpc64/ppc970.c Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/powerpc64/ppc970.c Thu Aug 17 22:24:30 2006 -0400
@@ -31,6 +31,8 @@
#undef SERIALIZE
+extern volatile struct processor_area * volatile global_cpu_table[];
+
unsigned int cpu_rma_order(void)
{
/* XXX what about non-HV mode? */
@@ -38,18 +40,15 @@ unsigned int cpu_rma_order(void)
return rma_log_size - PAGE_SHIFT;
}
-void cpu_initialize(void)
+void cpu_initialize(int cpuid)
{
- ulong stack;
+ ulong r1, r2;
+ __asm__ __volatile__ ("mr %0, 1" : "=r" (r1));
+ __asm__ __volatile__ ("mr %0, 2" : "=r" (r2));
- parea = xmalloc(struct processor_area);
+ /* This is SMP safe because the compiler must use r13 for it. */
+ parea = global_cpu_table[cpuid];
ASSERT(parea != NULL);
-
- stack = (ulong)alloc_xenheap_pages(STACK_ORDER);
-
- ASSERT(stack != 0);
- parea->hyp_stack_base = (void *)(stack + STACK_SIZE);
- printk("stack is here: %p\n", parea->hyp_stack_base);
mthsprg0((ulong)parea); /* now ready for exceptions */
@@ -79,7 +78,10 @@ void cpu_initialize(void)
s |= 1UL << (63-3); /* ser-gp */
hid0.word |= s;
#endif
- printk("hid0: 0x%016lx\n", hid0.word);
+
+ printk("CPU #%d: Hello World! SP = %lx TOC = %lx HID0 = %lx\n",
+ cpuid, r1, r2, hid0.word);
+
mthid0(hid0.word);
union hid1 hid1;
diff -r 7855f2e80748 xen/arch/powerpc/setup.c
--- a/xen/arch/powerpc/setup.c Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/setup.c Thu Aug 17 22:43:12 2006 -0400
@@ -78,6 +78,13 @@ extern void idle_loop(void);
/* move us to a header file */
extern void initialize_keytable(void);
+static void init_parea(int cpuid);
+
+volatile struct processor_area * volatile global_cpu_table[NR_CPUS];
+
+static int kick_secondary_cpus(void);
+int secondary_cpu_init(int cpuid, unsigned long);
+
int is_kernel_text(unsigned long addr)
{
if (addr >= (unsigned long) &_start &&
@@ -358,13 +365,16 @@ static void __init __start_xen(multiboot
percpu_init_areas();
- cpu_initialize();
+ init_parea(0);
+ cpu_initialize(0);
#ifdef CONFIG_GDB
initialise_gdb();
if (opt_earlygdb)
debugger_trap_immediate();
#endif
+
+ kick_secondary_cpus();
start_of_day();
@@ -441,6 +451,46 @@ extern void arch_get_xen_caps(xen_capabi
extern void arch_get_xen_caps(xen_capabilities_info_t info);
void arch_get_xen_caps(xen_capabilities_info_t info)
{
+}
+
+static void init_parea(int cpuid)
+{
+ /* Be careful not to shadow the global variable. */
+ volatile struct processor_area *pa;
+ void *stack;
+
+ pa = xmalloc(struct processor_area);
+ if (pa == NULL)
+ panic("failed to allocate parea\n");
+
+ stack = alloc_xenheap_pages(STACK_ORDER);
+ if (stack == NULL)
+ panic("failed to allocate stack\n");
+
+ pa->hyp_stack_base = (void *)((ulong)stack + STACK_SIZE);
+
+ /* This store has the effect of invoking secondary_cpu_init. */
+ global_cpu_table[cpuid] = pa;
+ mb();
+}
+
+static int kick_secondary_cpus()
+{
+ int i;
+
+ for (i = 1; i < NR_CPUS; i++) {
+ init_parea(i);
+ }
+
+ return 0;
+}
+
+/* This is the first C code that secondary processors invoke. */
+int secondary_cpu_init(int cpuid, unsigned long r4)
+{
+ cpu_initialize(cpuid);
+
+ while(1);
}
/*
diff -r 7855f2e80748 xen/include/asm-powerpc/config.h
--- a/xen/include/asm-powerpc/config.h Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/include/asm-powerpc/config.h Thu Aug 17 22:24:30 2006 -0400
@@ -51,7 +51,7 @@ extern char __bss_start[];
#define CONFIG_GDB 1
#define CONFIG_SMP 1
#define CONFIG_PCI 1
-#define NR_CPUS 1
+#define NR_CPUS 4
#ifndef ELFSIZE
#define ELFSIZE 64
diff -r 7855f2e80748 xen/include/asm-powerpc/current.h
--- a/xen/include/asm-powerpc/current.h Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/include/asm-powerpc/current.h Thu Aug 17 22:24:30 2006 -0400
@@ -27,7 +27,8 @@
struct vcpu;
-register struct processor_area *parea asm("r13");
+/* Compiler will not warn about shadowing this. */
+register volatile struct processor_area *parea asm("r13");
static inline struct vcpu *get_current(void)
{
diff -r 7855f2e80748 xen/include/asm-powerpc/processor.h
--- a/xen/include/asm-powerpc/processor.h Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/include/asm-powerpc/processor.h Thu Aug 17 22:24:30 2006 -0400
@@ -40,7 +40,7 @@ extern void show_registers(struct cpu_us
extern void show_registers(struct cpu_user_regs *);
extern void show_execution_state(struct cpu_user_regs *);
extern unsigned int cpu_rma_order(void);
-extern void cpu_initialize(void);
+extern void cpu_initialize(int cpuid);
extern void cpu_init_vcpu(struct vcpu *);
extern void save_cpu_sprs(struct vcpu *);
extern void load_cpu_sprs(struct vcpu *);
_______________________________________________
Xen-ppc-devel mailing list
Xen-ppc-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-ppc-devel
|
Home