WARNING - OLD ARCHIVES

This is an archived copy of the Xen.org mailing list, which we have preserved to ensure that existing links to archives are not broken. The live archive, which contains the latest emails, can be found at http://lists.xen.org/
   
 
 
Xen 
 
Home Products Support Community News
 
   
 

xen-ppc-devel

[XenPPC] [PATCH] Init secondary processors up to assigning r13 (take two

To: xen-ppc-devel@xxxxxxxxxxxxxxxxxxx
Subject: [XenPPC] [PATCH] Init secondary processors up to assigning r13 (take two)
From: Amos Waterland <apw@xxxxxxxxxx>
Date: Fri, 18 Aug 2006 04:33:44 -0400
Cc: Jimi Xenidis <jimix@xxxxxxxxxxxxxx>
Delivery-date: Fri, 18 Aug 2006 01:33:58 -0700
Envelope-to: www-data@xxxxxxxxxxxxxxxxxx
In-reply-to: <20060818025058.GA31132@xxxxxxxxxxxxxxxxxxxxx>
List-help: <mailto:xen-ppc-devel-request@lists.xensource.com?subject=help>
List-id: Xen PPC development <xen-ppc-devel.lists.xensource.com>
List-post: <mailto:xen-ppc-devel@lists.xensource.com>
List-subscribe: <http://lists.xensource.com/cgi-bin/mailman/listinfo/xen-ppc-devel>, <mailto:xen-ppc-devel-request@lists.xensource.com?subject=subscribe>
List-unsubscribe: <http://lists.xensource.com/cgi-bin/mailman/listinfo/xen-ppc-devel>, <mailto:xen-ppc-devel-request@lists.xensource.com?subject=unsubscribe>
References: <20060810031248.GA8233@xxxxxxxxxxxxxxxxxxxxx> <20060818025058.GA31132@xxxxxxxxxxxxxxxxxxxxx>
Sender: xen-ppc-devel-bounces@xxxxxxxxxxxxxxxxxxx
User-agent: Mutt/1.5.12-2006-07-14
This patch brings the secondary processors up to the point where they
have a per-processor structure pointer in r13, at which point they just
enter a spinloop.  It differs from take one in that it has an important
shadowed variable bug fix.  If you test it, you should see two
additional pieces of output on your console:
 
 spinning up secondary processor #1: ping = 0xffffffff: pong = 0x1
 spinning up secondary processor #2: ping = 0xffffffff: pong = 0x2
 spinning up secondary processor #3: ping = 0xffffffff: pong = 0x3
 
 (XEN) CPU #1: Hello World! SP = 1fe60 TOC = 757e78 HID0 = 51118180000000
 (XEN) CPU #2: Hello World! SP = 1be60 TOC = 757e78 HID0 = 51118180000000
 (XEN) CPU #3: Hello World! SP = 79fe60 TOC = 757e78 HID0 = 51118180000000
 
I think this patch is a candidate for merging.  I will submit a patch
next for smp_processor_id, and then one for making the secondary
processors join the idle domain, but I would like to have any necessary
discussion about this portion first.
 
Note that this patch also makes arch/powerpc build with -Wshadow, since
otherwise it is way too easy to accidentaly shadow the global parea
variable which is supposed to only be referred to in r13.

Tested on systemsim-gpul with up to four simulated processors, Maple-D
with two processors, JS20 with two processors, and JS21 with four processors.

Signed-off-by: Amos Waterland <apw@xxxxxxxxxx>
 
---

 arch/powerpc/Makefile               |    2 -
 arch/powerpc/boot_of.c              |   53 +++++++++++++++++++++++++++++++++---
 arch/powerpc/mpic.c                 |    2 -
 arch/powerpc/ofd_fixup.c            |    4 +-
 arch/powerpc/powerpc64/exceptions.S |   31 +++++++++++++++++++++
 arch/powerpc/powerpc64/ppc970.c     |   22 ++++++++------
 arch/powerpc/setup.c                |   52 ++++++++++++++++++++++++++++++++++-
 include/asm-powerpc/config.h        |    2 -
 include/asm-powerpc/current.h       |    2 -
 include/asm-powerpc/processor.h     |    2 -
 10 files changed, 151 insertions(+), 21 deletions(-)

diff -r 7855f2e80748 xen/arch/powerpc/Makefile
--- a/xen/arch/powerpc/Makefile Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/Makefile Thu Aug 17 22:44:27 2006 -0400
@@ -46,7 +46,7 @@ obj-y += elf32.o
 
 # These are extra warnings like for the arch/ppc directory but may not
 # allow the rest of the tree to build.
-PPC_C_WARNINGS += -Wundef -Wmissing-prototypes -Wmissing-declarations
+PPC_C_WARNINGS += -Wundef -Wmissing-prototypes -Wmissing-declarations -Wshadow
 CFLAGS += $(PPC_C_WARNINGS)
 
 LINK=0x400000
diff -r 7855f2e80748 xen/arch/powerpc/boot_of.c
--- a/xen/arch/powerpc/boot_of.c        Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/boot_of.c        Thu Aug 17 22:44:27 2006 -0400
@@ -31,6 +31,9 @@
 #include <asm/io.h>
 #include "exceptions.h"
 #include "of-devtree.h"
+
+/* Secondary processors use this for handshaking with main processor.  */
+volatile unsigned int __spin_ack;
 
 static ulong of_vec;
 static ulong of_msr;
@@ -955,7 +958,7 @@ static int __init boot_of_cpus(void)
 static int __init boot_of_cpus(void)
 {
     int cpus;
-    int cpu;
+    int cpu, bootcpu;
     int result;
     u32 cpu_clock[2];
 
@@ -980,10 +983,53 @@ static int __init boot_of_cpus(void)
     cpu_khz /= 1000;
     of_printf("OF: clock-frequency = %ld KHz\n", cpu_khz);
 
-    /* FIXME: should not depend on the boot CPU bring the first child */
+    /* Look up which CPU we are running on right now.  */
+    result = of_getprop(bof_chosen, "cpu", &bootcpu, sizeof (bootcpu));
+    if (result == OF_FAILURE)
+        of_panic("Failed to look up boot cpu\n");
+
     cpu = of_getpeer(cpu);
     while (cpu > 0) {
-        of_start_cpu(cpu, (ulong)spin_start, 0);
+        unsigned int cpuid, ping, pong;
+        unsigned long now, then, timeout;
+
+        if (cpu == bootcpu) {
+            of_printf("skipping boot cpu!\n");
+            continue;
+        }
+
+        result = of_getprop(cpu, "reg", &cpuid, sizeof(cpuid));
+        if (result == OF_FAILURE)
+            of_panic("cpuid lookup failed\n");
+
+        of_printf("spinning up secondary processor #%d: ", cpuid);
+
+        __spin_ack = ~0x0;
+        ping = __spin_ack;
+        pong = __spin_ack;
+        of_printf("ping = 0x%x: ", ping);
+
+        mb();
+        result = of_start_cpu(cpu, (ulong)spin_start, cpuid);
+        if (result == OF_FAILURE)
+            of_panic("start cpu failed\n");
+
+        /* We will give the secondary processor five seconds to reply.  */
+        then = mftb();
+        timeout = then + (5 * timebase_freq);
+
+        do {
+            now = mftb();
+            if (now >= timeout) {
+                of_printf("BROKEN: ");
+                break;
+            }
+
+            mb();
+            pong = __spin_ack;
+        } while (pong == ping);
+        of_printf("pong = 0x%x\n", pong);
+
         cpu = of_getpeer(cpu);
     }
     return 1;
@@ -1031,6 +1077,7 @@ multiboot_info_t __init *boot_of_init(
     boot_of_rtas();
 
     /* end of OF */
+    of_printf("Quiescing Open Firmware ...\n");
     of_call("quiesce", 0, 0, NULL);
 
     return &mbi;
diff -r 7855f2e80748 xen/arch/powerpc/mpic.c
--- a/xen/arch/powerpc/mpic.c   Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/mpic.c   Thu Aug 17 22:44:27 2006 -0400
@@ -498,7 +498,7 @@ static void mpic_enable_irq(unsigned int
 
 #ifdef CONFIG_MPIC_BROKEN_U3
        if (mpic->flags & MPIC_BROKEN_U3) {
-               unsigned int src = irq - mpic->irq_offset;
+               src = irq - mpic->irq_offset;
                if (mpic_is_ht_interrupt(mpic, src) &&
                    (irq_desc[irq].status & IRQ_LEVEL))
                        mpic_ht_end_irq(mpic, src);
diff -r 7855f2e80748 xen/arch/powerpc/ofd_fixup.c
--- a/xen/arch/powerpc/ofd_fixup.c      Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/ofd_fixup.c      Fri Aug 18 04:15:26 2006 -0400
@@ -497,8 +497,8 @@ int ofd_dom0_fixup(struct domain *d, ulo
     r = ofd_prop_add(m, n, "ibm,partition-no", &did, sizeof(did));
     ASSERT( r > 0 );
 
-    const char dom0[] = "dom0";
-    r = ofd_prop_add(m, n, "ibm,partition-name", dom0, sizeof (dom0));
+    const char dom[] = "dom0";
+    r = ofd_prop_add(m, n, "ibm,partition-name", dom, sizeof (dom));
     ASSERT( r > 0 );
 
 
diff -r 7855f2e80748 xen/arch/powerpc/powerpc64/exceptions.S
--- a/xen/arch/powerpc/powerpc64/exceptions.S   Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/powerpc64/exceptions.S   Thu Aug 17 22:44:27 2006 -0400
@@ -514,6 +514,37 @@ _GLOBAL(sleep)
     mtmsrd r3
     blr
 
+/* The primary processor issues a firmware call to spin us up at this
+ * address, passing our CPU number in r3.  We only need a function
+ * entry point instead of a descriptor since this is never called from
+ * C code.
+ */    
     .globl spin_start
 spin_start:
+    /* Write our processor number as an acknowledgment that we're alive.  */
+    LOADADDR(r14, __spin_ack)
+    stw r3, 0(r14)
+    sync
+    /* Find our index in the array of processor_area struct pointers.  */
+    LOADADDR(r14, global_cpu_table)
+    muli r15, r3, 8
+    add r14, r14, r15
+    /* Spin until the pointer for our processor goes valid.  */
+1:  ld r15, 0(r14)
+    cmpldi r15, 0
+    beq 1b
+    /* Dereference the pointer and load our stack pointer.  */
+    isync
+    ld r1, PAREA_stack(r15)
+    li r14, STACK_FRAME_OVERHEAD
+    sub r1, r1, r14
+    /* We must not speculatively execute beyond these loads.  */
+    LOADADDR(r14, secondary_cpu_init)
+    ld r2, 8(r14)
+    ld r11, 0(r14)
+    mtctr r11
+    isync
+    /* Jump into C code now.  */
+    bctrl
+    nop
     b .
diff -r 7855f2e80748 xen/arch/powerpc/powerpc64/ppc970.c
--- a/xen/arch/powerpc/powerpc64/ppc970.c       Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/powerpc64/ppc970.c       Thu Aug 17 22:44:27 2006 -0400
@@ -31,6 +31,8 @@
 
 #undef SERIALIZE
 
+extern volatile struct processor_area * volatile global_cpu_table[];
+
 unsigned int cpu_rma_order(void)
 {
     /* XXX what about non-HV mode? */
@@ -38,18 +40,15 @@ unsigned int cpu_rma_order(void)
     return rma_log_size - PAGE_SHIFT;
 }
 
-void cpu_initialize(void)
+void cpu_initialize(int cpuid)
 {
-    ulong stack;
+    ulong r1, r2;
+    __asm__ __volatile__ ("mr %0, 1" : "=r" (r1));
+    __asm__ __volatile__ ("mr %0, 2" : "=r" (r2));
 
-    parea = xmalloc(struct processor_area);
+    /* This is SMP safe because the compiler must use r13 for it.  */
+    parea = global_cpu_table[cpuid];
     ASSERT(parea != NULL);
-
-    stack = (ulong)alloc_xenheap_pages(STACK_ORDER);
-
-    ASSERT(stack != 0);
-    parea->hyp_stack_base = (void *)(stack + STACK_SIZE);
-    printk("stack is here: %p\n", parea->hyp_stack_base);
 
     mthsprg0((ulong)parea); /* now ready for exceptions */
 
@@ -79,7 +78,10 @@ void cpu_initialize(void)
     s |= 1UL << (63-3);     /* ser-gp */
     hid0.word |= s;
 #endif
-    printk("hid0: 0x%016lx\n", hid0.word);
+
+    printk("CPU #%d: Hello World! SP = %lx TOC = %lx HID0 = %lx\n", 
+           cpuid, r1, r2, hid0.word);
+
     mthid0(hid0.word);
 
     union hid1 hid1;
diff -r 7855f2e80748 xen/arch/powerpc/setup.c
--- a/xen/arch/powerpc/setup.c  Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/arch/powerpc/setup.c  Thu Aug 17 22:44:27 2006 -0400
@@ -78,6 +78,13 @@ extern void idle_loop(void);
 /* move us to a header file */
 extern void initialize_keytable(void);
 
+static void init_parea(int cpuid);
+
+volatile struct processor_area * volatile global_cpu_table[NR_CPUS];
+
+static int kick_secondary_cpus(void);
+int secondary_cpu_init(int cpuid, unsigned long);
+
 int is_kernel_text(unsigned long addr)
 {
     if (addr >= (unsigned long) &_start &&
@@ -358,13 +365,16 @@ static void __init __start_xen(multiboot
 
     percpu_init_areas();
 
-    cpu_initialize();
+    init_parea(0);
+    cpu_initialize(0);
 
 #ifdef CONFIG_GDB
     initialise_gdb();
     if (opt_earlygdb)
         debugger_trap_immediate();
 #endif
+
+    kick_secondary_cpus();
 
     start_of_day();
 
@@ -441,6 +451,46 @@ extern void arch_get_xen_caps(xen_capabi
 extern void arch_get_xen_caps(xen_capabilities_info_t info);
 void arch_get_xen_caps(xen_capabilities_info_t info)
 {
+}
+
+static void init_parea(int cpuid)
+{
+    /* Be careful not to shadow the global variable.  */
+    volatile struct processor_area *pa;
+    void *stack;
+
+    pa = xmalloc(struct processor_area);
+    if (pa == NULL)
+        panic("failed to allocate parea\n");
+
+    stack = alloc_xenheap_pages(STACK_ORDER);
+    if (stack == NULL)
+        panic("failed to allocate stack\n");
+
+    pa->hyp_stack_base = (void *)((ulong)stack + STACK_SIZE);
+
+    /* This store has the effect of invoking secondary_cpu_init.  */
+    global_cpu_table[cpuid] = pa;
+    mb();
+}
+
+static int kick_secondary_cpus()
+{
+    int i;
+
+    for (i = 1; i < NR_CPUS; i++) {
+        init_parea(i);
+    }
+
+    return 0;
+}
+
+/* This is the first C code that secondary processors invoke.  */
+int secondary_cpu_init(int cpuid, unsigned long r4)
+{
+    cpu_initialize(cpuid);
+
+    while(1);
 }
 
 /*
diff -r 7855f2e80748 xen/include/asm-powerpc/config.h
--- a/xen/include/asm-powerpc/config.h  Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/include/asm-powerpc/config.h  Thu Aug 17 22:44:27 2006 -0400
@@ -51,7 +51,7 @@ extern char __bss_start[];
 #define CONFIG_GDB 1
 #define CONFIG_SMP 1
 #define CONFIG_PCI 1
-#define NR_CPUS 1
+#define NR_CPUS 4
 
 #ifndef ELFSIZE
 #define ELFSIZE 64
diff -r 7855f2e80748 xen/include/asm-powerpc/current.h
--- a/xen/include/asm-powerpc/current.h Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/include/asm-powerpc/current.h Fri Aug 18 04:19:41 2006 -0400
@@ -27,7 +27,7 @@
 
 struct vcpu;
 
-register struct processor_area *parea asm("r13");
+register volatile struct processor_area *parea asm("r13");
 
 static inline struct vcpu *get_current(void)
 {
diff -r 7855f2e80748 xen/include/asm-powerpc/processor.h
--- a/xen/include/asm-powerpc/processor.h       Thu Aug 17 20:38:08 2006 -0400
+++ b/xen/include/asm-powerpc/processor.h       Thu Aug 17 22:44:27 2006 -0400
@@ -40,7 +40,7 @@ extern void show_registers(struct cpu_us
 extern void show_registers(struct cpu_user_regs *);
 extern void show_execution_state(struct cpu_user_regs *);
 extern unsigned int cpu_rma_order(void);
-extern void cpu_initialize(void);
+extern void cpu_initialize(int cpuid);
 extern void cpu_init_vcpu(struct vcpu *);
 extern void save_cpu_sprs(struct vcpu *);
 extern void load_cpu_sprs(struct vcpu *);

_______________________________________________
Xen-ppc-devel mailing list
Xen-ppc-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-ppc-devel