KEXEC: correctly revert x2apic state when kexecing

Introduce the boolean variable 'kexecing' which indicates to
the more general functions whether we are on the kexec path
or not.  This is used by disable_local_APIC() to try and revert
the APIC mode back to how it was found on boot, and used by
hpet_disable_legacy_broadcast() to prevent it IPI'ing the
other processors after they have already been shot down.

We also need some fudging of the x2apic_enabled variable
to prevent a protection fault when disabling the IOAPICs.

Finally, make sure we dont jump into the purgatory code with
interupts enabled.  We dont want to be servicing stray
interupts once we are out of Xen.

Signed-off-by: Andrew Cooper <andrew.cooper3@xxxxxxxxxx>

diff -r c656613b1e73 xen/arch/x86/apic.c
--- a/xen/arch/x86/apic.c	Fri Jun 03 13:22:33 2011 +0100
+++ b/xen/arch/x86/apic.c	Fri Jun 03 16:19:00 2011 +0100
@@ -37,6 +37,7 @@
 #include <asm/asm_defns.h> /* for BUILD_SMP_INTERRUPT */
 #include <mach_apic.h>
 #include <io_ports.h>
+#include <xen/kexec.h>
 
 static bool_t tdt_enabled __read_mostly;
 static bool_t tdt_enable __initdata = 1;
@@ -356,6 +357,32 @@ void disable_local_APIC(void)
         wrmsrl(MSR_IA32_APICBASE, msr_content &
                ~(MSR_IA32_APICBASE_ENABLE|MSR_IA32_APICBASE_EXTD));
     }
+
+    if (kexecing){
+        uint64_t msr_content;
+        rdmsrl(MSR_IA32_APICBASE, msr_content);
+        msr_content &= ~ ( MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD );
+        wrmsrl(MSR_IA32_APICBASE, msr_content);
+
+        switch(apic_boot_mode)
+        {
+        case APIC_MODE_DISABLED:
+            break; /* Nothing to do - we did this above */
+        case APIC_MODE_XAPIC:
+            msr_content |= MSR_IA32_APICBASE_ENABLE;
+            wrmsrl(MSR_IA32_APICBASE, msr_content);
+            break;
+        case APIC_MODE_X2APIC:
+            msr_content |= ( MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD );
+            wrmsrl(MSR_IA32_APICBASE, msr_content);
+            break;
+        default:
+            printk("Hit default case when reverting lapic to boot state on core #%d\n",
+                   smp_processor_id());
+            break;
+        }
+    }
+
 }
 
 extern int ioapic_ack_new;
diff -r c656613b1e73 xen/arch/x86/crash.c
--- a/xen/arch/x86/crash.c	Fri Jun 03 13:22:33 2011 +0100
+++ b/xen/arch/x86/crash.c	Fri Jun 03 16:19:00 2011 +0100
@@ -27,6 +27,7 @@
 #include <asm/hvm/support.h>
 #include <asm/apic.h>
 #include <asm/io_apic.h>
+#include <xen/iommu.h>
 
 static atomic_t waiting_for_crash_ipi;
 static unsigned int crashing_cpu;
@@ -78,6 +79,15 @@ static void nmi_shootdown_cpus(void)
     }
 
     __stop_this_cpu();
+
+    /* This is a bit of a hack due to the problems with the x2apic_enabled
+     * variable, but we can't do any better without a significant refactoring
+     * of the APIC code */
+    if ( current_local_apic_mode() == APIC_MODE_X2APIC )
+        x2apic_enabled = 1;
+    else
+        x2apic_enabled = 0;
+
     disable_IO_APIC();
 
     local_irq_restore(flags);
diff -r c656613b1e73 xen/arch/x86/hpet.c
--- a/xen/arch/x86/hpet.c	Fri Jun 03 13:22:33 2011 +0100
+++ b/xen/arch/x86/hpet.c	Fri Jun 03 16:19:00 2011 +0100
@@ -663,7 +663,8 @@ void hpet_disable_legacy_broadcast(void)
 
     spin_unlock_irqrestore(&legacy_hpet_event.lock, flags);
 
-    smp_send_event_check_mask(&cpu_online_map);
+    if (! kexecing)
+        smp_send_event_check_mask(&cpu_online_map);
 }
 
 void hpet_broadcast_enter(void)
diff -r c656613b1e73 xen/arch/x86/machine_kexec.c
--- a/xen/arch/x86/machine_kexec.c	Fri Jun 03 13:22:33 2011 +0100
+++ b/xen/arch/x86/machine_kexec.c	Fri Jun 03 16:19:00 2011 +0100
@@ -99,6 +99,11 @@ void machine_kexec(xen_kexec_image_t *im
     if ( hpet_broadcast_is_available() )
         hpet_disable_legacy_broadcast();
 
+    /* We are about to permenantly jump out of the Xen context into the kexec
+     * purgatory code.  We really dont want to be still servicing interupts.
+     */
+    local_irq_disable();
+
     /*
      * compat_machine_kexec() returns to idle pagetables, which requires us
      * to be running on a static GDT mapping (idle pagetables have no GDT
diff -r c656613b1e73 xen/common/kexec.c
--- a/xen/common/kexec.c	Fri Jun 03 13:22:33 2011 +0100
+++ b/xen/common/kexec.c	Fri Jun 03 16:19:00 2011 +0100
@@ -29,6 +29,8 @@
 #include <compat/kexec.h>
 #endif
 
+bool_t kexecing = FALSE;
+
 static DEFINE_PER_CPU_READ_MOSTLY(void *, crash_notes);
 
 static Elf_Note *xen_crash_note;
@@ -238,6 +240,8 @@ void kexec_crash(void)
     if ( !test_bit(KEXEC_IMAGE_CRASH_BASE + pos, &kexec_flags) )
         return;
 
+    kexecing = TRUE;
+
     kexec_common_shutdown();
     kexec_crash_save_cpu();
     machine_crash_shutdown();
@@ -250,6 +254,8 @@ static long kexec_reboot(void *_image)
 {
     xen_kexec_image_t *image = _image;
 
+    kexecing = TRUE;
+
     kexec_common_shutdown();
     machine_reboot_kexec(image);
 
diff -r c656613b1e73 xen/include/xen/kexec.h
--- a/xen/include/xen/kexec.h	Fri Jun 03 13:22:33 2011 +0100
+++ b/xen/include/xen/kexec.h	Fri Jun 03 16:19:00 2011 +0100
@@ -12,6 +12,8 @@ typedef struct xen_kexec_reserve {
 
 extern xen_kexec_reserve_t kexec_crash_area;
 
+extern bool_t kexecing;
+
 void set_kexec_crash_area_size(u64 system_ram);
 
 /* We have space for 4 images to support atomic update