Index: 2006-01-05/arch/x86_64/kernel/smp-xen.c =================================================================== --- 2006-01-05.orig/arch/x86_64/kernel/smp-xen.c 2006-01-06 11:14:53.000000000 +0100 +++ 2006-01-05/arch/x86_64/kernel/smp-xen.c 2006-01-06 10:13:36.000000000 +0100 @@ -27,10 +27,8 @@ #include #include #include -#ifdef CONFIG_XEN -#include -#else +#ifndef CONFIG_XEN /* * Smarter SMP flushing macros. * c/o Linus Torvalds. @@ -68,7 +66,6 @@ union smp_flush_state { to a full cache line because other CPUs can access it and we don't want false sharing in the per cpu data segment. */ static DEFINE_PER_CPU(union smp_flush_state, flush_state); -#endif /* * We cannot call mmdrop() because we are in interrupt context, @@ -82,7 +79,6 @@ static inline void leave_mm(unsigned lon load_cr3(swapper_pg_dir); } -#ifndef CONFIG_XEN /* * * The flush IPI assumes that a thread switch happens in this order: @@ -285,6 +281,7 @@ void flush_tlb_all(void) on_each_cpu(do_flush_tlb_all, NULL, 1, 1); } #else +#include asmlinkage void smp_invalidate_interrupt (void) { return; } void flush_tlb_current_task(void) Index: 2006-01-05/include/asm-x86_64/mach-xen/asm/mmu_context.h =================================================================== --- 2006-01-05.orig/include/asm-x86_64/mach-xen/asm/mmu_context.h 2006-01-06 11:47:19.575860968 +0100 +++ /dev/null 1970-01-01 00:00:00.000000000 +0000 @@ -1,130 +0,0 @@ -#ifndef __X86_64_MMU_CONTEXT_H -#define __X86_64_MMU_CONTEXT_H - -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * possibly do the LDT unload here? - */ -int init_new_context(struct task_struct *tsk, struct mm_struct *mm); -void destroy_context(struct mm_struct *mm); - -static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) -{ -#if 0 /* XEN: no lazy tlb */ - if (read_pda(mmu_state) == TLBSTATE_OK) - write_pda(mmu_state, TLBSTATE_LAZY); -#endif -} - -#define prepare_arch_switch(next) __prepare_arch_switch() - -static inline void __prepare_arch_switch(void) -{ - /* - * Save away %es, %ds, %fs and %gs. Must happen before reload - * of cr3/ldt (i.e., not in __switch_to). - */ - __asm__ __volatile__ ( - "mov %%es,%0 ; mov %%ds,%1 ; mov %%fs,%2 ; mov %%gs,%3" - : "=m" (current->thread.es), - "=m" (current->thread.ds), - "=m" (current->thread.fsindex), - "=m" (current->thread.gsindex) ); - - if (current->thread.ds) - __asm__ __volatile__ ( "movl %0,%%ds" : : "r" (0) ); - - if (current->thread.es) - __asm__ __volatile__ ( "movl %0,%%es" : : "r" (0) ); - - if (current->thread.fsindex) { - __asm__ __volatile__ ( "movl %0,%%fs" : : "r" (0) ); - current->thread.fs = 0; - } - - if (current->thread.gsindex) { - load_gs_index(0); - current->thread.gs = 0; - } -} - -extern void mm_pin(struct mm_struct *mm); -extern void mm_unpin(struct mm_struct *mm); -void mm_pin_all(void); - -static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, - struct task_struct *tsk) -{ - unsigned cpu = smp_processor_id(); - struct mmuext_op _op[3], *op = _op; - - if (likely(prev != next)) { - if (!next->context.pinned) - mm_pin(next); - - /* stop flush ipis for the previous mm */ - clear_bit(cpu, &prev->cpu_vm_mask); -#if 0 /* XEN: no lazy tlb */ - write_pda(mmu_state, TLBSTATE_OK); - write_pda(active_mm, next); -#endif - set_bit(cpu, &next->cpu_vm_mask); - - /* load_cr3(next->pgd) */ - per_cpu(cur_pgd, smp_processor_id()) = next->pgd; - op->cmd = MMUEXT_NEW_BASEPTR; - op->arg1.mfn = pfn_to_mfn(__pa(next->pgd) >> PAGE_SHIFT); - op++; - - /* xen_new_user_pt(__pa(__user_pgd(next->pgd))) */ - op->cmd = MMUEXT_NEW_USER_BASEPTR; - op->arg1.mfn = pfn_to_mfn(__pa(__user_pgd(next->pgd)) >> PAGE_SHIFT); - op++; - - if (unlikely(next->context.ldt != prev->context.ldt)) { - /* load_LDT_nolock(&next->context, cpu) */ - op->cmd = MMUEXT_SET_LDT; - op->arg1.linear_addr = (unsigned long)next->context.ldt; - op->arg2.nr_ents = next->context.size; - op++; - } - - BUG_ON(HYPERVISOR_mmuext_op(_op, op-_op, NULL, DOMID_SELF)); - } - -#if 0 /* XEN: no lazy tlb */ - else { - write_pda(mmu_state, TLBSTATE_OK); - if (read_pda(active_mm) != next) - out_of_line_bug(); - if(!test_and_set_bit(cpu, &next->cpu_vm_mask)) { - /* We were in lazy tlb mode and leave_mm disabled - * tlb flush IPI delivery. We must reload CR3 - * to make sure to use no freed page tables. - */ - load_cr3(next->pgd); - xen_new_user_pt(__pa(__user_pgd(next->pgd))); - load_LDT_nolock(&next->context, cpu); - } - } -#endif -} - -#define deactivate_mm(tsk,mm) do { \ - load_gs_index(0); \ - asm volatile("movl %0,%%fs"::"r"(0)); \ -} while(0) - -#define activate_mm(prev, next) do { \ - switch_mm((prev),(next),NULL); \ -} while (0) - -#endif Index: 2006-01-05/include/asm-x86_64/mmu_context.h =================================================================== --- 2006-01-05.orig/include/asm-x86_64/mmu_context.h 2006-01-06 11:47:19.594858080 +0100 +++ 2006-01-05/include/asm-x86_64/mmu_context.h 2006-01-06 10:27:33.000000000 +0100 @@ -15,42 +15,114 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm); void destroy_context(struct mm_struct *mm); -#ifdef CONFIG_SMP - static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) { +#if defined(CONFIG_SMP) && !defined(CONFIG_XEN) if (read_pda(mmu_state) == TLBSTATE_OK) write_pda(mmu_state, TLBSTATE_LAZY); +#endif +} + +#ifndef CONFIG_XEN + +static inline void load_cr3(pgd_t *pgd) +{ + asm volatile("movq %0,%%cr3" :: "r" (__pa(pgd)) : "memory"); } + +#define __SWITCH_DECLS +#define __switch_pgd(next, cpu) load_cr3((next)->pgd) +#define __switch_ldt(next, cpu) load_LDT_nolock(&(next)->context, cpu); +#define __switch_commit() 0 + #else -static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) + +#define prepare_arch_switch(next) __prepare_arch_switch() + +static inline void __prepare_arch_switch(void) { + /* + * Save away %es, %ds, %fs and %gs. Must happen before reload + * of cr3/ldt (i.e., not in __switch_to). + */ + __asm__ __volatile__ ( + "mov %%es,%0 ; mov %%ds,%1 ; mov %%fs,%2 ; mov %%gs,%3" + : "=m" (current->thread.es), + "=m" (current->thread.ds), + "=m" (current->thread.fsindex), + "=m" (current->thread.gsindex) ); + + if (current->thread.ds) + __asm__ __volatile__ ( "movl %0,%%ds" : : "r" (0) ); + + if (current->thread.es) + __asm__ __volatile__ ( "movl %0,%%es" : : "r" (0) ); + + if (current->thread.fsindex) { + __asm__ __volatile__ ( "movl %0,%%fs" : : "r" (0) ); + current->thread.fs = 0; + } + + if (current->thread.gsindex) { + load_gs_index(0); + current->thread.gs = 0; + } } -#endif -static inline void load_cr3(pgd_t *pgd) +extern void mm_pin(struct mm_struct *mm); +extern void mm_unpin(struct mm_struct *mm); +void mm_pin_all(void); + +#define __SWITCH_DECLS struct mmuext_op ops[3], *op = ops + +static inline struct mmuext_op *__switch_pgd(struct mm_struct *next, int cpu, struct mmuext_op *op) { - asm volatile("movq %0,%%cr3" :: "r" (__pa(pgd)) : "memory"); + if (!next->context.pinned) + mm_pin(next); + per_cpu(cur_pgd, cpu) = next->pgd; + op->cmd = MMUEXT_NEW_BASEPTR; + op->arg1.mfn = pfn_to_mfn(__pa(next->pgd) >> PAGE_SHIFT); + ++op; + /* xen_new_user_pt(__pa(__user_pgd(next->pgd))) */ + op->cmd = MMUEXT_NEW_USER_BASEPTR; + op->arg1.mfn = pfn_to_mfn(__pa(__user_pgd(next->pgd)) >> PAGE_SHIFT); + return op + 1; +} +#define __switch_pgd(next, cpu) ((void)(op = __switch_pgd(next, cpu, op))) + +static inline struct mmuext_op *__switch_ldt(struct mm_struct *next, int cpu, struct mmuext_op *op) +{ + op->cmd = MMUEXT_SET_LDT; + op->arg1.linear_addr = (unsigned long)next->context.ldt; + op->arg2.nr_ents = next->context.size; + return op + 1; } +#define __switch_ldt(next, cpu) ((void)(op = __switch_ldt(next, cpu, op))) + +#define __switch_commit() HYPERVISOR_mmuext_op(ops, op - ops, NULL, DOMID_SELF) + +#endif static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { unsigned cpu = smp_processor_id(); + __SWITCH_DECLS; + if (likely(prev != next)) { /* stop flush ipis for the previous mm */ clear_bit(cpu, &prev->cpu_vm_mask); -#ifdef CONFIG_SMP +#if defined(CONFIG_SMP) && !defined(CONFIG_XEN) write_pda(mmu_state, TLBSTATE_OK); write_pda(active_mm, next); #endif set_bit(cpu, &next->cpu_vm_mask); - load_cr3(next->pgd); + __switch_pgd(next, cpu); if (unlikely(next->context.ldt != prev->context.ldt)) - load_LDT_nolock(&next->context, cpu); + __switch_ldt(next, cpu); } -#ifdef CONFIG_SMP +#if defined(CONFIG_SMP) && !defined(CONFIG_XEN) else { write_pda(mmu_state, TLBSTATE_OK); if (read_pda(active_mm) != next) @@ -60,13 +132,19 @@ static inline void switch_mm(struct mm_s * tlb flush IPI delivery. We must reload CR3 * to make sure to use no freed page tables. */ - load_cr3(next->pgd); - load_LDT_nolock(&next->context, cpu); + __switch_pgd(next, cpu); + __switch_ldt(next, cpu); } } #endif + BUG_ON(__switch_commit()); } +#undef __switch_commit +#undef __switch_ldt +#undef __switch_pgd +#undef __SWITCH_DECLS + #define deactivate_mm(tsk,mm) do { \ load_gs_index(0); \ asm volatile("movl %0,%%fs"::"r"(0)); \