diff -urN a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/setup.c modified/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/setup.c --- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/setup.c 2005-05-01 22:11:31.000000000 -0500 +++ modified/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/setup.c 2005-05-04 16:06:25.000000000 -0500 @@ -1615,6 +1615,33 @@ } } +static irqreturn_t lpreempt_interrupt( + int irq, void *dev_id, struct pt_regs *regs) +{ + int cpu = smp_processor_id(); + vcpu_info_t *vcpu_info = &HYPERVISOR_shared_info->vcpu_data[cpu]; + + set_bit(VCPU_PREEMPT_YLD, (long *)&vcpu_info->vcpu_flags); + wmb(); + HYPERVISOR_yield(); + + return IRQ_HANDLED; +} + +static DEFINE_PER_CPU(int, lpreempt_irq); +static char lpreempt_name[NR_IRQS][15]; + +void lpreempt_setup(void) +{ + int cpu = smp_processor_id(); + + printk("CPU%d: installing preemption handler\n", cpu); + per_cpu(lpreempt_irq, cpu) = bind_virq_to_irq(VIRQ_PREEMPT); + sprintf(lpreempt_name[cpu], "lpreempt%d", cpu); + BUG_ON(request_irq(per_cpu(lpreempt_irq, cpu), lpreempt_interrupt, + SA_INTERRUPT, lpreempt_name[cpu], NULL)); +} + static int xen_panic_event(struct notifier_block *this, unsigned long event, void *ptr) { diff -urN a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c modified/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c --- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-05-01 22:11:26.000000000 -0500 +++ modified/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c 2005-05-04 16:06:25.000000000 -0500 @@ -458,6 +458,7 @@ extern void local_setup_timer(void); +extern void lpreempt_setup(void); /* * Activate a secondary processor. @@ -475,6 +476,7 @@ rep_nop(); local_setup_timer(); ldebug_setup(); + lpreempt_setup(); smp_intr_init(); local_irq_enable(); /* diff -urN a/linux-2.6.11-xen-sparse/init/main.c modified/linux-2.6.11-xen-sparse/init/main.c --- a/linux-2.6.11-xen-sparse/init/main.c 2005-03-02 01:37:49.000000000 -0600 +++ modified/linux-2.6.11-xen-sparse/init/main.c 2005-05-04 16:06:25.000000000 -0500 @@ -368,6 +368,8 @@ * gcc-3.4 accidentally inlines this function, so use noinline. */ +extern void lpreempt_setup(void); + static void noinline rest_init(void) __releases(kernel_lock) { @@ -375,6 +377,7 @@ numa_default_policy(); unlock_kernel(); preempt_enable_no_resched(); + lpreempt_setup(); cpu_idle(); } diff -urN a/xen/common/domain.c modified/xen/common/domain.c --- a/xen/common/domain.c 2005-05-01 22:11:28.000000000 -0500 +++ modified/xen/common/domain.c 2005-05-04 16:06:25.000000000 -0500 @@ -39,6 +39,9 @@ atomic_set(&d->refcnt, 1); atomic_set(&ed->pausecnt, 0); + ed->notifycnt=0; + ed->yieldcnt =0; + d->id = dom_id; ed->processor = cpu; diff -urN a/xen/common/keyhandler.c modified/xen/common/keyhandler.c --- a/xen/common/keyhandler.c 2005-05-01 22:11:31.000000000 -0500 +++ modified/xen/common/keyhandler.c 2005-05-04 16:06:25.000000000 -0500 @@ -95,6 +95,29 @@ machine_restart(NULL); } +static void do_dump_notify(unsigned char key) +{ + struct domain *d; + struct exec_domain *ed; + s_time_t now = NOW(); + + printk("'%c' pressed -> dumping notify info (now=0x%X:%08X)\n", key, + (u32)(now>>32), (u32)now); + + read_lock(&domlist_lock); + for_each_domain ( d ) + { + for_each_exec_domain ( d, ed ) + { + printk("Xen: DOM %d, VCPU %d, notify_virqs_sent=%d" + ", yields=%d\n", d->id, ed->eid, ed->notifycnt, + ed->yieldcnt); + } + + } + read_unlock(&domlist_lock); +} + static void do_task_queues(unsigned char key) { struct domain *d; @@ -183,6 +206,8 @@ register_keyhandler( 'q', do_task_queues, "dump task queues + guest state"); register_keyhandler( + 'D', do_dump_notify, "dump preemption notification info"); + register_keyhandler( 'r', dump_runq, "dump run queues"); register_irq_keyhandler( 'R', halt_machine, "reboot machine"); diff -urN a/xen/common/schedule.c modified/xen/common/schedule.c --- a/xen/common/schedule.c 2005-05-01 22:11:32.000000000 -0500 +++ modified/xen/common/schedule.c 2005-05-05 15:41:55.000000000 -0500 @@ -385,7 +385,73 @@ if ( unlikely(prev == next) ) return continue_running(prev); - + /* prev != next */ +#if 1 + /* since prev != next, we should send upcall to prev + * HACK! we want to ignore non-smp domains, but + * 1) dom0 is currently started with the n_vcpu + * var equal to smp_num_cpus even though the + * kernel it runs may not be smp and boot the next + * vcpu. So, for now, we don't send upcalls to dom0 + * + * Note, that the check on number of vcpus in a + * domain will remain to limit sending upcalls only + * to SMP domains where lock preemption is an issue + */ + if ( + !is_idle_task(prev->domain) && + domain_runnable(prev) && + prev->domain->id != 0 && + prev->domain->shared_info->n_vcpu > 1 + ) { + switch(prev->vcpu_info->vcpu_flags) + { + /* SNT=0, YLD=0; */ + case (0<vcpu_info->vcpu_flags); + prev->notifycnt++; + + /* set timeout for vcpu to yield */ + spin_lock_irq(&schedule_data[cpu].schedule_lock); + rem_ac_timer(&schedule_data[cpu].s_timer); + schedule_data[cpu].s_timer.expires = now + (s32)MICROSECS(55); + add_ac_timer(&schedule_data[cpu].s_timer); + spin_unlock_irq(&schedule_data[cpu].schedule_lock); + + /* send VIRQ */ + send_guest_virq(prev, VIRQ_PREEMPT); + + return continue_running(prev); + } + /* SNT=1, YLD=? */ + case (1<yieldcnt++; + clear_bit(VCPU_PREEMPT_SNT,&prev->vcpu_info->vcpu_flags); + clear_bit(VCPU_PREEMPT_YLD,&prev->vcpu_info->vcpu_flags); + break; + } + case (1<vcpu_info->vcpu_flags); + clear_bit(VCPU_PREEMPT_YLD,&prev->vcpu_info->vcpu_flags); + } + + } + } +#endif perfc_incrc(sched_ctx); #if defined(WAKE_HISTO) diff -urN a/xen/include/public/xen.h modified/xen/include/public/xen.h --- a/xen/include/public/xen.h 2005-05-01 22:11:26.000000000 -0500 +++ modified/xen/include/public/xen.h 2005-05-04 16:06:25.000000000 -0500 @@ -81,7 +81,8 @@ #define VIRQ_DOM_EXC 3 /* (DOM0) Exceptional event for some domain. */ #define VIRQ_PARITY_ERR 4 /* (DOM0) NMI parity error. */ #define VIRQ_IO_ERR 5 /* (DOM0) NMI I/O error. */ -#define NR_VIRQS 7 +#define VIRQ_PREEMPT 6 /* VCPU Preemption pending. */ +#define NR_VIRQS 8 /* * MMU-UPDATE REQUESTS @@ -300,6 +301,10 @@ /* Support for multi-processor guests. */ #define MAX_VIRT_CPUS 32 +/* VCPU Flags */ +#define VCPU_PREEMPT_SNT 0 +#define VCPU_PREEMPT_YLD 1 + /* * Per-VCPU information goes here. This will be cleaned up more when Xen * actually supports multi-VCPU guests. @@ -333,7 +338,7 @@ */ u8 evtchn_upcall_pending; /* 0 */ u8 evtchn_upcall_mask; /* 1 */ - u8 pad0, pad1; + u16 vcpu_flags; /* 2 */ u32 evtchn_pending_sel; /* 4 */ arch_vcpu_info_t arch; /* 8 */ } PACKED vcpu_info_t; /* 8 + arch */ diff -urN a/xen/include/xen/sched.h modified/xen/include/xen/sched.h --- a/xen/include/xen/sched.h 2005-05-01 22:11:26.000000000 -0500 +++ modified/xen/include/xen/sched.h 2005-05-04 16:06:25.000000000 -0500 @@ -83,6 +83,9 @@ atomic_t pausecnt; + u32 notifycnt; + u32 yieldcnt; + struct arch_exec_domain arch; };