# HG changeset patch
# User Juergen Gross <juergen.gross@xxxxxxxxxxxxxx>
# Date 1298541607 -3600
# Node ID 5485071c8b0a6a49f65b7cc47841f7d14b358247
# Parent  c0a46434347b265fc8c45e9be3adc41b43f4f682
Avoid race when moving cpu between cpupools

Moving cpus between cpupools is done under the schedule lock of the moved cpu.
When checking a cpu being member of a cpupool this must be done with the lock
of that cpu being held.
Hot-unplugging of physical cpus might encounter the same problems, but this
should happen only very rarely.

Signed-off-by: juergen.gross@xxxxxxxxxxxxxx

diff -r c0a46434347b -r 5485071c8b0a xen/common/sched_credit.c
--- a/xen/common/sched_credit.c	Wed Feb 16 18:23:48 2011 +0000
+++ b/xen/common/sched_credit.c	Thu Feb 24 11:00:07 2011 +0100
@@ -1268,7 +1268,8 @@ csched_load_balance(struct csched_privat
         /*
          * Any work over there to steal?
          */
-        speer = csched_runq_steal(peer_cpu, cpu, snext->pri);
+        speer = cpu_isset(peer_cpu, *online) ?
+            csched_runq_steal(peer_cpu, cpu, snext->pri) : NULL;
         pcpu_schedule_unlock(peer_cpu);
         if ( speer != NULL )
         {
diff -r c0a46434347b -r 5485071c8b0a xen/common/schedule.c
--- a/xen/common/schedule.c	Wed Feb 16 18:23:48 2011 +0000
+++ b/xen/common/schedule.c	Thu Feb 24 11:00:07 2011 +0100
@@ -394,8 +394,32 @@ static void vcpu_migrate(struct vcpu *v)
 {
     unsigned long flags;
     int old_cpu, new_cpu;
+    int same_lock;
 
-    vcpu_schedule_lock_irqsave(v, flags);
+    for (;;)
+    {
+        vcpu_schedule_lock_irqsave(v, flags);
+
+        /* Select new CPU. */
+        old_cpu = v->processor;
+        new_cpu = SCHED_OP(VCPU2OP(v), pick_cpu, v);
+        same_lock = (per_cpu(schedule_data, new_cpu).schedule_lock ==
+                     per_cpu(schedule_data, old_cpu).schedule_lock);
+
+        if ( same_lock )
+            break;
+
+        if ( !pcpu_schedule_trylock(new_cpu) )
+        {
+            vcpu_schedule_unlock_irqrestore(v, flags);
+            continue;
+        }
+        if ( cpu_isset(new_cpu, v->domain->cpupool->cpu_valid) )
+            break;
+
+        pcpu_schedule_unlock(new_cpu);
+        vcpu_schedule_unlock_irqrestore(v, flags);
+    }
 
     /*
      * NB. Check of v->running happens /after/ setting migration flag
@@ -405,13 +429,12 @@ static void vcpu_migrate(struct vcpu *v)
     if ( v->is_running ||
          !test_and_clear_bit(_VPF_migrating, &v->pause_flags) )
     {
+        if ( !same_lock )
+            pcpu_schedule_unlock(new_cpu);
+
         vcpu_schedule_unlock_irqrestore(v, flags);
         return;
     }
-
-    /* Select new CPU. */
-    old_cpu = v->processor;
-    new_cpu = SCHED_OP(VCPU2OP(v), pick_cpu, v);
 
     /*
      * Transfer urgency status to new CPU before switching CPUs, as once
@@ -424,9 +447,13 @@ static void vcpu_migrate(struct vcpu *v)
         atomic_dec(&per_cpu(schedule_data, old_cpu).urgent_count);
     }
 
-    /* Switch to new CPU, then unlock old CPU.  This is safe because
+    /* Switch to new CPU, then unlock new and old CPU.  This is safe because
      * the lock pointer cant' change while the current lock is held. */
     v->processor = new_cpu;
+
+    if ( !same_lock )
+        pcpu_schedule_unlock(new_cpu);
+
     spin_unlock_irqrestore(
         per_cpu(schedule_data, old_cpu).schedule_lock, flags);