From: K. Y. Srinivasan <ksrinivasan@xxxxxxxxxx>
Subject: Fix issues in netif cleanup. 


December 6, 2009: Aggressively cleanup transmit resources held in netback
when the guest has exited. 

Signed-off-by: K. Y. Srinivasan <ksrinivasan@xxxxxxxxxx>


Index: linux-2.6.18-xen.hg/drivers/xen/netback/common.h
===================================================================
--- linux-2.6.18-xen.hg.orig/drivers/xen/netback/common.h	2009-12-06 11:34:54.000000000 -0500
+++ linux-2.6.18-xen.hg/drivers/xen/netback/common.h	2009-12-06 11:36:52.000000000 -0500
@@ -105,7 +105,6 @@ typedef struct netif_st {
 
 	unsigned int carrier;
 
-	wait_queue_head_t waiting_to_free;
 } netif_t;
 
 /*
@@ -187,13 +186,10 @@ int netif_map(netif_t *netif, unsigned l
 	      unsigned long rx_ring_ref, unsigned int evtchn);
 
 #define netif_get(_b) (atomic_inc(&(_b)->refcnt))
-#define netif_put(_b)						\
-	do {							\
-		if ( atomic_dec_and_test(&(_b)->refcnt) )	\
-			wake_up(&(_b)->waiting_to_free);	\
-	} while (0)
+#define netif_put(_b) (atomic_dec(&(_b)->refcnt))
 
 void netif_xenbus_init(void);
+void netif_cleanup(netif_t *netif);
 
 #define netif_schedulable(netif)				\
 	(netif_running((netif)->dev) && netback_carrier_ok(netif))
Index: linux-2.6.18-xen.hg/drivers/xen/netback/interface.c
===================================================================
--- linux-2.6.18-xen.hg.orig/drivers/xen/netback/interface.c	2009-12-06 11:34:54.000000000 -0500
+++ linux-2.6.18-xen.hg/drivers/xen/netback/interface.c	2009-12-06 11:36:52.000000000 -0500
@@ -196,7 +196,6 @@ netif_t *netif_alloc(struct device *pare
 	netif->domid  = domid;
 	netif->handle = handle;
 	atomic_set(&netif->refcnt, 1);
-	init_waitqueue_head(&netif->waiting_to_free);
 	netif->dev = dev;
 
 	netback_carrier_off(netif);
@@ -371,9 +370,8 @@ void netif_disconnect(netif_t *netif)
 		rtnl_unlock();
 		netif_put(netif);
 	}
-
+	netif_cleanup(netif);
 	atomic_dec(&netif->refcnt);
-	wait_event(netif->waiting_to_free, atomic_read(&netif->refcnt) == 0);
 
 	del_timer_sync(&netif->credit_timeout);
 	del_timer_sync(&netif->tx_queue_timeout);
Index: linux-2.6.18-xen.hg/drivers/xen/netback/netback.c
===================================================================
--- linux-2.6.18-xen.hg.orig/drivers/xen/netback/netback.c	2009-12-06 11:34:54.000000000 -0500
+++ linux-2.6.18-xen.hg/drivers/xen/netback/netback.c	2009-12-06 11:36:52.000000000 -0500
@@ -934,9 +934,14 @@ inline static void net_tx_action_dealloc
 		    list_empty(&pending_inuse_head))
 			break;
 
-		/* Copy any entries that have been pending for too long. */
+		/* Copy any entries that have been pending for too long; or
+		 * if the interface is being shutdown.
+		 */
 		list_for_each_entry_safe(inuse, n, &pending_inuse_head, list) {
-			if (time_after(inuse->alloc_time + HZ / 2, jiffies))
+			pending_idx = inuse - pending_inuse;
+			netif = pending_tx_info[pending_idx].netif;
+			if ((time_after(inuse->alloc_time + HZ / 2, jiffies)) &&
+			    netback_carrier_ok(netif))
 				break;
 
 			pending_idx = inuse - pending_inuse;
@@ -983,6 +988,33 @@ inline static void net_tx_action_dealloc
 	}
 }
 
+void msleep(unsigned int msecs);
+void netif_cleanup(netif_t *netif)
+{
+cleanup_again:
+	if (atomic_read(&netif->refcnt) == 1)
+		return;
+	if (!__on_net_schedule_list(netif)) {
+		/* Still outstanding refrences and we are not on the
+		 * netif scheduler list. Since we have turned the carrier off,
+		 * we are considered ineligible for getting on the scheduler
+	 	 * list! Go on the scheduler list to cleanup.
+		 */
+		spin_lock_irq(&net_schedule_list_lock);
+		list_add_tail(&netif->list, &net_schedule_list);
+		netif_get(netif);
+		spin_unlock_irq(&net_schedule_list_lock);
+		tasklet_schedule(&net_tx_tasklet);
+	}
+	/*
+	 * We are still cleaning up the netif in the context of the 
+	 * xenbus thread; it might be better to decouple this.
+	 */
+	msleep(1);
+	goto cleanup_again;
+}
+
+
 static void netbk_tx_err(netif_t *netif, netif_tx_request_t *txp, RING_IDX end)
 {
 	RING_IDX cons = netif->tx.req_cons;
@@ -1233,6 +1265,7 @@ static void net_tx_action(unsigned long 
 	unsigned int data_len;
 	int ret, work_to_do;
 
+	smp_rmb();
 	if (dealloc_cons != dealloc_prod)
 		net_tx_action_dealloc();
 
@@ -1247,6 +1280,12 @@ static void net_tx_action(unsigned long 
 
 		RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, work_to_do);
 		if (!work_to_do) {
+			/*
+			 * If netif is being shutdown; try to cleanup.
+			 */
+			if (!netback_carrier_ok(netif))
+				net_tx_action_dealloc();
+
 			netif_put(netif);
 			continue;
 		}