WARNING - OLD ARCHIVES

This is an archived copy of the Xen.org mailing list, which we have preserved to ensure that existing links to archives are not broken. The live archive, which contains the latest emails, can be found at http://lists.xen.org/
   
 
 
Xen 
 
Home Products Support Community News
 
   
 

xen-devel

[Xen-devel] [PATCH] 4/7 xen: Add basic NUMA support - Debug Keyhandler

To: xen-devel@xxxxxxxxxxxxxxxxxxx
Subject: [Xen-devel] [PATCH] 4/7 xen: Add basic NUMA support - Debug Keyhandler
From: Ryan Harper <ryanh@xxxxxxxxxx>
Date: Fri, 16 Dec 2005 17:09:04 -0600
Cc: Ryan Grimm <grimm@xxxxxxxxxx>
Delivery-date: Fri, 16 Dec 2005 23:11:17 +0000
Envelope-to: www-data@xxxxxxxxxxxxxxxxxxx
List-help: <mailto:xen-devel-request@lists.xensource.com?subject=help>
List-id: Xen developer discussion <xen-devel.lists.xensource.com>
List-post: <mailto:xen-devel@lists.xensource.com>
List-subscribe: <http://lists.xensource.com/cgi-bin/mailman/listinfo/xen-devel>, <mailto:xen-devel-request@lists.xensource.com?subject=subscribe>
List-unsubscribe: <http://lists.xensource.com/cgi-bin/mailman/listinfo/xen-devel>, <mailto:xen-devel-request@lists.xensource.com?subject=unsubscribe>
Sender: xen-devel-bounces@xxxxxxxxxxxxxxxxxxx
User-agent: Mutt/1.5.6+20040907i
This patch adds a new keyhandler function, dump_numa_stats, which is
triggered on the Xen debug console by pressing 'n'.  This function will
dump a collection of NUMA-related information.  Such information is
useful when debugging domain memory placement on NUMA systems.

In a future patch we would like to export this information via a
hypercall so it can be collected automatically rather than
interactively.

Here is an example of the output on a Dual Opteron (2 Node, 2 CPU, with
one guest running).

(XEN) 'n' pressed -> dumping numa stats
(XEN) ZONE:XEN NUMA hits 33 misses 4 allocs 41 frees 2579
(XEN) ZONE:DOM NUMA hits 17028 misses 17292 allocs 34321 frees 410828
(XEN) ***********************************
(XEN) stats for dom 0:
(XEN) vcpu[0] -> cpu[0]
(XEN) vcpu[1] -> cpu[1]
(XEN) numa hits  : 677
(XEN) numa misses: 912
(XEN) page allocs: 1589
(XEN) total pages currently possessed: 131072
(XEN) total xen heap pages           : 5
(XEN) pages in node         0: 18520
(XEN) pages in node         1: 112552
(XEN) xenheap pages in node 0: 5
(XEN) xenheap pages in node 1: 0
(XEN) pages not matched: 0
(XEN) ***********************************
(XEN) ***********************************
(XEN) stats for dom 1:
(XEN) vcpu[0] -> cpu[1]
(XEN) vcpu[1] -> cpu[0]
(XEN) vcpu[2] -> cpu[1]
(XEN) vcpu[3] -> cpu[0]
(XEN) numa hits  : 16388
(XEN) numa misses: 16383
(XEN) page allocs: 32771
(XEN) total pages currently possessed: 32768
(XEN) total xen heap pages           : 5
(XEN) pages in node         0: 0
(XEN) pages in node         1: 32768
(XEN) xenheap pages in node 0: 5
(XEN) xenheap pages in node 1: 0
(XEN) pages not matched: 0
(XEN) ***********************************
(XEN) free pages in node 0: 499800
(XEN) free pages in node 1: 246712

-- 
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253   T/L: 678-9253
ryanh@xxxxxxxxxx


diffstat output:
 common/keyhandler.c |   99 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 common/page_alloc.c |   43 ++++++++++++++++++++++
 include/xen/numa.h  |    2 +
 3 files changed, 143 insertions(+), 1 deletion(-)

Signed-off-by: Ryan Harper <ryanh@xxxxxxxxxx>
Signed-off-by: Ryan Grimm <grimm@xxxxxxxxxx>
---
diff -r 31e1def2471e xen/common/keyhandler.c
--- a/xen/common/keyhandler.c   Mon Dec 12 19:46:49 2005
+++ b/xen/common/keyhandler.c   Mon Dec 12 14:16:30 2005
@@ -15,6 +15,16 @@
 
 #define KEY_MAX 256
 #define STR_MAX  64
+#ifdef CONFIG_NUMA
+#include <xen/numa.h>
+/* NUMA/Alloc stats */
+extern unsigned long page_alloc[];  /* total page allocs */
+extern unsigned long page_free[];   /* total page frees */
+extern unsigned long numa_hit[];    /* allocated in intended node */
+extern unsigned long numa_miss[];   /* allocated in non intended node */
+#define MEMZONE_XEN 0
+#define MEMZONE_DOM 1
+#endif
 
 static struct {
     union {
@@ -145,6 +155,91 @@
     read_unlock(&domlist_lock);
 }
 
+#ifdef CONFIG_NUMA
+static void dump_numa_stats(unsigned char key)
+{
+     struct domain   *d  = NULL;
+     struct vcpu     *v  = NULL;
+     struct pfn_info *pg = NULL;
+     int i = 0;
+     u32 free_pages[nodes_detected];
+     
+     printk("'%c' pressed -> dumping numa stats\n", key); 
+     printk("ZONE:XEN NUMA hits %ld misses %ld allocs %ld frees %ld\n",
+            numa_hit[MEMZONE_XEN], 
+            numa_miss[MEMZONE_XEN], 
+            page_alloc[MEMZONE_XEN], 
+            page_free[MEMZONE_XEN]);
+     printk("ZONE:DOM NUMA hits %ld misses %ld allocs %ld frees %ld\n",
+            numa_hit[MEMZONE_DOM], 
+            numa_miss[MEMZONE_DOM], 
+            page_alloc[MEMZONE_DOM], 
+            page_free[MEMZONE_DOM]);
+     /* show per-domain vcpu->cpu mappings and numa stats */
+     read_lock(&domlist_lock);
+ 
+     for_each_domain (d) {
+         u32 domain_page_to_nodes[nodes_detected];
+         u32 xenheap_page_to_nodes[nodes_detected];
+         u32 lost_pages = 0;
+ 
+         memset(domain_page_to_nodes, 0, sizeof(domain_page_to_nodes));
+         memset(xenheap_page_to_nodes, 0, sizeof(xenheap_page_to_nodes));
+ 
+         printk("***********************************\n");
+         printk("stats for dom %d:\n", d->domain_id);
+         /* vcpu->cpu mappings */
+         for_each_vcpu (d, v) 
+             /* make sure vcpu is up */
+             if (!test_bit(_VCPUF_down, &v->vcpu_flags))
+                 printk("vcpu[%d] -> cpu[%d]\n", v->vcpu_id, v->processor);
+         
+         /* numa stats */
+         printk("numa hits  : %ld\n", d->numa_hit);
+         printk("numa misses: %ld\n", d->numa_miss);
+         printk("page allocs: %ld\n", d->page_alloc);
+ 
+         /* page to node mappings */
+         spin_lock(&d->page_alloc_lock);
+         list_for_each_entry(pg, &d->page_list, list) {
+             int nid = page_to_node(pg);
+ 
+             if (nid >= 0)
+                 domain_page_to_nodes[nid]++;
+             else 
+                 lost_pages++;
+         }        
+ 
+         list_for_each_entry(pg, &d->xenpage_list, list) {
+             int nid = page_to_node(pg);
+ 
+             if (nid >= 0)
+                 xenheap_page_to_nodes[nid]++;
+             else 
+                 lost_pages++;
+         }        
+         spin_unlock(&d->page_alloc_lock);
+         printk("total pages currently possessed: %d\n", d->tot_pages);
+         printk("total xen heap pages           : %d\n", d->xenheap_pages); 
+         for (i = 0; i < nodes_detected; i++) {
+             printk("pages in node         %d: %d\n", i, 
+                     domain_page_to_nodes[i]);
+         } 
+         for (i = 0; i < nodes_detected; i++) {
+             printk("xenheap pages in node %d: %d\n", i, 
+                     xenheap_page_to_nodes[i]);
+         } 
+         printk("pages not matched: %d\n", lost_pages); 
+         printk("***********************************\n");
+     }
+     read_unlock(&domlist_lock);
+ 
+     memset(free_pages, 0, sizeof(free_pages));
+ 
+     for ( i = 0; i < nodes_detected; i++) 
+         printk("free pages in node %d: %d\n", i, node_free_pages(i)); 
+}
+#endif
 extern void dump_runq(unsigned char key);
 extern void print_sched_histo(unsigned char key);
 extern void reset_sched_histo(unsigned char key);
@@ -187,6 +282,10 @@
         'l', print_sched_histo, "print sched latency histogram");
     register_keyhandler(
         'L', reset_sched_histo, "reset sched latency histogram");
+#ifdef CONFIG_NUMA
+    register_keyhandler(
+        'n', dump_numa_stats, "dump numa stats");
+#endif
     register_keyhandler(
         'q', do_task_queues, "dump task queues + guest state");
     register_keyhandler(
diff -r 31e1def2471e xen/common/page_alloc.c
--- a/xen/common/page_alloc.c   Mon Dec 12 19:46:49 2005
+++ b/xen/common/page_alloc.c   Mon Dec 12 14:16:30 2005
@@ -308,7 +308,7 @@
    u64 cpu_end;   /* highest end addr near cpu */
    u64 start = 0, end = 0, length = (PAGE_SIZE << list_order);
    struct pfn_info *pg = NULL;
-   struct node_memory_chunk_s *c;
+   struct node_memory_chunk_s *c=NULL;
 
    nid = cpu_to_node[cpu];
    cpu_start = node_min_paddr[nid];
@@ -349,6 +349,47 @@
 
  out:
     return pg;
+}
+
+/* page to node mapping */
+int page_to_node(struct pfn_info *pg)
+{
+    u64 pg_paddr = page_to_phys(pg);
+    u64 node_start_paddr = 0ULL, node_end_paddr = 0ULL;
+    u8 nid = 0;
+    int i = 0;
+
+    for (i = 0; i < num_memory_chunks; i++) {
+        node_start_paddr = node_memory_chunk[i].start_paddr;
+        node_end_paddr   = node_memory_chunk[i].end_paddr;
+        nid              = node_memory_chunk[i].nid;
+        
+        if (pg_paddr >= node_start_paddr && pg_paddr <= node_end_paddr)
+           goto found; 
+    }
+    return -1;
+    
+found:
+    return nid;
+} 
+
+int node_free_pages(int nid) 
+{
+    int i = 0, j = 0, free_pages = 0;
+    struct pfn_info *pg = NULL;
+    spin_lock(&heap_lock);
+    for ( i = 0; i < NR_ZONES; i++) 
+        for ( j = 0; j < MAX_ORDER+1; j++) {
+            /* if page is on free list */
+            list_for_each_entry( pg, &heap[i][j], list) {
+                if ( (pg->count_info & PGC_count_mask) == 0) {
+                    if ( page_to_node(pg) == nid)
+                        free_pages+=1<<j;
+                }
+            }
+        }
+    spin_unlock(&heap_lock);
+    return free_pages;
 }
 #endif
 
diff -r 31e1def2471e xen/include/xen/numa.h
--- a/xen/include/xen/numa.h    Mon Dec 12 19:46:49 2005
+++ b/xen/include/xen/numa.h    Mon Dec 12 14:16:30 2005
@@ -42,5 +36,7 @@
 extern unsigned long numa_miss[];   /* allocated in non intended node */
 
 int numa_init(void);
+int page_to_node(struct pfn_info *pg);
+int node_free_pages(int nid);
 
 #endif /* _LINUX_NUMA_H */

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel

<Prev in Thread] Current Thread [Next in Thread>
  • [Xen-devel] [PATCH] 4/7 xen: Add basic NUMA support - Debug Keyhandler, Ryan Harper <=