diff --git a/arch/ia64/xen/grant-table.c b/arch/ia64/xen/grant-table.c index 777dd9a..8bdc7a9 100644 --- a/arch/ia64/xen/grant-table.c +++ b/arch/ia64/xen/grant-table.c @@ -77,7 +77,7 @@ void xen_free_vm_area(struct vm_struct *area) unsigned long gpfn = (phys_addr >> PAGE_SHIFT) + i; struct xen_memory_reservation reservation = { .nr_extents = 1, - .address_bits = 0, + .mem_flags = 0, .extent_order = 0, .domid = DOMID_SELF }; diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c index c5e31cb..1617dd6 100644 --- a/arch/x86/xen/mmu.c +++ b/arch/x86/xen/mmu.c @@ -2239,7 +2239,7 @@ static int xen_exchange_memory(unsigned long extents_in, unsigned int order_in, .nr_extents = extents_out, .extent_order = order_out, .extent_start = mfns_out, - .address_bits = address_bits, + .mem_flags = address_bits, .domid = DOMID_SELF } }; diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c index 612f2c9..45838f2 100644 --- a/arch/x86/xen/setup.c +++ b/arch/x86/xen/setup.c @@ -38,7 +38,7 @@ extern void xen_syscall32_target(void); static unsigned long __init xen_release_chunk(phys_addr_t start_addr, phys_addr_t end_addr) { struct xen_memory_reservation reservation = { - .address_bits = 0, + .mem_flags = 0, .extent_order = 0, .domid = DOMID_SELF }; diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index d2f862d..ea53eef 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -44,6 +44,8 @@ #include #include #include +#include +#include #include #include @@ -98,7 +100,7 @@ extern unsigned long totalhigh_pages; #endif /* List of ballooned pages, threaded through the mem_map array. */ -static LIST_HEAD(ballooned_pages); +static struct list_head ballooned_pages[MAX_NUMNODES]; /* Main work function, always executed in process context. */ static void balloon_process(struct work_struct *work); @@ -120,13 +122,14 @@ static void scrub_page(struct page *page) /* balloon_append: add the given page to the balloon. */ static void balloon_append(struct page *page) { + int node = page_to_nid(page); /* Lowmem is re-populated first, so highmem pages go at list tail. */ if (PageHighMem(page)) { - list_add_tail(&page->lru, &ballooned_pages); + list_add_tail(&page->lru, &ballooned_pages[node]); balloon_stats.balloon_high++; dec_totalhigh_pages(); } else { - list_add(&page->lru, &ballooned_pages); + list_add(&page->lru, &ballooned_pages[node]); balloon_stats.balloon_low++; } @@ -134,14 +137,14 @@ static void balloon_append(struct page *page) } /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */ -static struct page *balloon_retrieve(void) +static struct page *balloon_retrieve(int node) { struct page *page; - if (list_empty(&ballooned_pages)) + if (list_empty(&ballooned_pages[node])) return NULL; - page = list_entry(ballooned_pages.next, struct page, lru); + page = list_entry(ballooned_pages[node].next, struct page, lru); list_del(&page->lru); if (PageHighMem(page)) { @@ -156,17 +159,17 @@ static struct page *balloon_retrieve(void) return page; } -static struct page *balloon_first_page(void) +static struct page *balloon_first_page(int node) { - if (list_empty(&ballooned_pages)) + if (list_empty(&ballooned_pages[node])) return NULL; - return list_entry(ballooned_pages.next, struct page, lru); + return list_entry(ballooned_pages[node].next, struct page, lru); } -static struct page *balloon_next_page(struct page *page) +static struct page *balloon_next_page(int node, struct page *page) { struct list_head *next = page->lru.next; - if (next == &ballooned_pages) + if (next == &ballooned_pages[node]) return NULL; return list_entry(next, struct page, lru); } @@ -188,38 +191,37 @@ static unsigned long current_target(void) return target; } -static int increase_reservation(unsigned long nr_pages) +static int __increase_node_reservation(int node, unsigned long nr_pages) { unsigned long pfn, i, flags; struct page *page; long rc; + struct xen_memory_reservation reservation = { - .address_bits = 0, + .mem_flags = XENMEMF_node(node), .extent_order = 0, .domid = DOMID_SELF }; - if (nr_pages > ARRAY_SIZE(frame_list)) - nr_pages = ARRAY_SIZE(frame_list); - spin_lock_irqsave(&xen_reservation_lock, flags); - page = balloon_first_page(); - for (i = 0; i < nr_pages; i++) { - BUG_ON(page == NULL); + page = balloon_first_page(node); + for (i = 0; page && i ARRAY_SIZE(frame_list)) + nr_pages = ARRAY_SIZE(frame_list); + + node = next_node(node, node_online_map); + if (node == MAX_NUMNODES) + node = first_node(node_online_map); + + rc = __increase_node_reservation(node, nr_pages); + + return rc; +} + static int decrease_reservation(unsigned long nr_pages) { unsigned long pfn, i, flags; struct page *page; int need_sleep = 0; int ret; + static int node; + struct xen_memory_reservation reservation = { - .address_bits = 0, + .mem_flags = 0, .extent_order = 0, .domid = DOMID_SELF }; @@ -267,7 +288,7 @@ static int decrease_reservation(unsigned long nr_pages) nr_pages = ARRAY_SIZE(frame_list); for (i = 0; i < nr_pages; i++) { - if ((page = alloc_page(GFP_BALLOON)) == NULL) { + if ((page = alloc_pages_node(node, GFP_BALLOON, 0)) == NULL) { nr_pages = i; need_sleep = 1; break; @@ -284,7 +305,6 @@ static int decrease_reservation(unsigned long nr_pages) __pte_ma(0), 0); BUG_ON(ret); } - } /* Ensure that ballooned highmem pages don't have kmaps. */ @@ -309,6 +329,12 @@ static int decrease_reservation(unsigned long nr_pages) spin_unlock_irqrestore(&xen_reservation_lock, flags); + /* ballooned pages are distributed approximately over the nodes. + * No need to check for need_sleep. */ + node = next_node(node, node_online_map); + if (node == MAX_NUMNODES) + node = first_node(node_online_map); + return need_sleep; } @@ -395,12 +421,17 @@ static struct notifier_block xenstore_notifier; static int __init balloon_init(void) { unsigned long pfn; + int node; if (!xen_pv_domain()) return -ENODEV; pr_info("xen_balloon: Initialising balloon driver.\n"); + /* Only the nodes in node_possible_map are used/initialized */ + for_each_node(node) + INIT_LIST_HEAD(&ballooned_pages[node]); + balloon_stats.current_pages = min(xen_start_info->nr_pages, max_pfn); balloon_stats.target_pages = balloon_stats.current_pages; balloon_stats.balloon_low = 0; diff --git a/include/xen/interface/memory.h b/include/xen/interface/memory.h index eac3ce1..e99d6c2 100644 --- a/include/xen/interface/memory.h +++ b/include/xen/interface/memory.h @@ -19,6 +19,21 @@ #define XENMEM_increase_reservation 0 #define XENMEM_decrease_reservation 1 #define XENMEM_populate_physmap 6 + +/* + * Maximum # bits addressable by the user of the allocated region (e.g., I/O + * devices often have a 32-bit limitation even in 64-bit systems). If zero + * then the user has no addressing restriction. This field is not used by + * XENMEM_decrease_reservation. + */ +#define XENMEMF_address_bits(x) (x) +#define XENMEMF_get_address_bits(x) ((x) & 0xffu) +/* NUMA node to allocate from. */ +#define XENMEMF_node(x) (((x) + 1) << 8) +#define XENMEMF_get_node(x) ((((x) >> 8) - 1) & 0xffu) +/* Flag to populate physmap with populate-on-demand entries */ +#define XENMEMF_populate_on_demand (1<<16) + struct xen_memory_reservation { /* @@ -37,13 +52,8 @@ struct xen_memory_reservation { unsigned long nr_extents; unsigned int extent_order; - /* - * Maximum # bits addressable by the user of the allocated region (e.g., - * I/O devices often have a 32-bit limitation even in 64-bit systems). If - * zero then the user has no addressing restriction. - * This field is not used by XENMEM_decrease_reservation. - */ - unsigned int address_bits; + /* XENMEMF flags. */ + unsigned int mem_flags; /* previously called address_bits */ /* * Domain whose reservation is being changed.