# HG changeset patch
# User kfraser@xxxxxxxxxxxxxxxxxxxxx
# Date 1173364149 0
# Node ID 18cf0c56226db12b6d4d80e087569b2b9dd05b46
# Parent a7f6392ea8501e1130854ddc66bca8998e9a6448
xen: Clean up heap allocator.
Move some common free/alloc code into core heap-allocator
functions. Make it clear that alloc/free can only ever be done outside
hardirq context (previously it was unsafe: could deadlock on heap_lock).
Signed-off-by: Keir Fraser <keir@xxxxxxxxxxxxx>
---
xen/common/page_alloc.c | 123 +++++++++++++++++++++---------------------------
xen/common/xmalloc.c | 6 ++
2 files changed, 62 insertions(+), 67 deletions(-)
diff -r a7f6392ea850 -r 18cf0c56226d xen/common/page_alloc.c
--- a/xen/common/page_alloc.c Thu Mar 08 10:54:56 2007 +0000
+++ b/xen/common/page_alloc.c Thu Mar 08 14:29:09 2007 +0000
@@ -49,7 +49,7 @@ string_param("badpage", opt_badpage);
* Bit width of the DMA heap.
*/
static unsigned int dma_bitsize = CONFIG_DMA_BITSIZE;
-static unsigned long max_dma_mfn = (1UL << (CONFIG_DMA_BITSIZE - PAGE_SHIFT))
- 1;
+static unsigned long max_dma_mfn = (1UL<<(CONFIG_DMA_BITSIZE-PAGE_SHIFT))-1;
static void parse_dma_bits(char *s)
{
unsigned int v = simple_strtol(s, NULL, 0);
@@ -345,6 +345,7 @@ static struct page_info *alloc_heap_page
unsigned int i, j, zone;
unsigned int node = cpu_to_node(cpu), num_nodes = num_online_nodes();
unsigned long request = 1UL << order;
+ cpumask_t extra_cpus_mask, mask;
struct page_info *pg;
ASSERT(node >= 0);
@@ -403,6 +404,29 @@ static struct page_info *alloc_heap_page
spin_unlock(&heap_lock);
+ cpus_clear(mask);
+
+ for ( i = 0; i < (1 << order); i++ )
+ {
+ /* Reference count must continuously be zero for free pages. */
+ BUG_ON(pg[i].count_info != 0);
+
+ /* Add in any extra CPUs that need flushing because of this page. */
+ cpus_andnot(extra_cpus_mask, pg[i].u.free.cpumask, mask);
+ tlbflush_filter(extra_cpus_mask, pg[i].tlbflush_timestamp);
+ cpus_or(mask, mask, extra_cpus_mask);
+
+ /* Initialise fields which have other uses for free pages. */
+ pg[i].u.inuse.type_info = 0;
+ page_set_owner(&pg[i], NULL);
+ }
+
+ if ( unlikely(!cpus_empty(mask)) )
+ {
+ perfc_incrc(need_flush_tlb_flush);
+ flush_tlb_mask(mask);
+ }
+
return pg;
}
@@ -411,12 +435,27 @@ static void free_heap_pages(
unsigned int zone, struct page_info *pg, unsigned int order)
{
unsigned long mask;
- unsigned int node = phys_to_nid(page_to_maddr(pg));
+ unsigned int i, node = phys_to_nid(page_to_maddr(pg));
+ struct domain *d;
ASSERT(zone < NR_ZONES);
ASSERT(order <= MAX_ORDER);
ASSERT(node >= 0);
ASSERT(node < num_online_nodes());
+
+ for ( i = 0; i < (1 << order); i++ )
+ {
+ BUG_ON(pg[i].count_info != 0);
+ if ( (d = page_get_owner(&pg[i])) != NULL )
+ {
+ pg[i].tlbflush_timestamp = tlbflush_current_time();
+ pg[i].u.free.cpumask = d->domain_dirty_cpumask;
+ }
+ else
+ {
+ cpus_clear(pg[i].u.free.cpumask);
+ }
+ }
spin_lock(&heap_lock);
@@ -554,7 +593,7 @@ void end_boot_allocator(void)
/*
* Scrub all unallocated pages in all heap zones. This function is more
* convoluted than appears necessary because we do not want to continuously
- * hold the lock or disable interrupts while scrubbing very large memory areas.
+ * hold the lock while scrubbing very large memory areas.
*/
void scrub_heap_pages(void)
{
@@ -575,7 +614,7 @@ void scrub_heap_pages(void)
if ( (mfn % ((100*1024*1024)/PAGE_SIZE)) == 0 )
printk(".");
- spin_lock_irq(&heap_lock);
+ spin_lock(&heap_lock);
/* Re-check page status with lock held. */
if ( !allocated_in_map(mfn) )
@@ -595,7 +634,7 @@ void scrub_heap_pages(void)
}
}
- spin_unlock_irq(&heap_lock);
+ spin_unlock(&heap_lock);
}
printk("done.\n");
@@ -609,8 +648,6 @@ void scrub_heap_pages(void)
void init_xenheap_pages(paddr_t ps, paddr_t pe)
{
- unsigned long flags;
-
ps = round_pgup(ps);
pe = round_pgdown(pe);
if ( pe <= ps )
@@ -625,33 +662,21 @@ void init_xenheap_pages(paddr_t ps, padd
if ( !IS_XEN_HEAP_FRAME(maddr_to_page(pe)) )
pe -= PAGE_SIZE;
- local_irq_save(flags);
init_heap_pages(MEMZONE_XEN, maddr_to_page(ps), (pe - ps) >> PAGE_SHIFT);
- local_irq_restore(flags);
}
void *alloc_xenheap_pages(unsigned int order)
{
- unsigned long flags;
struct page_info *pg;
- int i;
-
- local_irq_save(flags);
+
+ ASSERT(!in_irq());
+
pg = alloc_heap_pages(MEMZONE_XEN, MEMZONE_XEN, smp_processor_id(), order);
- local_irq_restore(flags);
-
if ( unlikely(pg == NULL) )
goto no_memory;
memguard_unguard_range(page_to_virt(pg), 1 << (order + PAGE_SHIFT));
-
- for ( i = 0; i < (1 << order); i++ )
- {
- pg[i].count_info = 0;
- pg[i].u.inuse._domain = 0;
- pg[i].u.inuse.type_info = 0;
- }
return page_to_virt(pg);
@@ -663,16 +688,14 @@ void *alloc_xenheap_pages(unsigned int o
void free_xenheap_pages(void *v, unsigned int order)
{
- unsigned long flags;
+ ASSERT(!in_irq());
if ( v == NULL )
return;
- memguard_guard_range(v, 1 << (order + PAGE_SHIFT));
-
- local_irq_save(flags);
+ memguard_guard_range(v, 1 << (order + PAGE_SHIFT));
+
free_heap_pages(MEMZONE_XEN, virt_to_page(v), order);
- local_irq_restore(flags);
}
@@ -762,8 +785,6 @@ struct page_info *__alloc_domheap_pages(
unsigned int memflags)
{
struct page_info *pg = NULL;
- cpumask_t mask;
- unsigned long i;
unsigned int bits = memflags >> _MEMF_bits, zone_hi = NR_ZONES - 1;
ASSERT(!in_irq());
@@ -792,38 +813,10 @@ struct page_info *__alloc_domheap_pages(
return NULL;
}
- if ( pg == NULL )
- if ( (pg = alloc_heap_pages(MEMZONE_XEN + 1,
- zone_hi,
- cpu, order)) == NULL )
- return NULL;
-
- mask = pg->u.free.cpumask;
- tlbflush_filter(mask, pg->tlbflush_timestamp);
-
- pg->count_info = 0;
- pg->u.inuse._domain = 0;
- pg->u.inuse.type_info = 0;
-
- for ( i = 1; i < (1 << order); i++ )
- {
- /* Add in any extra CPUs that need flushing because of this page. */
- cpumask_t extra_cpus_mask;
- cpus_andnot(extra_cpus_mask, pg[i].u.free.cpumask, mask);
- tlbflush_filter(extra_cpus_mask, pg[i].tlbflush_timestamp);
- cpus_or(mask, mask, extra_cpus_mask);
-
- pg[i].count_info = 0;
- pg[i].u.inuse._domain = 0;
- pg[i].u.inuse.type_info = 0;
- page_set_owner(&pg[i], NULL);
- }
-
- if ( unlikely(!cpus_empty(mask)) )
- {
- perfc_incrc(need_flush_tlb_flush);
- flush_tlb_mask(mask);
- }
+ if ( (pg == NULL) &&
+ ((pg = alloc_heap_pages(MEMZONE_XEN + 1, zone_hi,
+ cpu, order)) == NULL) )
+ return NULL;
if ( (d != NULL) && assign_pages(d, pg, order, memflags) )
{
@@ -867,10 +860,7 @@ void free_domheap_pages(struct page_info
for ( i = 0; i < (1 << order); i++ )
{
- shadow_drop_references(d, &pg[i]);
- ASSERT((pg[i].u.inuse.type_info & PGT_count_mask) == 0);
- pg[i].tlbflush_timestamp = tlbflush_current_time();
- pg[i].u.free.cpumask = d->domain_dirty_cpumask;
+ BUG_ON((pg[i].u.inuse.type_info & PGT_count_mask) != 0);
list_del(&pg[i].list);
}
@@ -892,6 +882,7 @@ void free_domheap_pages(struct page_info
*/
for ( i = 0; i < (1 << order); i++ )
{
+ page_set_owner(&pg[i], NULL);
spin_lock(&page_scrub_lock);
list_add(&pg[i].list, &page_scrub_list);
scrub_pages++;
@@ -902,8 +893,6 @@ void free_domheap_pages(struct page_info
else
{
/* Freeing anonymous domain-heap pages. */
- for ( i = 0; i < (1 << order); i++ )
- cpus_clear(pg[i].u.free.cpumask);
free_heap_pages(pfn_dom_zone_type(page_to_mfn(pg)), pg, order);
drop_dom_ref = 0;
}
diff -r a7f6392ea850 -r 18cf0c56226d xen/common/xmalloc.c
--- a/xen/common/xmalloc.c Thu Mar 08 10:54:56 2007 +0000
+++ b/xen/common/xmalloc.c Thu Mar 08 14:29:09 2007 +0000
@@ -33,6 +33,8 @@
#include <xen/timer.h>
#include <xen/cache.h>
#include <xen/prefetch.h>
+#include <xen/irq.h>
+#include <xen/smp.h>
/*
* XMALLOC_DEBUG:
@@ -175,6 +177,8 @@ void *_xmalloc(size_t size, size_t align
struct xmalloc_hdr *i;
unsigned long flags;
+ ASSERT(!in_irq());
+
/* We currently always return cacheline aligned. */
BUG_ON(align > SMP_CACHE_BYTES);
@@ -212,6 +216,8 @@ void xfree(void *p)
{
unsigned long flags;
struct xmalloc_hdr *i, *tmp, *hdr;
+
+ ASSERT(!in_irq());
if ( p == NULL )
return;
_______________________________________________
Xen-changelog mailing list
Xen-changelog@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-changelog
|
Home