diff -r 4ac8bc60c000 xen/arch/x86/mm.c --- a/xen/arch/x86/mm.c Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/arch/x86/mm.c Thu Feb 19 14:58:13 2009 -0700 @@ -3469,6 +3469,42 @@ int replace_grant_host_mapping( return rc; } +int donate_page( + struct domain *d, struct page_info *page, unsigned int memflags) +{ + spin_lock(&d->page_alloc_lock); + + if ( is_xen_heap_page(page) || (page_get_owner(page) != NULL) ) + goto fail; + + if ( d->is_dying ) + goto fail; + + if ( page->count_info & ~(PGC_allocated | 1) ) + goto fail; + + if ( !(memflags & MEMF_no_refcount) ) + { + if ( d->tot_pages >= d->max_pages ) + goto fail; + d->tot_pages++; + } + + page->count_info = PGC_allocated | 1; + page_set_owner(page, d); + page_list_add_tail(page,&d->page_list); + + spin_unlock(&d->page_alloc_lock); + return 0; + + fail: + spin_unlock(&d->page_alloc_lock); + MEM_LOG("Bad page %p: ed=%p(%u), sd=%p, caf=%08lx, taf=%" PRtype_info, + (void *)page_to_mfn(page), d, d->domain_id, + page_get_owner(page), page->count_info, page->u.inuse.type_info); + return -1; +} + int steal_page( struct domain *d, struct page_info *page, unsigned int memflags) { diff -r 4ac8bc60c000 xen/arch/x86/setup.c --- a/xen/arch/x86/setup.c Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/arch/x86/setup.c Thu Feb 19 14:58:13 2009 -0700 @@ -109,6 +109,7 @@ extern void early_cpu_init(void); extern void early_cpu_init(void); extern void vesa_init(void); extern void vesa_mtrr_init(void); +extern void init_tmem(void); DEFINE_PER_CPU(struct desc_struct *, gdt_table) = boot_cpu_gdt_table; #ifdef CONFIG_COMPAT @@ -1057,6 +1058,8 @@ void __init __start_xen(unsigned long mb init_trace_bufs(); + init_tmem(); + console_endboot(); /* Hide UART from DOM0 if we're using it */ diff -r 4ac8bc60c000 xen/arch/x86/x86_32/entry.S --- a/xen/arch/x86/x86_32/entry.S Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/arch/x86/x86_32/entry.S Thu Feb 19 14:58:13 2009 -0700 @@ -703,6 +703,7 @@ ENTRY(hypercall_table) .long do_sysctl /* 35 */ .long do_domctl .long do_kexec_op + .long do_tmem_op .rept __HYPERVISOR_arch_0-((.-hypercall_table)/4) .long do_ni_hypercall .endr @@ -750,6 +751,7 @@ ENTRY(hypercall_args_table) .byte 1 /* do_sysctl */ /* 35 */ .byte 1 /* do_domctl */ .byte 2 /* do_kexec_op */ + .byte 1 /* do_tmem_op */ .rept __HYPERVISOR_arch_0-(.-hypercall_args_table) .byte 0 /* do_ni_hypercall */ .endr diff -r 4ac8bc60c000 xen/arch/x86/x86_64/compat/entry.S --- a/xen/arch/x86/x86_64/compat/entry.S Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/arch/x86/x86_64/compat/entry.S Thu Feb 19 14:58:13 2009 -0700 @@ -408,6 +408,7 @@ ENTRY(compat_hypercall_table) .quad do_sysctl /* 35 */ .quad do_domctl .quad compat_kexec_op + .quad do_tmem_op .rept __HYPERVISOR_arch_0-((.-compat_hypercall_table)/8) .quad compat_ni_hypercall .endr @@ -455,6 +456,7 @@ ENTRY(compat_hypercall_args_table) .byte 1 /* do_sysctl */ /* 35 */ .byte 1 /* do_domctl */ .byte 2 /* compat_kexec_op */ + .byte 1 /* do_tmem_op */ .rept __HYPERVISOR_arch_0-(.-compat_hypercall_args_table) .byte 0 /* compat_ni_hypercall */ .endr diff -r 4ac8bc60c000 xen/arch/x86/x86_64/entry.S --- a/xen/arch/x86/x86_64/entry.S Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/arch/x86/x86_64/entry.S Thu Feb 19 14:58:13 2009 -0700 @@ -692,6 +692,7 @@ ENTRY(hypercall_table) .quad do_sysctl /* 35 */ .quad do_domctl .quad do_kexec_op + .quad do_tmem_op .rept __HYPERVISOR_arch_0-((.-hypercall_table)/8) .quad do_ni_hypercall .endr @@ -739,6 +740,7 @@ ENTRY(hypercall_args_table) .byte 1 /* do_sysctl */ /* 35 */ .byte 1 /* do_domctl */ .byte 2 /* do_kexec */ + .byte 1 /* do_tmem_op */ .rept __HYPERVISOR_arch_0-(.-hypercall_args_table) .byte 0 /* do_ni_hypercall */ .endr diff -r 4ac8bc60c000 xen/common/Makefile --- a/xen/common/Makefile Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/common/Makefile Thu Feb 19 14:58:13 2009 -0700 @@ -28,6 +28,10 @@ obj-y += vsprintf.o obj-y += vsprintf.o obj-y += xmalloc_tlsf.o obj-y += rcupdate.o +obj-y += tmem.o +obj-y += tmem_xen.o +obj-y += radix-tree.o +obj-y += lzo.o obj-$(perfc) += perfc.o obj-$(crash_debug) += gdbstub.o diff -r 4ac8bc60c000 xen/common/compat/Makefile --- a/xen/common/compat/Makefile Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/common/compat/Makefile Thu Feb 19 14:58:13 2009 -0700 @@ -3,3 +3,4 @@ obj-y += memory.o obj-y += memory.o obj-y += multicall.o obj-y += xlat.o +obj-y += tmem_xen.o diff -r 4ac8bc60c000 xen/common/domain.c --- a/xen/common/domain.c Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/common/domain.c Thu Feb 19 14:58:13 2009 -0700 @@ -30,6 +30,7 @@ #include #include #include +#include /* Linux config option: propageted to domain0 */ /* xen_processor_pmbits: xen control Cx, Px, ... */ @@ -555,6 +556,9 @@ static void complete_domain_destroy(stru grant_table_destroy(d); + if ( d->tmem != NULL ) + tmem_destroy(d->tmem); + arch_domain_destroy(d); rangeset_domain_destroy(d); diff -r 4ac8bc60c000 xen/common/page_alloc.c --- a/xen/common/page_alloc.c Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/common/page_alloc.c Thu Feb 19 14:58:13 2009 -0700 @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -328,9 +329,9 @@ static unsigned long init_node_heap(int /* Allocate 2^@order contiguous pages. */ static struct page_info *alloc_heap_pages( unsigned int zone_lo, unsigned int zone_hi, - unsigned int node, unsigned int order) + unsigned int node, unsigned int order, unsigned int memflags) { - unsigned int i, j, zone; + unsigned int i, j, zone = 0; unsigned int num_nodes = num_online_nodes(); unsigned long request = 1UL << order; cpumask_t extra_cpus_mask, mask; @@ -371,6 +372,14 @@ static struct page_info *alloc_heap_page /* Pick next node, wrapping around if needed. */ if ( ++node == num_nodes ) node = 0; + } + + /* Try to free memory from tmem */ + if ( (pg = tmem_relinquish_pages(order,memflags)) != NULL ) + { + /* reassigning an already allocated anonymous heap page */ + spin_unlock(&heap_lock); + return pg; } /* No suitable memory blocks. Fail the request. */ @@ -693,8 +702,8 @@ void *alloc_xenheap_pages(unsigned int o ASSERT(!in_irq()); - pg = alloc_heap_pages( - MEMZONE_XEN, MEMZONE_XEN, cpu_to_node(smp_processor_id()), order); + pg = alloc_heap_pages(MEMZONE_XEN, MEMZONE_XEN, + cpu_to_node(smp_processor_id()), order, memflags); if ( unlikely(pg == NULL) ) return NULL; @@ -800,8 +809,11 @@ int assign_pages( { if ( unlikely((d->tot_pages + (1 << order)) > d->max_pages) ) { +#if 0 + /* too noisy if running tmem */ gdprintk(XENLOG_INFO, "Over-allocation for domain %u: %u > %u\n", d->domain_id, d->tot_pages + (1 << order), d->max_pages); +#endif goto fail; } @@ -847,11 +859,11 @@ struct page_info *alloc_domheap_pages( return NULL; if ( dma_bitsize && ((dma_zone = bits_to_zone(dma_bitsize)) < zone_hi) ) - pg = alloc_heap_pages(dma_zone + 1, zone_hi, node, order); + pg = alloc_heap_pages(dma_zone + 1, zone_hi, node, order, memflags); if ( (pg == NULL) && ((pg = alloc_heap_pages(MEMZONE_XEN + 1, zone_hi, - node, order)) == NULL) ) + node, order, memflags)) == NULL) ) return NULL; if ( (d != NULL) && assign_pages(d, pg, order, memflags) ) @@ -1048,6 +1060,24 @@ static void page_scrub_softirq(void) spin_unlock(&serialise_lock); } +void scrub_list_splice(struct page_info *pg) +{ + page_list_splice(pg, &page_scrub_list); +} + +void scrub_list_add(struct page_info *pg) +{ + page_list_add(pg, &page_scrub_list); +} + +void scrub_one_page(struct page_info *pg) +{ + void *p = map_domain_page(page_to_mfn(pg)); + + scrub_page(p); + unmap_domain_page(p); +} + static void page_scrub_timer_fn(void *unused) { page_scrub_schedule_work(); diff -r 4ac8bc60c000 xen/common/xmalloc_tlsf.c --- a/xen/common/xmalloc_tlsf.c Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/common/xmalloc_tlsf.c Thu Feb 19 14:58:13 2009 -0700 @@ -320,10 +320,11 @@ struct xmem_pool *xmem_pool_create( pool->put_mem = put_mem; strlcpy(pool->name, name, sizeof(pool->name)); region = get_mem(init_size); - if ( region == NULL ) - goto out_region; - ADD_REGION(region, init_size, pool); - pool->init_region = region; + if ( region != NULL ) + { + ADD_REGION(region, init_size, pool); + pool->init_region = region; + } spin_lock_init(&pool->lock); @@ -332,10 +333,6 @@ struct xmem_pool *xmem_pool_create( spin_unlock(&pool_list_lock); return pool; - - out_region: - free_xenheap_pages(pool, pool_order); - return NULL; } unsigned long xmem_pool_get_used_size(struct xmem_pool *pool) @@ -360,8 +357,11 @@ void xmem_pool_destroy(struct xmem_pool /* User is destroying without ever allocating from this pool */ if ( xmem_pool_get_used_size(pool) == BHDR_OVERHEAD ) { - pool->put_mem(pool->init_region); - pool->used_size -= BHDR_OVERHEAD; + if ( pool->init_region ) + { + pool->put_mem(pool->init_region); + pool->used_size -= BHDR_OVERHEAD; + } } /* Check for memory leaks in this pool */ @@ -381,6 +381,14 @@ void *xmem_pool_alloc(unsigned long size struct bhdr *b, *b2, *next_b, *region; int fl, sl; unsigned long tmp_size; + + if ( pool->init_region == NULL ) + { + if ( (region = pool->get_mem(pool->init_size)) == NULL ) + goto out; + ADD_REGION(region, pool->init_size, pool); + pool->init_region = region; + } size = (size < MIN_BLOCK_SIZE) ? MIN_BLOCK_SIZE : ROUNDUP_SIZE(size); /* Rounding up the requested size and calculating fl and sl */ @@ -496,6 +504,11 @@ void xmem_pool_free(void *ptr, struct xm spin_unlock(&pool->lock); } +int xmem_pool_maxalloc(struct xmem_pool *pool) +{ + return pool->grow_size - (2 * BHDR_OVERHEAD); +} + /* * Glue for xmalloc(). */ diff -r 4ac8bc60c000 xen/include/Makefile --- a/xen/include/Makefile Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/include/Makefile Thu Feb 19 14:58:13 2009 -0700 @@ -14,6 +14,7 @@ headers-y := \ compat/physdev.h \ compat/platform.h \ compat/sched.h \ + compat/tmem.h \ compat/trace.h \ compat/vcpu.h \ compat/version.h \ diff -r 4ac8bc60c000 xen/include/asm-x86/mm.h --- a/xen/include/asm-x86/mm.h Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/include/asm-x86/mm.h Thu Feb 19 14:58:13 2009 -0700 @@ -466,6 +466,8 @@ int compat_subarch_memory_op(int op, XEN int steal_page( struct domain *d, struct page_info *page, unsigned int memflags); +int donate_page( + struct domain *d, struct page_info *page, unsigned int memflags); int map_ldt_shadow_page(unsigned int); diff -r 4ac8bc60c000 xen/include/public/xen.h --- a/xen/include/public/xen.h Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/include/public/xen.h Thu Feb 19 14:58:13 2009 -0700 @@ -91,6 +91,7 @@ DEFINE_XEN_GUEST_HANDLE(xen_pfn_t); #define __HYPERVISOR_sysctl 35 #define __HYPERVISOR_domctl 36 #define __HYPERVISOR_kexec_op 37 +#define __HYPERVISOR_tmem_op 38 /* Architecture-specific hypercall definitions. */ #define __HYPERVISOR_arch_0 48 diff -r 4ac8bc60c000 xen/include/xen/hypercall.h --- a/xen/include/xen/hypercall.h Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/include/xen/hypercall.h Thu Feb 19 14:58:13 2009 -0700 @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -117,6 +118,10 @@ do_xsm_op( do_xsm_op( XEN_GUEST_HANDLE(xsm_op_t) u_xsm_op); +extern long +do_tmem_op( + XEN_GUEST_HANDLE(tmem_op_t) uops); + #ifdef CONFIG_COMPAT extern int diff -r 4ac8bc60c000 xen/include/xen/mm.h --- a/xen/include/xen/mm.h Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/include/xen/mm.h Thu Feb 19 14:58:13 2009 -0700 @@ -74,6 +74,8 @@ int assign_pages( #define MEMF_no_refcount (1U<<_MEMF_no_refcount) #define _MEMF_populate_on_demand 1 #define MEMF_populate_on_demand (1U<<_MEMF_populate_on_demand) +#define _MEMF_no_scrub 2 +#define MEMF_no_scrub (1U<<_MEMF_no_scrub) #define _MEMF_node 8 #define MEMF_node(n) ((((n)+1)&0xff)<<_MEMF_node) #define _MEMF_bits 24 @@ -219,6 +221,12 @@ page_list_remove_head(struct page_list_h return page; } +static inline void +page_list_splice(struct page_info *page, struct page_list_head *head) +{ + BUG_ON(1); //FIXME +} + #define page_list_for_each(pos, head) \ for ( pos = (head)->next; pos; pos = page_list_next(pos, head) ) #define page_list_for_each_safe(pos, tmp, head) \ @@ -255,6 +263,7 @@ page_list_remove_head(struct page_list_h list_for_each_entry_safe(pos, tmp, head, list) # define page_list_for_each_safe_reverse(pos, tmp, head) \ list_for_each_entry_safe_reverse(pos, tmp, head, list) +# define page_list_splice(pg, hd) list_splice(&(pg)->list, hd) #endif /* Automatic page scrubbing for dead domains. */ @@ -269,9 +278,13 @@ extern struct page_list_head page_scrub_ if ( !page_list_empty(&page_scrub_list) ) \ cpumask_raise_softirq(cpu_online_map, PAGE_SCRUB_SOFTIRQ); \ } while ( 0 ) +void scrub_list_splice(struct page_info *); +void scrub_list_add(struct page_info *); +void scrub_one_page(struct page_info *); unsigned long avail_scrub_pages(void); int guest_remove_page(struct domain *d, unsigned long gmfn); + /* Returns TRUE if the whole page at @mfn is ordinary RAM. */ int page_is_conventional_ram(unsigned long mfn); diff -r 4ac8bc60c000 xen/include/xen/sched.h --- a/xen/include/xen/sched.h Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/include/xen/sched.h Thu Feb 19 14:58:13 2009 -0700 @@ -267,6 +267,9 @@ struct domain /* VRAM dirty support. */ struct sh_dirty_vram *dirty_vram; + + /* transcendent memory, auto-allocated on first tmem op by each domain */ + void *tmem; }; struct domain_setup_info diff -r 4ac8bc60c000 xen/include/xen/xmalloc.h --- a/xen/include/xen/xmalloc.h Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/include/xen/xmalloc.h Thu Feb 19 14:58:13 2009 -0700 @@ -76,7 +76,13 @@ void *xmem_pool_alloc(unsigned long size void *xmem_pool_alloc(unsigned long size, struct xmem_pool *pool); /** - * xmem_pool_free - free memory from given pool + * xmem_pool_maxalloc - xmem_pool_alloc's greater than this size will fail + * @mem_pool: pool + */ +int xmem_pool_maxalloc(struct xmem_pool *pool); + +/** + * xmem_pool_maxsize - * @ptr: address of memory to be freed * @mem_pool: pool to free from */ diff -r 4ac8bc60c000 xen/include/xlat.lst --- a/xen/include/xlat.lst Tue Feb 10 05:51:00 2009 +0000 +++ b/xen/include/xlat.lst Thu Feb 19 14:58:13 2009 -0700 @@ -58,3 +58,6 @@ ? processor_px platform.h ! psd_package platform.h ! processor_performance platform.h +# ? tmem_op_t tmem.h +# ? tmem_cli_mfn_t tmem.h +# ? tmem_cli_va_t tmem.h