Tim,
thanks for push in save/restore patches, but this one for save/restore PV
driver
on HVM seems to be missing.
if any concerns about this patch, pls. let me know.
thanks,
On Thu, Jan 11, 2007 at 10:13:33PM +0800, Zhai, Edwin wrote:
> [PATCH 8/8] HVM save restore: PV driver support
>
> Signed-off-by: Zhai Edwin <edwin.zhai@xxxxxxxxx>
>
> enable PV driver's save/restore in HVM domain by:
> HV:
> * send a pseudo PCI dev intr to guest in restore
> * rebuild the shared info on behalf of HVM guest
> * set a resume flag in shared info
>
> Guest:
> * check the resmume flag in the pseudo PCI dev intr handler
> * if set do the PV driver resume work
>
>
> diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/blkfront/blkfront.c
> --- a/linux-2.6-xen-sparse/drivers/xen/blkfront/blkfront.c Thu Jan 11
> 17:03:17 2007 +0800
> +++ b/linux-2.6-xen-sparse/drivers/xen/blkfront/blkfront.c Thu Jan 11
> 17:03:21 2007 +0800
> @@ -138,6 +138,10 @@ static int blkfront_resume(struct xenbus
>
> DPRINTK("blkfront_resume: %s\n", dev->nodename);
>
> + /* resuming qemu disk would cause error when restore and skip next vbd
> resume */
> + if (info->connected == BLKIF_STATE_DISCONNECTED)
> + return 0;
> +
> blkif_free(info, info->connected == BLKIF_STATE_CONNECTED);
>
> err = talk_to_backend(dev, info);
> diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/blkfront/vbd.c
> --- a/linux-2.6-xen-sparse/drivers/xen/blkfront/vbd.c Thu Jan 11 17:03:17
> 2007 +0800
> +++ b/linux-2.6-xen-sparse/drivers/xen/blkfront/vbd.c Thu Jan 11 17:03:21
> 2007 +0800
> @@ -183,7 +183,16 @@ xlbd_put_major_info(struct xlbd_major_in
> xlbd_put_major_info(struct xlbd_major_info *mi)
> {
> mi->usage--;
> - /* XXX: release major if 0 */
> + /* release major if 0 */
> + if (mi->usage)
> + return;
> +
> + printk("Unregistering block device major %i\n", mi->major);
> + if (unregister_blkdev(mi->major, mi->type->devname)) {
> + WPRINTK("can't put major %d with name %s\n",
> + mi->major, mi->type->devname);
> + kfree(mi);
> + }
> }
>
> static int
> diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c
> --- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c Thu Jan 11
> 17:03:17 2007 +0800
> +++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c Thu Jan 11
> 17:03:21 2007 +0800
> @@ -815,6 +815,7 @@ static int xsd_port_read(char *page, cha
> }
> #endif
>
> +extern void set_restore_handler( void (*hdl1)(void), void (*hdl2)(void));
> static int __init xenbus_probe_init(void)
> {
> int err = 0;
> @@ -882,6 +883,7 @@ static int __init xenbus_probe_init(void
> xen_store_mfn = hvm_get_parameter(HVM_PARAM_STORE_PFN);
> xen_store_interface = ioremap(xen_store_mfn << PAGE_SHIFT,
> PAGE_SIZE);
> + set_restore_handler(xenbus_suspend, xenbus_resume);
> #endif
> }
>
> diff -r d18c6a3c676a unmodified_drivers/linux-2.6/platform-pci/evtchn.c
> --- a/unmodified_drivers/linux-2.6/platform-pci/evtchn.c Thu Jan 11
> 17:03:17 2007 +0800
> +++ b/unmodified_drivers/linux-2.6/platform-pci/evtchn.c Thu Jan 11
> 20:12:33 2007 +0800
> @@ -165,6 +165,7 @@ void notify_remote_via_irq(int irq)
> }
> EXPORT_SYMBOL(notify_remote_via_irq);
>
> +extern void pvdrv_restore(void);
> irqreturn_t evtchn_interrupt(int irq, void *dev_id, struct pt_regs *regs)
> {
> unsigned int l1i, port;
> @@ -173,6 +174,16 @@ irqreturn_t evtchn_interrupt(int irq, vo
> shared_info_t *s = shared_info_area;
> vcpu_info_t *v = &s->vcpu_info[cpu];
> unsigned long l1, l2;
> +
> + /* add a check to see if need resume after restore */
> + if (s->pvdrv_resume == 0x58585858) {
> + printk("evtchn_interrupt:resume PV driver.\n");
> + s->pvdrv_resume= 0;
> + v->evtchn_upcall_pending = 0;
> +
> + pvdrv_restore();
> + return IRQ_HANDLED;
> + }
>
> v->evtchn_upcall_pending = 0;
> /* NB. No need for a barrier here -- XCHG is a barrier on x86. */
> diff -r d18c6a3c676a unmodified_drivers/linux-2.6/platform-pci/platform-pci.c
> --- a/unmodified_drivers/linux-2.6/platform-pci/platform-pci.c Thu Jan
> 11 17:03:17 2007 +0800
> +++ b/unmodified_drivers/linux-2.6/platform-pci/platform-pci.c Thu Jan
> 11 17:03:21 2007 +0800
> @@ -28,6 +28,7 @@
> #include <linux/interrupt.h>
> #include <linux/vmalloc.h>
> #include <linux/mm.h>
> +#include <linux/kthread.h>
> #include <asm/system.h>
> #include <asm/io.h>
> #include <asm/irq.h>
> @@ -280,9 +281,85 @@ static struct pci_driver platform_driver
>
> static int pci_device_registered;
>
> +extern int gnttab_suspend(void);
> +void platform_pci_suspend(void)
> +{
> + gnttab_suspend();
> +}
> +
> +extern int gnttab_resume(void);
> +void platform_pci_resume(void)
> +{
> + phys_to_machine_mapping = NULL;
> +
> + gnttab_resume();
> +}
> +
> +/* support for PV driver save/restore in HVM domain*/
> +void (*suspend_handler)(void);
> +void (*resume_handler)(void);
> +void set_restore_handler( void (*hdl1)(void), void (*hdl2)(void))
> +{
> + suspend_handler = hdl1;
> + resume_handler = hdl2;
> +}
> +EXPORT_SYMBOL(set_restore_handler);
> +
> +/* scheduled suspend&resume */
> +static void restore_handler(void *unused);
> +static int __do_pvdrv_restore(void *ignored);
> +
> +static DECLARE_WORK(restore_work, restore_handler, NULL);
> +
> +static int kthread_create_on_cpu(int (*f)(void *arg),
> + void *arg,
> + const char *name,
> + int cpu)
> +{
> + struct task_struct *p;
> + p = kthread_create(f, arg, name);
> + if (IS_ERR(p))
> + return PTR_ERR(p);
> + kthread_bind(p, cpu);
> + wake_up_process(p);
> + return 0;
> +}
> +
> +static int __do_pvdrv_restore(void *ignored)
> +{
> + if (suspend_handler)
> + suspend_handler();
> + platform_pci_suspend();
> +
> + platform_pci_resume();
> + if (resume_handler)
> + resume_handler();
> +
> + return 0;
> +}
> +
> +static void restore_handler(void *unused)
> +{
> + int err;
> +
> + err = kthread_create_on_cpu(__do_pvdrv_restore, NULL, "pvdrv_suspend",
> 0);
> + if (err < 0) {
> + printk(KERN_WARNING "error creating PV driver suspend process
> (%d): retrying...\n",
> + err);
> + schedule_delayed_work(&restore_work, 50/2);
> + }
> +}
> +
> +void pvdrv_restore(void)
> +{
> + schedule_work(&restore_work);
> +}
> +
> static int __init platform_pci_module_init(void)
> {
> int rc;
> +
> + set_restore_handler(NULL, NULL);
>
> rc = pci_module_init(&platform_driver);
> if (rc)
> diff -r d18c6a3c676a xen/arch/x86/hvm/hvm.c
> --- a/xen/arch/x86/hvm/hvm.c Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/arch/x86/hvm/hvm.c Thu Jan 11 17:03:21 2007 +0800
> @@ -200,6 +200,9 @@ int hvm_vcpu_initialise(struct vcpu *v)
>
> /* init hvm sharepage */
> shpage_init(v->domain, get_sp(v->domain));
> +
> + /* other hvm info need for save/restore */
> + hvminfo_init(v->domain);
>
> /* Init guest TSC to start from zero. */
> hvm_set_guest_time(v, 0);
> diff -r d18c6a3c676a xen/arch/x86/hvm/intercept.c
> --- a/xen/arch/x86/hvm/intercept.c Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/arch/x86/hvm/intercept.c Thu Jan 11 17:03:37 2007 +0800
> @@ -445,6 +445,71 @@ void shpage_init(struct domain *d, share
> hvm_register_savevm(d, "xen_hvm_shpage", 0x10, 1, shpage_save,
> shpage_load, sp);
> }
>
> +void hvminfo_print(struct domain* d)
> +{
> + printk("********hvm other info***********\n");
> + printk("callback irq=%"PRId64".\n",
> d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ]);
> + printk("shared info pfn=0x%"PRIx64".\n",
> d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN]);
> +}
> +
> +static void hvminfo_save(hvm_domain_context_t *h, void *opaque)
> +{
> + struct domain *d = opaque;
> +
> +#ifdef HVM_DEBUG_SUSPEND
> + hvminfo_print(d);
> +#endif
> +
> + hvm_put_64u(h, d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ]);
> + hvm_put_64u(h, d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN]);
> +}
> +
> +extern int rebuild_shared_info(struct domain *d, unsigned long
> share_info_pfn);
> +static int hvminfo_load(hvm_domain_context_t *h, void *opaque, int
> version_id)
> +{
> + struct domain *d = opaque;
> + int callback_irq = hvm_get_64u(h);
> + unsigned long share_info_pfn = hvm_get_64u(h);
> +
> + if (callback_irq == 0) {
> + return 0;
> + }
> +
> + d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ] = callback_irq;
> +
> + /* reconstruct the share info for HVM guest */
> + rebuild_shared_info(d, share_info_pfn);
> + d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN] = share_info_pfn;
> +
> + /* set resume state to notify HVM PV driver */
> + shared_info(d, pvdrv_resume) = 0x58585858;
> +
> + /* set all the evtchn_upcall_pending on all vcpus */
> + if (callback_irq) {
> + struct vcpu *v;
> +
> + for_each_vcpu(d, v) {
> + if ( !test_and_set_bit(0, &vcpu_info(v, evtchn_upcall_pending)) )
> + printk("on vcpu %d raise a vir level intr %d to restore PV
> driver in HVM guest!\n",
> + v->vcpu_id,
> + callback_irq);
> + }
> + }
> +
> +
> +#ifdef HVM_DEBUG_SUSPEND
> + hvminfo_print(d);
> +#endif
> +
> + return 0;
> +
> +}
> +
> +void hvminfo_init(struct domain* d)
> +{
> + hvm_register_savevm(d, "xen_hvm_other_info", 0x11, 1, hvminfo_save,
> hvminfo_load, d);
> +}
> +
> int hvm_buffered_io_intercept(ioreq_t *p)
> {
> struct vcpu *v = current;
> diff -r d18c6a3c676a xen/arch/x86/mm.c
> --- a/xen/arch/x86/mm.c Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/arch/x86/mm.c Thu Jan 11 17:03:21 2007 +0800
> @@ -3000,6 +3000,7 @@ long arch_memory_op(int op, XEN_GUEST_HA
> case XENMAPSPACE_shared_info:
> if ( xatp.idx == 0 )
> mfn = virt_to_mfn(d->shared_info);
> + d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN] = xatp.gpfn;
> break;
> case XENMAPSPACE_grant_table:
> if ( xatp.idx < NR_GRANT_FRAMES )
> @@ -3143,6 +3144,43 @@ long arch_memory_op(int op, XEN_GUEST_HA
> return 0;
> }
>
> +/* reconstruct the share info for HVM guest */
> +int rebuild_shared_info(struct domain *d, unsigned long share_info_pfn)
> +{
> + unsigned long mfn, prev_mfn, gpfn = 0;
> +
> + mfn = virt_to_mfn(d->shared_info);
> + if ( !shadow_mode_translate(d) || (mfn == 0)) {
> + printk("reconstruct share info for HVM guest failed!\n");
> + return -1;
> + }
> +
> + LOCK_BIGLOCK(d);
> +
> + /* Remove previously mapped page if it was present. */
> + prev_mfn = gmfn_to_mfn(d, share_info_pfn);
> + if ( mfn_valid(prev_mfn) )
> + {
> + if ( IS_XEN_HEAP_FRAME(mfn_to_page(prev_mfn)) )
> + /* Xen heap frames are simply unhooked from this phys slot. */
> + guest_physmap_remove_page(d, share_info_pfn, prev_mfn);
> + else
> + /* Normal domain memory is freed, to avoid leaking memory. */
> + guest_remove_page(d, share_info_pfn);
> + }
> +
> + /* Unmap from old location, if any. */
> + gpfn = get_gpfn_from_mfn(mfn);
> + if ( gpfn != INVALID_M2P_ENTRY )
> + guest_physmap_remove_page(d, gpfn, mfn);
> +
> + /* Map at new location. */
> + guest_physmap_add_page(d, share_info_pfn, mfn);
> +
> + UNLOCK_BIGLOCK(d);
> +
> + return 0;
> +}
>
> /*************************
> * Writable Pagetables
> diff -r d18c6a3c676a xen/include/asm-x86/hvm/support.h
> --- a/xen/include/asm-x86/hvm/support.h Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/include/asm-x86/hvm/support.h Thu Jan 11 17:03:21 2007 +0800
> @@ -248,6 +248,8 @@ extern int arch_gethvm_ctxt(struct vcpu
>
> extern void shpage_init(struct domain *d, shared_iopage_t *sp);
>
> +extern void hvminfo_init(struct domain* d);
> +
> extern int hvm_enabled;
>
> int hvm_copy_to_guest_phys(paddr_t paddr, void *buf, int size);
> diff -r d18c6a3c676a xen/include/public/hvm/params.h
> --- a/xen/include/public/hvm/params.h Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/include/public/hvm/params.h Thu Jan 11 17:03:21 2007 +0800
> @@ -31,6 +31,7 @@
> #define HVM_PARAM_PAE_ENABLED 4
> #define HVM_PARAM_IOREQ_PFN 5
> #define HVM_PARAM_BUFIOREQ_PFN 6
> -#define HVM_NR_PARAMS 7
> +#define HVM_PARAM_SHINFO_PFN 7
> +#define HVM_NR_PARAMS 8
>
> #endif /* __XEN_PUBLIC_HVM_PARAMS_H__ */
> diff -r d18c6a3c676a xen/include/public/xen.h
> --- a/xen/include/public/xen.h Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/include/public/xen.h Thu Jan 11 17:03:37 2007 +0800
> @@ -465,6 +465,9 @@ struct shared_info {
> uint32_t wc_sec; /* Secs 00:00:00 UTC, Jan 1, 1970. */
> uint32_t wc_nsec; /* Nsecs 00:00:00 UTC, Jan 1, 1970. */
>
> + /* flag for resume PV driver in HVM guest */
> + uint32_t pvdrv_resume;
> +
> struct arch_shared_info arch;
>
> };
>
--
best rgds,
edwin
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
|
Home