WARNING - OLD ARCHIVES

This is an archived copy of the Xen.org mailing list, which we have preserved to ensure that existing links to archives are not broken. The live archive, which contains the latest emails, can be found at http://lists.xen.org/
   
 
 
Xen 
 
Home Products Support Community News
 
   
 

xen-devel

[Xen-devel] Re: [PATCH 8/8] HVM save restore: PV driver support

To: Tim Deegan <Tim.Deegan@xxxxxxxxxxxxx>
Subject: [Xen-devel] Re: [PATCH 8/8] HVM save restore: PV driver support
From: "Zhai, Edwin" <edwin.zhai@xxxxxxxxx>
Date: Fri, 19 Jan 2007 13:14:51 +0800
Cc: Ian Pratt <Ian.Pratt@xxxxxxxxxxxx>, xen-devel@xxxxxxxxxxxxxxxxxxx, "Zhai, Edwin" <edwin.zhai@xxxxxxxxx>
Delivery-date: Thu, 18 Jan 2007 21:16:53 -0800
Envelope-to: www-data@xxxxxxxxxxxxxxxxxx
In-reply-to: <20070111141333.GA2916@xxxxxxxxxxxxxxxxxxxxxx>
List-help: <mailto:xen-devel-request@lists.xensource.com?subject=help>
List-id: Xen developer discussion <xen-devel.lists.xensource.com>
List-post: <mailto:xen-devel@lists.xensource.com>
List-subscribe: <http://lists.xensource.com/cgi-bin/mailman/listinfo/xen-devel>, <mailto:xen-devel-request@lists.xensource.com?subject=subscribe>
List-unsubscribe: <http://lists.xensource.com/cgi-bin/mailman/listinfo/xen-devel>, <mailto:xen-devel-request@lists.xensource.com?subject=unsubscribe>
References: <20070111141333.GA2916@xxxxxxxxxxxxxxxxxxxxxx>
Sender: xen-devel-bounces@xxxxxxxxxxxxxxxxxxx
User-agent: Mutt/1.5.11
Tim, 

thanks for push in save/restore patches, but this one for save/restore PV 
driver 
on HVM seems to be missing.

if any concerns about this patch, pls. let me know.

thanks,


On Thu, Jan 11, 2007 at 10:13:33PM +0800, Zhai, Edwin wrote:
> [PATCH 8/8] HVM save restore: PV driver support
> 
> Signed-off-by: Zhai Edwin <edwin.zhai@xxxxxxxxx>
> 
> enable PV driver's save/restore in HVM domain by:
> HV:
> * send a pseudo PCI dev intr to guest in restore
> * rebuild the shared info on behalf of HVM guest
> * set a resume flag in shared info
> 
> Guest:
> * check the resmume flag in the pseudo PCI dev intr handler
> * if set do the PV driver resume work
> 
> 
> diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/blkfront/blkfront.c
> --- a/linux-2.6-xen-sparse/drivers/xen/blkfront/blkfront.c    Thu Jan 11 
> 17:03:17 2007 +0800
> +++ b/linux-2.6-xen-sparse/drivers/xen/blkfront/blkfront.c    Thu Jan 11 
> 17:03:21 2007 +0800
> @@ -138,6 +138,10 @@ static int blkfront_resume(struct xenbus
>  
>       DPRINTK("blkfront_resume: %s\n", dev->nodename);
>  
> +     /* resuming qemu disk would cause error when restore and skip next vbd 
> resume */
> +     if (info->connected == BLKIF_STATE_DISCONNECTED)
> +             return 0;
> + 
>       blkif_free(info, info->connected == BLKIF_STATE_CONNECTED);
>  
>       err = talk_to_backend(dev, info);
> diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/blkfront/vbd.c
> --- a/linux-2.6-xen-sparse/drivers/xen/blkfront/vbd.c Thu Jan 11 17:03:17 
> 2007 +0800
> +++ b/linux-2.6-xen-sparse/drivers/xen/blkfront/vbd.c Thu Jan 11 17:03:21 
> 2007 +0800
> @@ -183,7 +183,16 @@ xlbd_put_major_info(struct xlbd_major_in
>  xlbd_put_major_info(struct xlbd_major_info *mi)
>  {
>       mi->usage--;
> -     /* XXX: release major if 0 */
> +     /* release major if 0 */
> +     if (mi->usage)
> +             return;
> +
> +     printk("Unregistering block device major %i\n", mi->major);
> +     if (unregister_blkdev(mi->major, mi->type->devname)) {
> +             WPRINTK("can't put major %d with name %s\n",
> +                     mi->major, mi->type->devname);
> +             kfree(mi);
> +     }
>  }
>  
>  static int
> diff -r d18c6a3c676a linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c
> --- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c  Thu Jan 11 
> 17:03:17 2007 +0800
> +++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c  Thu Jan 11 
> 17:03:21 2007 +0800
> @@ -815,6 +815,7 @@ static int xsd_port_read(char *page, cha
>  }
>  #endif
>  
> +extern void set_restore_handler( void (*hdl1)(void), void (*hdl2)(void));
>  static int __init xenbus_probe_init(void)
>  {
>       int err = 0;
> @@ -882,6 +883,7 @@ static int __init xenbus_probe_init(void
>               xen_store_mfn = hvm_get_parameter(HVM_PARAM_STORE_PFN);
>               xen_store_interface = ioremap(xen_store_mfn << PAGE_SHIFT,
>                                             PAGE_SIZE);
> +             set_restore_handler(xenbus_suspend, xenbus_resume);
>  #endif
>       }
>  
> diff -r d18c6a3c676a unmodified_drivers/linux-2.6/platform-pci/evtchn.c
> --- a/unmodified_drivers/linux-2.6/platform-pci/evtchn.c      Thu Jan 11 
> 17:03:17 2007 +0800
> +++ b/unmodified_drivers/linux-2.6/platform-pci/evtchn.c      Thu Jan 11 
> 20:12:33 2007 +0800
> @@ -165,6 +165,7 @@ void notify_remote_via_irq(int irq)
>  }
>  EXPORT_SYMBOL(notify_remote_via_irq);
>  
> +extern void pvdrv_restore(void);
>  irqreturn_t evtchn_interrupt(int irq, void *dev_id, struct pt_regs *regs)
>  {
>       unsigned int l1i, port;
> @@ -173,6 +174,16 @@ irqreturn_t evtchn_interrupt(int irq, vo
>       shared_info_t *s = shared_info_area;
>       vcpu_info_t *v = &s->vcpu_info[cpu];
>       unsigned long l1, l2;
> +
> +     /* add a check to see if need resume after restore */ 
> +     if (s->pvdrv_resume == 0x58585858) {
> +             printk("evtchn_interrupt:resume PV driver.\n");
> +             s->pvdrv_resume= 0;
> +             v->evtchn_upcall_pending = 0;
> +
> +             pvdrv_restore();
> +             return IRQ_HANDLED;
> +     }
>  
>       v->evtchn_upcall_pending = 0;
>       /* NB. No need for a barrier here -- XCHG is a barrier on x86. */
> diff -r d18c6a3c676a unmodified_drivers/linux-2.6/platform-pci/platform-pci.c
> --- a/unmodified_drivers/linux-2.6/platform-pci/platform-pci.c        Thu Jan 
> 11 17:03:17 2007 +0800
> +++ b/unmodified_drivers/linux-2.6/platform-pci/platform-pci.c        Thu Jan 
> 11 17:03:21 2007 +0800
> @@ -28,6 +28,7 @@
>  #include <linux/interrupt.h>
>  #include <linux/vmalloc.h>
>  #include <linux/mm.h>
> +#include <linux/kthread.h>
>  #include <asm/system.h>
>  #include <asm/io.h>
>  #include <asm/irq.h>
> @@ -280,9 +281,85 @@ static struct pci_driver platform_driver
>  
>  static int pci_device_registered;
>  
> +extern int gnttab_suspend(void);
> +void platform_pci_suspend(void)
> +{
> +     gnttab_suspend();
> +}
> +
> +extern int gnttab_resume(void);
> +void platform_pci_resume(void)
> +{
> +     phys_to_machine_mapping = NULL;
> +
> +     gnttab_resume();
> +}
> +
> +/* support for PV driver save/restore in HVM domain*/
> +void (*suspend_handler)(void);
> +void (*resume_handler)(void);
> +void set_restore_handler( void (*hdl1)(void), void (*hdl2)(void))
> +{
> +     suspend_handler = hdl1;
> +     resume_handler  = hdl2;
> +}
> +EXPORT_SYMBOL(set_restore_handler);
> +
> +/* scheduled suspend&resume */
> +static void restore_handler(void *unused);
> +static int __do_pvdrv_restore(void *ignored);
> +
> +static DECLARE_WORK(restore_work, restore_handler, NULL);
> +
> +static int kthread_create_on_cpu(int (*f)(void *arg),
> +     void *arg,
> +     const char *name,
> +     int cpu)
> +{
> +     struct task_struct *p;
> +     p = kthread_create(f, arg, name);
> +     if (IS_ERR(p))
> +             return PTR_ERR(p);
> +     kthread_bind(p, cpu);
> +     wake_up_process(p);
> +     return 0;
> +}
> +
> +static int __do_pvdrv_restore(void *ignored)
> +{
> +     if (suspend_handler)
> +             suspend_handler();
> +     platform_pci_suspend();
> +
> +     platform_pci_resume();
> +     if (resume_handler)
> +             resume_handler();
> +
> +     return 0;
> +}
> +
> +static void restore_handler(void *unused)
> +{
> +     int err;
> +
> +     err = kthread_create_on_cpu(__do_pvdrv_restore, NULL, "pvdrv_suspend", 
> 0);
> +     if (err < 0) {
> +             printk(KERN_WARNING "error creating PV driver suspend process 
> (%d): retrying...\n",
> +                     err);
> +             schedule_delayed_work(&restore_work, 50/2);
> +     }
> +}
> +
> +void pvdrv_restore(void)
> +{
> +     schedule_work(&restore_work);
> +}
> +
>  static int __init platform_pci_module_init(void)
>  {
>       int rc;
> +
> +     set_restore_handler(NULL, NULL);
>  
>       rc = pci_module_init(&platform_driver);
>       if (rc)
> diff -r d18c6a3c676a xen/arch/x86/hvm/hvm.c
> --- a/xen/arch/x86/hvm/hvm.c  Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/arch/x86/hvm/hvm.c  Thu Jan 11 17:03:21 2007 +0800
> @@ -200,6 +200,9 @@ int hvm_vcpu_initialise(struct vcpu *v)
>   
>      /* init hvm sharepage */
>      shpage_init(v->domain, get_sp(v->domain));
> +
> +    /* other hvm info need for save/restore */
> +    hvminfo_init(v->domain);
>  
>      /* Init guest TSC to start from zero. */
>      hvm_set_guest_time(v, 0);
> diff -r d18c6a3c676a xen/arch/x86/hvm/intercept.c
> --- a/xen/arch/x86/hvm/intercept.c    Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/arch/x86/hvm/intercept.c    Thu Jan 11 17:03:37 2007 +0800
> @@ -445,6 +445,71 @@ void shpage_init(struct domain *d, share
>      hvm_register_savevm(d, "xen_hvm_shpage", 0x10, 1, shpage_save, 
> shpage_load, sp);
>  }
>  
> +void hvminfo_print(struct domain* d)
> +{
> +    printk("********hvm other info***********\n");
> +    printk("callback irq=%"PRId64".\n", 
> d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ]);
> +    printk("shared info pfn=0x%"PRIx64".\n", 
> d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN]);
> +}
> +
> +static void hvminfo_save(hvm_domain_context_t *h, void *opaque)
> +{
> +    struct domain *d = opaque;
> +
> +#ifdef HVM_DEBUG_SUSPEND
> +    hvminfo_print(d);
> +#endif
> +
> +    hvm_put_64u(h, d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ]);
> +    hvm_put_64u(h, d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN]);
> +}
> +
> +extern int rebuild_shared_info(struct domain *d, unsigned long 
> share_info_pfn);
> +static int hvminfo_load(hvm_domain_context_t *h, void *opaque, int 
> version_id)
> +{
> +    struct domain *d = opaque;
> +    int callback_irq = hvm_get_64u(h);
> +    unsigned long share_info_pfn = hvm_get_64u(h);
> +
> +    if (callback_irq == 0) {
> +        return 0;
> +    }
> +
> +    d->arch.hvm_domain.params[HVM_PARAM_CALLBACK_IRQ] = callback_irq;
> +
> +    /* reconstruct the share info for HVM guest */
> +    rebuild_shared_info(d, share_info_pfn);
> +    d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN] = share_info_pfn;
> +
> +    /* set resume state to notify HVM PV driver */
> +    shared_info(d, pvdrv_resume) = 0x58585858;
> +
> +    /* set all the evtchn_upcall_pending on all vcpus */
> +    if (callback_irq) {
> +        struct vcpu *v;
> +
> +        for_each_vcpu(d, v) {
> +            if ( !test_and_set_bit(0, &vcpu_info(v, evtchn_upcall_pending)) )
> +                printk("on vcpu %d raise a vir level intr %d to restore PV 
> driver in HVM guest!\n",
> +                        v->vcpu_id,
> +                        callback_irq);
> +        }
> +    }
> +
> +
> +#ifdef HVM_DEBUG_SUSPEND
> +    hvminfo_print(d);
> +#endif
> +
> +    return 0;
> +
> +}
> +
> +void hvminfo_init(struct domain* d)
> +{
> +    hvm_register_savevm(d, "xen_hvm_other_info", 0x11, 1, hvminfo_save, 
> hvminfo_load, d);
> +}
> +
>  int hvm_buffered_io_intercept(ioreq_t *p)
>  {
>      struct vcpu *v = current;
> diff -r d18c6a3c676a xen/arch/x86/mm.c
> --- a/xen/arch/x86/mm.c       Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/arch/x86/mm.c       Thu Jan 11 17:03:21 2007 +0800
> @@ -3000,6 +3000,7 @@ long arch_memory_op(int op, XEN_GUEST_HA
>          case XENMAPSPACE_shared_info:
>              if ( xatp.idx == 0 )
>                  mfn = virt_to_mfn(d->shared_info);
> +            d->arch.hvm_domain.params[HVM_PARAM_SHINFO_PFN] = xatp.gpfn;
>              break;
>          case XENMAPSPACE_grant_table:
>              if ( xatp.idx < NR_GRANT_FRAMES )
> @@ -3143,6 +3144,43 @@ long arch_memory_op(int op, XEN_GUEST_HA
>      return 0;
>  }
>  
> +/* reconstruct the share info for HVM guest */
> +int rebuild_shared_info(struct domain *d, unsigned long share_info_pfn)
> +{
> +    unsigned long mfn, prev_mfn, gpfn = 0;
> +
> +    mfn = virt_to_mfn(d->shared_info);
> +    if ( !shadow_mode_translate(d) || (mfn == 0)) {
> +        printk("reconstruct share info for HVM guest failed!\n");
> +        return -1;
> +    }
> +
> +    LOCK_BIGLOCK(d);
> +
> +    /* Remove previously mapped page if it was present. */
> +    prev_mfn = gmfn_to_mfn(d, share_info_pfn);
> +    if ( mfn_valid(prev_mfn) )
> +    {
> +        if ( IS_XEN_HEAP_FRAME(mfn_to_page(prev_mfn)) )
> +            /* Xen heap frames are simply unhooked from this phys slot. */
> +            guest_physmap_remove_page(d, share_info_pfn, prev_mfn);
> +        else
> +            /* Normal domain memory is freed, to avoid leaking memory. */
> +            guest_remove_page(d, share_info_pfn);
> +    }
> +
> +    /* Unmap from old location, if any. */
> +    gpfn = get_gpfn_from_mfn(mfn);
> +    if ( gpfn != INVALID_M2P_ENTRY )
> +        guest_physmap_remove_page(d, gpfn, mfn);
> +
> +    /* Map at new location. */
> +    guest_physmap_add_page(d, share_info_pfn, mfn);
> +
> +    UNLOCK_BIGLOCK(d);
> +
> +    return 0;
> +}
>  
>  /*************************
>   * Writable Pagetables
> diff -r d18c6a3c676a xen/include/asm-x86/hvm/support.h
> --- a/xen/include/asm-x86/hvm/support.h       Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/include/asm-x86/hvm/support.h       Thu Jan 11 17:03:21 2007 +0800
> @@ -248,6 +248,8 @@ extern int arch_gethvm_ctxt(struct vcpu 
>  
>  extern void shpage_init(struct domain *d, shared_iopage_t *sp);
>  
> +extern void hvminfo_init(struct domain* d);
> +
>  extern int hvm_enabled;
>  
>  int hvm_copy_to_guest_phys(paddr_t paddr, void *buf, int size);
> diff -r d18c6a3c676a xen/include/public/hvm/params.h
> --- a/xen/include/public/hvm/params.h Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/include/public/hvm/params.h Thu Jan 11 17:03:21 2007 +0800
> @@ -31,6 +31,7 @@
>  #define HVM_PARAM_PAE_ENABLED  4
>  #define HVM_PARAM_IOREQ_PFN    5
>  #define HVM_PARAM_BUFIOREQ_PFN 6
> -#define HVM_NR_PARAMS          7
> +#define HVM_PARAM_SHINFO_PFN   7
> +#define HVM_NR_PARAMS          8
>  
>  #endif /* __XEN_PUBLIC_HVM_PARAMS_H__ */
> diff -r d18c6a3c676a xen/include/public/xen.h
> --- a/xen/include/public/xen.h        Thu Jan 11 17:03:17 2007 +0800
> +++ b/xen/include/public/xen.h        Thu Jan 11 17:03:37 2007 +0800
> @@ -465,6 +465,9 @@ struct shared_info {
>      uint32_t wc_sec;          /* Secs  00:00:00 UTC, Jan 1, 1970.  */
>      uint32_t wc_nsec;         /* Nsecs 00:00:00 UTC, Jan 1, 1970.  */
>  
> +    /* flag for resume PV driver in HVM guest */
> +    uint32_t pvdrv_resume;
> +
>      struct arch_shared_info arch;
>  
>  };
> 

-- 
best rgds,
edwin

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel