[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [PATCH 3/3] [linux-2.6.18-xen.hg] delay xenbus init if xenbus master is a separate domain




This patch modifies xenbus to lazily initialize. This is necessary when
xenstored is running in a seperate minios domain, since it can't connect as a
slave just yet.

Ian Campbell has looked at an earlier version of this patch, and seems
to generally approve.

Here's an explanation of the states:

It starts out in XENBUS_XSD_UNCOMMITTED.

As the master xenbus (the one local to xenstored), it will receive an
mmap from xenstore, putting it in XENBUS_XSD_LOCAL_INIT. This enables
the wake_waiting IRQ, which will put it in XENBUS_XSD_LOCAL_READY.

Alternatively, as a slave xenbus, it will receive an ioctl from the
xenstore domain builder, putting it in XENBUS_XSD_FOREIGN_INIT. This
enables the wake_waiting IRQ, which will put it in
XENBUS_XSD_FOREIGN_READY.

DomU's are immediately initialized to XENBUS_XSD_FOREIGN_READY.

TODO: The types in the IOCTL might not be correct.

Signed-off-by: Diego Ongaro <diego.ongaro@xxxxxxxxxx>
Signed-off-by: Alex Zeffertt <alex.zeffertt@xxxxxxxxxxxxx>
---

diff -r e68a58170585 drivers/xen/core/xen_sysfs.c
--- a/drivers/xen/core/xen_sysfs.c      Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/core/xen_sysfs.c      Thu Mar 19 10:46:26 2009 +0000
@@ -16,6 +16,7 @@
 #include <xen/hypervisor_sysfs.h>
 #include <xen/xenbus.h>
 #include <xen/interface/kexec.h>
+#include "../xenbus/xenbus_comms.h"
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Mike D. Day <ncmike@xxxxxxxxxx>");
@@ -105,9 +106,8 @@
 {
        char *vm, *val;
        int ret;
-       extern int xenstored_ready;
 
-       if (!xenstored_ready)
+       if (!is_xenstored_ready())
                return -EBUSY;
 
        vm = xenbus_read(XBT_NIL, "vm", "", NULL);
diff -r e68a58170585 drivers/xen/xenbus/xenbus_comms.c
--- a/drivers/xen/xenbus/xenbus_comms.c Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/xenbus/xenbus_comms.c Thu Mar 19 10:46:26 2009 +0000
@@ -50,18 +50,39 @@
 static int xenbus_irq;
 
 extern void xenbus_probe(void *);
-extern int xenstored_ready;
 static DECLARE_WORK(probe_work, xenbus_probe, NULL);
 
 static DECLARE_WAIT_QUEUE_HEAD(xb_waitq);
 
 static irqreturn_t wake_waiting(int irq, void *unused, struct pt_regs *regs)
 {
-       if (unlikely(xenstored_ready == 0)) {
-               xenstored_ready = 1;
-               schedule_work(&probe_work);
+       int old, new;
+
+       old = atomic_read(&xenbus_xsd_state);
+       switch (old) {
+               case XENBUS_XSD_UNCOMMITTED:
+                       BUG();
+                       return IRQ_HANDLED;
+
+               case XENBUS_XSD_FOREIGN_INIT:
+                       new = XENBUS_XSD_FOREIGN_READY;
+                       break;
+               
+               case XENBUS_XSD_LOCAL_INIT:
+                       new = XENBUS_XSD_LOCAL_READY;
+                       break;
+
+               case XENBUS_XSD_FOREIGN_READY:
+               case XENBUS_XSD_LOCAL_READY:
+               default:
+                       goto wake;
        }
 
+       old = atomic_cmpxchg(&xenbus_xsd_state, old, new);
+       if (old != new)
+               schedule_work(&probe_work);
+
+wake:
        wake_up(&xb_waitq);
        return IRQ_HANDLED;
 }
diff -r e68a58170585 drivers/xen/xenbus/xenbus_comms.h
--- a/drivers/xen/xenbus/xenbus_comms.h Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/xenbus/xenbus_comms.h Thu Mar 19 10:46:26 2009 +0000
@@ -43,4 +43,20 @@
 extern struct xenstore_domain_interface *xen_store_interface;
 extern int xen_store_evtchn;
 
+/* For xenbus internal use. */
+enum {
+       XENBUS_XSD_UNCOMMITTED = 0,
+       XENBUS_XSD_FOREIGN_INIT,
+       XENBUS_XSD_FOREIGN_READY,
+       XENBUS_XSD_LOCAL_INIT,
+       XENBUS_XSD_LOCAL_READY,
+};
+extern atomic_t xenbus_xsd_state;
+
+static inline int is_xenstored_ready(void)
+{
+       int s = atomic_read(&xenbus_xsd_state);
+       return s == XENBUS_XSD_FOREIGN_READY || s == XENBUS_XSD_LOCAL_READY;
+}
+
 #endif /* _XENBUS_COMMS_H */
diff -r e68a58170585 drivers/xen/xenbus/xenbus_dev.c
--- a/drivers/xen/xenbus/xenbus_dev.c   Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/xenbus/xenbus_dev.c   Thu Mar 19 10:46:26 2009 +0000
@@ -53,6 +53,8 @@
 #include <xen/platform-compat.h>
 #endif
 
+#include <xen/public/xenbus.h>
+
 struct xenbus_dev_transaction {
        struct list_head list;
        struct xenbus_transaction handle;
@@ -95,6 +97,9 @@
        struct xenbus_dev_data *u = filp->private_data;
        struct read_buffer *rb;
        int i, ret;
+
+       if (!is_xenstored_ready())
+               return -ENODEV;
 
        mutex_lock(&u->reply_mutex);
        while (list_empty(&u->read_buffers)) {
@@ -205,6 +210,9 @@
        char *path, *token;
        struct watch_adapter *watch, *tmp_watch;
        int err, rc = len;
+
+       if (!is_xenstored_ready())
+               return -ENODEV;
 
        if ((len + u->len) > sizeof(u->u.buffer)) {
                rc = -EINVAL;
@@ -391,10 +399,63 @@
 {
        struct xenbus_dev_data *u = file->private_data;
 
+       if (!is_xenstored_ready())
+               return -ENODEV;
+
        poll_wait(file, &u->read_waitq, wait);
        if (!list_empty(&u->read_buffers))
                return POLLIN | POLLRDNORM;
        return 0;
+}
+
+static long xenbus_dev_ioctl(struct file *file,
+                             unsigned int cmd, unsigned long data)
+{
+       extern int xenbus_conn(domid_t remote_dom, int *grant_ref,
+                              evtchn_port_t *local_port);
+       void __user *udata = (void __user *) data;
+       int ret = -ENOTTY;
+       
+       if (!is_initial_xendomain())
+               return -ENODEV;
+
+
+       switch (cmd) {
+       case IOCTL_XENBUS_ALLOC: {
+               xenbus_alloc_t xa;
+               int old;
+
+               old = atomic_cmpxchg(&xenbus_xsd_state,
+                                    XENBUS_XSD_UNCOMMITTED,
+                                    XENBUS_XSD_FOREIGN_INIT);
+               if (old != XENBUS_XSD_UNCOMMITTED)
+                       return -EBUSY;
+
+               if (copy_from_user(&xa, udata, sizeof(xa))) {
+                       ret = -EFAULT;
+                       atomic_set(&xenbus_xsd_state, XENBUS_XSD_UNCOMMITTED);
+                       break;
+               }
+
+               ret = xenbus_conn(xa.dom, &xa.grant_ref, &xa.port);
+               if (ret != 0) {
+                       atomic_set(&xenbus_xsd_state, XENBUS_XSD_UNCOMMITTED);
+                       break;
+               }
+
+               if (copy_to_user(udata, &xa, sizeof(xa))) {
+                       ret = -EFAULT;
+                       atomic_set(&xenbus_xsd_state, XENBUS_XSD_UNCOMMITTED);
+                       break;
+               }
+       }
+       break;
+
+       default:
+               break;
+       }
+
+       return ret;
 }
 
 static const struct file_operations xenbus_dev_file_ops = {
@@ -403,6 +464,7 @@
        .open = xenbus_dev_open,
        .release = xenbus_dev_release,
        .poll = xenbus_dev_poll,
+       .unlocked_ioctl = xenbus_dev_ioctl
 };
 
 int xenbus_dev_init(void)
diff -r e68a58170585 drivers/xen/xenbus/xenbus_probe.c
--- a/drivers/xen/xenbus/xenbus_probe.c Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/xenbus/xenbus_probe.c Thu Mar 19 10:46:26 2009 +0000
@@ -44,6 +44,7 @@
 #include <linux/notifier.h>
 #include <linux/mutex.h>
 #include <linux/module.h>
+#include <xen/gnttab.h>
 
 #include <asm/io.h>
 #include <asm/page.h>
@@ -347,7 +348,12 @@
 
        DPRINTK("%s", dev->nodename);
 
+/* Commented out since xenstored stubdom is now minios based not linux based
+#define XENSTORE_DOMAIN_SHARES_THIS_KERNEL 
+*/
+#ifndef XENSTORE_DOMAIN_SHARES_THIS_KERNEL 
        if (is_initial_xendomain())
+#endif
                return;
 
        get_device(&dev->dev);
@@ -820,14 +826,13 @@
 EXPORT_SYMBOL_GPL(xenbus_suspend_cancel);
 
 /* A flag to determine if xenstored is 'ready' (i.e. has started) */
-int xenstored_ready = 0;
-
+atomic_t xenbus_xsd_state = ATOMIC_INIT(XENBUS_XSD_UNCOMMITTED);
 
 int register_xenstore_notifier(struct notifier_block *nb)
 {
        int ret = 0;
 
-       if (xenstored_ready > 0)
+       if (is_xenstored_ready())
                ret = nb->notifier_call(nb, 0, NULL);
        else
                blocking_notifier_chain_register(&xenstore_chain, nb);
@@ -845,7 +850,7 @@
 
 void xenbus_probe(void *unused)
 {
-       BUG_ON((xenstored_ready <= 0));
+       BUG_ON(!is_xenstored_ready());
 
        /* Enumerate devices in xenstore and watch for changes. */
        xenbus_probe_devices(&xenbus_frontend);
@@ -865,6 +870,28 @@
 static int xsd_kva_mmap(struct file *file, struct vm_area_struct *vma)
 {
        size_t size = vma->vm_end - vma->vm_start;
+       int old;
+       int rc;
+
+       old = atomic_cmpxchg(&xenbus_xsd_state,
+                          XENBUS_XSD_UNCOMMITTED,
+                          XENBUS_XSD_LOCAL_INIT);
+       switch (old) {
+               case XENBUS_XSD_UNCOMMITTED:
+                       rc = xb_init_comms();
+                       if (rc != 0)
+                               return rc;
+                       break;
+
+               case XENBUS_XSD_FOREIGN_INIT:
+               case XENBUS_XSD_FOREIGN_READY:
+                       return -EBUSY;
+               
+               case XENBUS_XSD_LOCAL_INIT:
+               case XENBUS_XSD_LOCAL_READY:
+               default:
+                       break;
+       }
 
        if ((size > PAGE_SIZE) || (vma->vm_pgoff != 0))
                return -EINVAL;
@@ -896,6 +923,62 @@
        return len;
 }
 #endif
+
+static int xb_free_port(evtchn_port_t port)
+{
+       struct evtchn_close close;
+       close.port = port;
+       return HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
+}
+
+int xenbus_conn(domid_t remote_dom, unsigned long *grant_ref, evtchn_port_t 
*local_port)
+{
+       struct evtchn_alloc_unbound alloc_unbound;
+       int rc, rc2;
+
+       BUG_ON(atomic_read(&xenbus_xsd_state) != XENBUS_XSD_FOREIGN_INIT);
+       BUG_ON(!is_initial_xendomain());
+
+#if defined(CONFIG_PROC_FS) && defined(CONFIG_XEN_PRIVILEGED_GUEST)
+       remove_xen_proc_entry("xsd_kva");
+       remove_xen_proc_entry("xsd_port");
+#endif
+
+       rc = xb_free_port(xen_store_evtchn);
+       if (rc != 0)
+               goto fail0;
+
+       alloc_unbound.dom = DOMID_SELF;
+       alloc_unbound.remote_dom = remote_dom;
+       rc = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
+                                        &alloc_unbound);
+       if (rc != 0)
+               goto fail0;
+       *local_port = xen_store_evtchn = alloc_unbound.port;
+
+       /* keep the old page (xen_store_mfn, xen_store_interface) */
+       rc = gnttab_grant_foreign_access(remote_dom, xen_store_mfn,
+                                        GTF_permit_access);
+       if (rc < 0)
+               goto fail1;
+       *grant_ref = rc;
+
+       rc = xb_init_comms();
+       if (rc != 0)
+               goto fail1;
+
+       return 0;
+
+fail1:
+       rc2 = xb_free_port(xen_store_evtchn);
+       if (rc2 != 0)
+               printk(KERN_WARNING
+                      "XENBUS: Error freeing xenstore event channel: %d\n",
+                      rc2);
+fail0:
+       xen_store_evtchn = -1;
+       return rc;
+}
 
 static int xenbus_probe_init(void)
 {
@@ -958,7 +1041,7 @@
 #endif
                xen_store_interface = mfn_to_virt(xen_store_mfn);
        } else {
-               xenstored_ready = 1;
+               atomic_set(&xenbus_xsd_state, XENBUS_XSD_FOREIGN_READY);
 #ifdef CONFIG_XEN
                xen_store_evtchn = xen_start_info->store_evtchn;
                xen_store_mfn = xen_start_info->store_mfn;
@@ -969,8 +1052,11 @@
                xen_store_interface = ioremap(xen_store_mfn << PAGE_SHIFT,
                                              PAGE_SIZE);
 #endif
+               /* Initialize the shared memory rings to talk to xenstored */
+               err = xb_init_comms();
+               if (err)
+                       goto err;
        }
-
 
        xenbus_dev_init();
 
diff -r e68a58170585 drivers/xen/xenbus/xenbus_xs.c
--- a/drivers/xen/xenbus/xenbus_xs.c    Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/xenbus/xenbus_xs.c    Thu Mar 19 10:46:26 2009 +0000
@@ -872,7 +872,6 @@
 
 int xs_init(void)
 {
-       int err;
        struct task_struct *task;
 
        INIT_LIST_HEAD(&xs_state.reply_list);
@@ -883,11 +882,6 @@
        mutex_init(&xs_state.response_mutex);
        init_rwsem(&xs_state.transaction_mutex);
        init_rwsem(&xs_state.watch_mutex);
-
-       /* Initialize the shared memory rings to talk to xenstored */
-       err = xb_init_comms();
-       if (err)
-               return err;
 
        task = kthread_run(xenwatch_thread, NULL, "xenwatch");
        if (IS_ERR(task))
diff -r e68a58170585 include/xen/public/xenbus.h
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/include/xen/public/xenbus.h       Thu Mar 19 10:46:26 2009 +0000
@@ -0,0 +1,56 @@
+/******************************************************************************
+ * xenbus.h
+ * 
+ * Interface to /proc/xen/xenbus.
+ * 
+ * Copyright (c) 2008, Diego Ongaro <diego.ongaro@xxxxxxxxxx>
+ * 
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation; or, when distributed
+ * separately from the Linux kernel or incorporated into other
+ * software packages, subject to the following license:
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this source file (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef __LINUX_PUBLIC_XENBUS_H__
+#define __LINUX_PUBLIC_XENBUS_H__
+
+#include <linux/types.h>
+
+#ifndef __user
+#define __user
+#endif
+
+typedef struct xenbus_alloc {
+       domid_t dom;
+       __u32 port;
+       __u32 grant_ref;
+} xenbus_alloc_t;
+
+/*
+ * @cmd: IOCTL_XENBUS_ALLOC
+ * @arg: &xenbus_alloc_t
+ * Return: 0, or -1 for error
+ */
+#define IOCTL_XENBUS_ALLOC                                     \
+       _IOC(_IOC_NONE, 'X', 0, sizeof(xenbus_alloc_t))
+
+#endif /* __LINUX_PUBLIC_XENBUS_H__ */

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel

 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.