Hi all,
this patch implements pci passthrough (hotplug and coldplug) in
libxenlight, it also adds three new commands to xl: pci-attach,
pci-detach and pci-list.
Currently flr on a device is done writing to
/sys/bus/pci/drivers/pciback/do_flr
pciback do_flr is present in both XCI and XCP 2.6.27 kernels and it is
implemented in this patch:
http://xenbits.xen.org/gitweb?p=xenclient/linux-2.6.27-pq.git;a=blob;f=master/pciback-flr;h=e993f18dbda740cef63176b75e114bbbf9a3801d;hb=HEAD
Signed-off-by: Stefano Stabellini <stefano.stabellini@xxxxxxxxxxxxx>
---
diff -r 32a27fe01e07 tools/libxl/libxl.c
--- a/tools/libxl/libxl.c Thu Nov 12 15:34:37 2009 +0000
+++ b/tools/libxl/libxl.c Thu Nov 12 18:04:19 2009 +0000
@@ -156,6 +156,7 @@
xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/uuid", vm_path), uuid_string,
strlen(uuid_string));
xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/name", vm_path), info->name,
strlen(info->name));
+ xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/image/ostype", vm_path),
"hvm", strlen("hvm"));
libxl_xs_writev(ctx, t, dom_path, info->xsdata);
libxl_xs_writev(ctx, t, libxl_sprintf(ctx, "%s/platform", dom_path),
info->platformdata);
@@ -368,6 +369,8 @@
XL_LOG(ctx, XL_LOG_ERROR, "failed ot get uuid for %d\n", domid);
return -1;
}
+ if (libxl_device_pci_shutdown(ctx, domid) < 0)
+ XL_LOG(ctx, XL_LOG_ERROR, "pci shutdown failed for domid %d\n", domid);
xs_write(ctx->xsh, XBT_NULL,
libxl_sprintf(ctx, "/local/domain/0/device-model/%d/command",
domid),
"shutdown", strlen("shutdown"));
@@ -375,7 +378,6 @@
XL_LOG(ctx, XL_LOG_ERROR, "xc_domain_pause failed for %d\n", domid);
return -1;
}
- /* do_FLR */
if (xc_domain_destroy(ctx->xch, domid) < 0) {
XL_LOG(ctx, XL_LOG_ERROR, "xc_domain_destroy failed for %d\n", domid);
return -1;
@@ -746,17 +748,417 @@
}
/******************************************************************************/
-int libxl_device_pci_add(struct libxl_ctx *ctx, uint32_t domid)
+
+int libxl_device_pci_init(libxl_device_pci *pcidev, unsigned int domain,
+ unsigned int bus, unsigned int dev,
+ unsigned int func, unsigned int vdevfn)
{
- return ERROR_NI;
+ pcidev->domain = domain;
+ pcidev->bus = bus;
+ pcidev->dev = dev;
+ pcidev->func = func;
+ pcidev->vdevfn = vdevfn;
+ return 0;
}
-int libxl_device_pci_clean_shutdown(struct libxl_ctx *ctx, uint32_t domid)
+static int libxl_create_pci_backend(struct libxl_ctx *ctx, uint32_t domid,
libxl_device_pci *pcidev, int num)
{
- return ERROR_NI;
+ flexarray_t *front;
+ flexarray_t *back;
+ unsigned int boffset = 0;
+ unsigned int foffset = 0;
+ libxl_device device;
+ int i;
+
+ front = flexarray_make(16, 1);
+ if (!front)
+ return ERROR_NOMEM;
+ back = flexarray_make(16, 1);
+ if (!back)
+ return ERROR_NOMEM;
+
+ XL_LOG(ctx, XL_LOG_DEBUG, "Creating pci backend\n");
+
+ /* add pci device */
+ device.backend_devid = 0;
+ device.backend_domid = 0;
+ device.backend_kind = DEVICE_PCI;
+ device.devid = 0;
+ device.domid = domid;
+ device.kind = DEVICE_PCI;
+
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "frontend-id"));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", domid));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "online"));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "1"));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "state"));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 1));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "domain"));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "%s",
libxl_domid_to_name(ctx, domid)));
+ for (i = 0; i < num; i++) {
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "key-%d", i));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF,
pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "dev-%d", i));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF,
pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func));
+ if (pcidev->vdevfn) {
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "vdevfn-%d", i));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "%x",
pcidev->vdevfn));
+ }
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "opts-%d", i));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx,
"msitranslate=%d,power_mgmt=%d", pcidev->msitranslate, pcidev->power_mgmt));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "state-%d", i));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 1));
+ }
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "num_devs"));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", num));
+
+ flexarray_set(front, foffset++, libxl_sprintf(ctx, "backend-id"));
+ flexarray_set(front, foffset++, libxl_sprintf(ctx, "%d", 0));
+ flexarray_set(front, foffset++, libxl_sprintf(ctx, "state"));
+ flexarray_set(front, foffset++, libxl_sprintf(ctx, "%d", 1));
+
+ libxl_device_generic_add(ctx, &device,
+ libxl_xs_kvs_of_flexarray(ctx, back, boffset),
+ libxl_xs_kvs_of_flexarray(ctx, front, foffset));
+
+ flexarray_free(back);
+ flexarray_free(front);
+ return 0;
}
-int libxl_device_pci_hard_shutdown(struct libxl_ctx *ctx, uint32_t domid)
+static int libxl_device_pci_add_xenstore(struct libxl_ctx *ctx, uint32_t
domid, libxl_device_pci *pcidev)
{
- return ERROR_NI;
+ flexarray_t *back;
+ char *num_devs, *be_path;
+ int num = 0;
+ unsigned int boffset = 0;
+ xs_transaction_t t;
+
+ be_path = libxl_sprintf(ctx, "%s/backend/pci/%d/0",
xs_get_domain_path(ctx->xsh, 0), domid);
+ num_devs = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/num_devs",
be_path));
+ if (!num_devs)
+ return libxl_create_pci_backend(ctx, domid, pcidev, 1);
+
+ if (!is_hvm(ctx, domid)) {
+ if (libxl_wait_for_backend(ctx, be_path, "4") < 0)
+ return -1;
+ }
+
+ back = flexarray_make(16, 1);
+ if (!back)
+ return ERROR_NOMEM;
+
+ XL_LOG(ctx, XL_LOG_DEBUG, "Adding new pci device to xenstore\n");
+ num = atoi(num_devs);
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "key-%d", num));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF, pcidev->domain,
pcidev->bus, pcidev->dev, pcidev->func));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "dev-%d", num));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF, pcidev->domain,
pcidev->bus, pcidev->dev, pcidev->func));
+ if (pcidev->vdevfn) {
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "vdevfn-%d", num));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "%x",
pcidev->vdevfn));
+ }
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "opts-%d", num));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx,
"msitranslate=%d,power_mgmt=%d", pcidev->msitranslate, pcidev->power_mgmt));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "state-%d", num));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 1));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "num_devs"));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", num + 1));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "state"));
+ flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 7));
+
+retry_transaction:
+ t = xs_transaction_start(ctx->xsh);
+ libxl_xs_writev(ctx, t, be_path,
+ libxl_xs_kvs_of_flexarray(ctx, back, boffset));
+ if (!xs_transaction_end(ctx->xsh, t, 0))
+ if (errno == EAGAIN)
+ goto retry_transaction;
+
+ flexarray_free(back);
+ return 0;
}
+
+static int libxl_device_pci_remove_xenstore(struct libxl_ctx *ctx, uint32_t
domid, libxl_device_pci *pcidev)
+{
+ char *be_path, *num_devs_path, *num_devs, *xsdev;
+ int num, i;
+ xs_transaction_t t;
+ unsigned int domain = 0, bus = 0, dev = 0, func = 0;
+
+ be_path = libxl_sprintf(ctx, "%s/backend/pci/%d/0",
xs_get_domain_path(ctx->xsh, 0), domid);
+ num_devs_path = libxl_sprintf(ctx, "%s/num_devs", be_path);
+ num_devs = libxl_xs_read(ctx, XBT_NULL, num_devs_path);
+ if (!num_devs)
+ return -1;
+ num = atoi(num_devs);
+ if (num == 1) {
+ libxl_device_destroy(ctx, be_path, 1);
+ xs_rm(ctx->xsh, XBT_NULL, be_path);
+ return 0;
+ }
+
+ if (!is_hvm(ctx, domid)) {
+ if (libxl_wait_for_backend(ctx, be_path, "4") < 0) {
+ XL_LOG(ctx, XL_LOG_DEBUG, "pci backend at %s is not ready\n");
+ return -1;
+ }
+ }
+
+ for (i = 0; i < num; i++) {
+ xsdev = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/dev-%d",
be_path, i));
+ sscanf(xsdev, PCI_BDF, &domain, &bus, &dev, &func);
+ if (domain == pcidev->domain && bus == pcidev->bus &&
+ pcidev->dev == dev && pcidev->func == func) {
+ break;
+ }
+ }
+ if (i == num) {
+ XL_LOG(ctx, XL_LOG_ERROR, "Couldn't find the device on xenstore\n");
+ return -1;
+ }
+
+retry_transaction:
+ t = xs_transaction_start(ctx->xsh);
+ libxl_xs_write(ctx, t, num_devs_path, "%d", num - 1);
+ xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/state-%d", be_path, i), "6",
strlen("6"));
+ xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/state", be_path), "7",
strlen("7"));
+ if (!xs_transaction_end(ctx->xsh, t, 0))
+ if (errno == EAGAIN)
+ goto retry_transaction;
+ return 0;
+}
+
+int libxl_device_pci_add(struct libxl_ctx *ctx, uint32_t domid,
libxl_device_pci *pcidev)
+{
+ char path[50];
+ char *state, *vdevfn;
+ int rc, hvm;
+
+ /* TODO: check if the device can be assigned */
+
+ libxl_device_pci_flr(ctx, pcidev->domain, pcidev->bus, pcidev->dev,
pcidev->func);
+
+ hvm = is_hvm(ctx, domid);
+ if (hvm) {
+ if (libxl_wait_for_device_model(ctx, domid, "running") < 0) {
+ return -1;
+ }
+ snprintf(path, sizeof(path), "/local/domain/0/device-model/%d/state",
domid);
+ state = libxl_xs_read(ctx, XBT_NULL, path);
+ snprintf(path, sizeof(path),
"/local/domain/0/device-model/%d/parameter", domid);
+ if (pcidev->vdevfn)
+ libxl_xs_write(ctx, XBT_NULL, path, PCI_BDF_VDEVFN, pcidev->domain,
+ pcidev->bus, pcidev->dev, pcidev->func,
pcidev->vdevfn);
+ else
+ libxl_xs_write(ctx, XBT_NULL, path, PCI_BDF, pcidev->domain,
+ pcidev->bus, pcidev->dev, pcidev->func);
+ snprintf(path, sizeof(path),
"/local/domain/0/device-model/%d/command", domid);
+ xs_write(ctx->xsh, XBT_NULL, path, "pci-ins", strlen("pci-ins"));
+ if (libxl_wait_for_device_model(ctx, domid, "pci-inserted") < 0)
+ XL_LOG(ctx, XL_LOG_ERROR, "Device Model didn't respond in time\n");
+ snprintf(path, sizeof(path),
"/local/domain/0/device-model/%d/parameter", domid);
+ vdevfn = libxl_xs_read(ctx, XBT_NULL, path);
+ sscanf(vdevfn + 2, "%x", &pcidev->vdevfn);
+ snprintf(path, sizeof(path), "/local/domain/0/device-model/%d/state",
domid);
+ xs_write(ctx->xsh, XBT_NULL, path, state, strlen(state));
+ } else {
+ char *sysfs_path = libxl_sprintf(ctx,
"SYSFS_PCI_DEV/"PCI_BDF"/resource", pcidev->domain,
+ pcidev->bus, pcidev->dev,
pcidev->func);
+ FILE *f = fopen(sysfs_path, "r");
+ unsigned int start = 0, end = 0, flags = 0, size = 0;
+ int irq = 0;
+ int i;
+
+ if (f == NULL) {
+ XL_LOG(ctx, XL_LOG_ERROR, "Couldn't open %s\n", sysfs_path);
+ return -1;
+ }
+ for (i = 0; i < PROC_PCI_NUM_RESOURCES; i++) {
+ fscanf(f, "0x%x 0x%x 0x%x\n", &start, &end, &flags);
+ size = end - start + 1;
+ if (start) {
+ if (flags & PCI_BAR_IO) {
+ rc = xc_domain_ioport_permission(ctx->xch, domid, start,
size, 1);
+ if (rc < 0)
+ XL_LOG(ctx, XL_LOG_ERROR, "Error:
xc_domain_ioport_permission error 0x%x/0x%x: %d\n", start, size, rc);
+ } else {
+ rc = xc_domain_iomem_permission(ctx->xch, domid,
start>>XC_PAGE_SHIFT,
+
(size+(XC_PAGE_SIZE-1))>>XC_PAGE_SHIFT, 1);
+ if (rc < 0)
+ XL_LOG(ctx, XL_LOG_ERROR, "Error:
xc_domain_iomem_permission error 0x%x/0x%x: %d\n", start, size, rc);
+ }
+ }
+ }
+ fclose(f);
+ sysfs_path = libxl_sprintf(ctx, "SYSFS_PCI_DEV/"PCI_BDF"/irq",
pcidev->domain,
+ pcidev->bus, pcidev->dev, pcidev->func);
+ f = fopen(sysfs_path, "r");
+ if (f == NULL) {
+ XL_LOG(ctx, XL_LOG_ERROR, "Couldn't open %s\n", sysfs_path);
+ goto out;
+ }
+ fscanf(f, "%u", &irq);
+ if (irq) {
+ rc = xc_physdev_map_pirq(ctx->xch, domid, irq, &irq);
+ if (rc < 0) {
+ XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_physdev_map_pirq irq=%d:
%d\n", irq, rc);
+ }
+ rc = xc_domain_irq_permission(ctx->xch, domid, irq, 1);
+ if (rc < 0) {
+ XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_domain_irq_permission
irq=%d: %d\n", irq, rc);
+ }
+ }
+ fclose(f);
+ }
+out:
+ if ((rc = xc_assign_device(ctx->xch, domid, pcidev->value)) < 0)
+ XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_assign_device error %d\n", rc);
+
+ libxl_device_pci_add_xenstore(ctx, domid, pcidev);
+ return 0;
+}
+
+int libxl_device_pci_remove(struct libxl_ctx *ctx, uint32_t domid,
libxl_device_pci *pcidev)
+{
+ char path[50];
+ char *state;
+ int hvm, rc;
+
+ /* TODO: check if the device can be detached */
+
+ hvm = is_hvm(ctx, domid);
+ if (hvm) {
+ if (libxl_wait_for_device_model(ctx, domid, "running") < 0) {
+ return -1;
+ }
+ snprintf(path, sizeof(path), "/local/domain/0/device-model/%d/state",
domid);
+ state = libxl_xs_read(ctx, XBT_NULL, path);
+ snprintf(path, sizeof(path),
"/local/domain/0/device-model/%d/parameter", domid);
+ libxl_xs_write(ctx, XBT_NULL, path, PCI_BDF, pcidev->domain,
+ pcidev->bus, pcidev->dev, pcidev->func);
+ snprintf(path, sizeof(path),
"/local/domain/0/device-model/%d/command", domid);
+ xs_write(ctx->xsh, XBT_NULL, path, "pci-rem", strlen("pci-rem"));
+ if (libxl_wait_for_device_model(ctx, domid, "pci-removed") < 0) {
+ XL_LOG(ctx, XL_LOG_ERROR, "Device Model didn't respond in time\n");
+ return -1;
+ }
+ snprintf(path, sizeof(path), "/local/domain/0/device-model/%d/state",
domid);
+ xs_write(ctx->xsh, XBT_NULL, path, state, strlen(state));
+ } else {
+ char *sysfs_path = libxl_sprintf(ctx,
"SYSFS_PCI_DEV/"PCI_BDF"/resource", pcidev->domain,
+ pcidev->bus, pcidev->dev,
pcidev->func);
+ FILE *f = fopen(sysfs_path, "r");
+ unsigned int start = 0, end = 0, flags = 0, size = 0;
+ int irq = 0;
+ int i;
+
+ if (f == NULL) {
+ XL_LOG(ctx, XL_LOG_ERROR, "Couldn't open %s\n", sysfs_path);
+ goto skip1;
+ }
+ for (i = 0; i < PROC_PCI_NUM_RESOURCES; i++) {
+ fscanf(f, "0x%x 0x%x 0x%x\n", &start, &end, &flags);
+ size = end - start + 1;
+ if (start) {
+ if (flags & PCI_BAR_IO) {
+ rc = xc_domain_ioport_permission(ctx->xch, domid, start,
size, 0);
+ if (rc < 0)
+ XL_LOG(ctx, XL_LOG_ERROR, "Error:
xc_domain_ioport_permission error 0x%x/0x%x: %d\n", start, size, rc);
+ } else {
+ rc = xc_domain_iomem_permission(ctx->xch, domid,
start>>XC_PAGE_SHIFT,
+
(size+(XC_PAGE_SIZE-1))>>XC_PAGE_SHIFT, 0);
+ if (rc < 0)
+ XL_LOG(ctx, XL_LOG_ERROR, "Error:
xc_domain_iomem_permission error 0x%x/0x%x: %d\n", start, size, rc);
+ }
+ }
+ }
+ fclose(f);
+skip1:
+ sysfs_path = libxl_sprintf(ctx, "SYSFS_PCI_DEV/"PCI_BDF"/irq",
pcidev->domain,
+ pcidev->bus, pcidev->dev, pcidev->func);
+ f = fopen(sysfs_path, "r");
+ if (f == NULL) {
+ XL_LOG(ctx, XL_LOG_ERROR, "Couldn't open %s\n", sysfs_path);
+ goto out;
+ }
+ fscanf(f, "%u", &irq);
+ if (irq) {
+ rc = xc_physdev_unmap_pirq(ctx->xch, domid, irq);
+ if (rc < 0) {
+ XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_physdev_map_pirq irq=%d:
%d\n", irq, rc);
+ }
+ rc = xc_domain_irq_permission(ctx->xch, domid, irq, 0);
+ if (rc < 0) {
+ XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_domain_irq_permission
irq=%d: %d\n", irq, rc);
+ }
+ }
+ fclose(f);
+ }
+out:
+ libxl_device_pci_remove_xenstore(ctx, domid, pcidev);
+
+ libxl_device_pci_flr(ctx, pcidev->domain, pcidev->bus, pcidev->dev,
pcidev->func);
+
+ if ((rc = xc_deassign_device(ctx->xch, domid, pcidev->value)) < 0)
+ XL_LOG(ctx, XL_LOG_ERROR, "Error: xc_deassign_device error %d\n", rc);
+ return 0;
+}
+
+libxl_device_pci *libxl_device_pci_list(struct libxl_ctx *ctx, uint32_t domid,
int *num)
+{
+ char *be_path, *num_devs, *xsdev, *xsvdevfn, *xsopts;
+ int n, i;
+ unsigned int domain = 0, bus = 0, dev = 0, func = 0, vdevfn = 0;
+ libxl_device_pci *pcidevs;
+
+ be_path = libxl_sprintf(ctx, "%s/backend/pci/%d/0",
xs_get_domain_path(ctx->xsh, 0), domid);
+ num_devs = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/num_devs",
be_path));
+ if (!num_devs) {
+ *num = 0;
+ return NULL;
+ }
+ n = atoi(num_devs);
+ pcidevs = (libxl_device_pci *) libxl_calloc(ctx, n,
sizeof(libxl_device_pci));
+ *num = n;
+
+ for (i = 0; i < n; i++) {
+ xsdev = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/dev-%d",
be_path, i));
+ sscanf(xsdev, PCI_BDF, &domain, &bus, &dev, &func);
+ xsvdevfn = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx,
"%s/vdevfn-%d", be_path, i));
+ if (xsvdevfn)
+ vdevfn = strtol(xsvdevfn, (char **) NULL, 16);
+ libxl_device_pci_init(pcidevs + i, domain, bus, dev, func, vdevfn);
+ xsopts = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/opts-%d",
be_path, i));
+ if (xsopts) {
+ char *saveptr;
+ char *p = strtok_r(xsopts, ",=", &saveptr);
+ do {
+ while (*p == ' ')
+ p++;
+ if (!strcmp(p, "msitranslate")) {
+ p = strtok_r(NULL, ",=", &saveptr);
+ pcidevs[i].msitranslate = atoi(p);
+ } else if (!strcmp(p, "power_mgmt")) {
+ p = strtok_r(NULL, ",=", &saveptr);
+ pcidevs[i].power_mgmt = atoi(p);
+ }
+ } while ((p = strtok_r(NULL, ",=", &saveptr)) != NULL);
+ }
+ }
+ return pcidevs;
+}
+
+int libxl_device_pci_shutdown(struct libxl_ctx *ctx, uint32_t domid)
+{
+ libxl_device_pci *pcidevs;
+ int num, i;
+
+ pcidevs = libxl_device_pci_list(ctx, domid, &num);
+ for (i = 0; i < num; i++) {
+ if (libxl_device_pci_remove(ctx, domid, pcidevs + i) < 0)
+ return -1;
+ }
+ return 0;
+}
+
diff -r 32a27fe01e07 tools/libxl/libxl.h
--- a/tools/libxl/libxl.h Thu Nov 12 15:34:37 2009 +0000
+++ b/tools/libxl/libxl.h Thu Nov 12 18:04:19 2009 +0000
@@ -148,6 +148,25 @@
libxl_nic_type nictype;
} libxl_device_nic;
+typedef struct {
+ union {
+ unsigned int value;
+ struct {
+ unsigned int reserved1:2;
+ unsigned int reg:6;
+ unsigned int func:3;
+ unsigned int dev:5;
+ unsigned int bus:8;
+ unsigned int reserved2:7;
+ unsigned int enable:1;
+ };
+ };
+ unsigned int domain;
+ unsigned int vdevfn;
+ bool msitranslate;
+ bool power_mgmt;
+} libxl_device_pci;
+
#define ERROR_FAIL (-2)
#define ERROR_NI (-101)
#define ERROR_NOMEM (-1032)
@@ -194,8 +213,14 @@
int libxl_device_vfb_clean_shutdown(struct libxl_ctx *ctx, uint32_t domid);
int libxl_device_vfb_hard_shutdown(struct libxl_ctx *ctx, uint32_t domid);
-int libxl_device_pci_add(struct libxl_ctx *ctx, uint32_t domid);
-int libxl_device_pci_clean_shutdown(struct libxl_ctx *ctx, uint32_t domid);
-int libxl_device_pci_hard_shutdown(struct libxl_ctx *ctx, uint32_t domid);
+#define PCI_BDF "%04x:%02x:%02x.%01x"
+#define PCI_BDF_VDEVFN "%04x:%02x:%02x.%01x@%02x"
+int libxl_device_pci_add(struct libxl_ctx *ctx, uint32_t domid,
libxl_device_pci *pcidev);
+int libxl_device_pci_remove(struct libxl_ctx *ctx, uint32_t domid,
libxl_device_pci *pcidev);
+int libxl_device_pci_shutdown(struct libxl_ctx *ctx, uint32_t domid);
+libxl_device_pci *libxl_device_pci_list(struct libxl_ctx *ctx, uint32_t domid,
int *num);
+int libxl_device_pci_init(libxl_device_pci *pcidev, unsigned int domain,
+ unsigned int bus, unsigned int dev,
+ unsigned int func, unsigned int vdevfn);
#endif
diff -r 32a27fe01e07 tools/libxl/libxl_device.c
--- a/tools/libxl/libxl_device.c Thu Nov 12 15:34:37 2009 +0000
+++ b/tools/libxl/libxl_device.c Thu Nov 12 18:04:19 2009 +0000
@@ -15,6 +15,7 @@
*/
#include <string.h>
+#include <stdio.h>
#include "libxl.h"
#include "libxl_internal.h"
#include <sys/time.h> /* for struct timeval */
@@ -83,9 +84,12 @@
libxl_xs_writev(ctx, t, backend_path, bents);
libxl_xs_writev(ctx, t, frontend_path, fents);
- if (!xs_transaction_end(ctx->xsh, t, 0))
+ if (!xs_transaction_end(ctx->xsh, t, 0)) {
if (errno == EAGAIN)
goto retry_transaction;
+ else
+ XL_LOG(ctx, XL_LOG_ERROR, "xs transaction failed errno=%d\n",
errno);
+ }
return 0;
}
@@ -154,7 +158,7 @@
char *state = libxl_xs_read(ctx, XBT_NULL, state_path);
if (!state)
return 0;
- if (atoi(state) <= 3) {
+ if (atoi(state) != 4) {
xs_rm(ctx->xsh, XBT_NULL, be_path);
return 0;
}
@@ -240,3 +244,72 @@
flexarray_free(toremove);
return 0;
}
+
+int libxl_device_pci_flr(struct libxl_ctx *ctx, unsigned int domain, unsigned
int bus,
+ unsigned int dev, unsigned int func)
+{
+ FILE *fd;
+
+ fd = fopen("/sys/bus/pci/drivers/pciback/do_flr", "w");
+ if (fd != NULL) {
+ fprintf(fd, PCI_BDF, domain, bus, dev, func);
+ fclose(fd);
+ return 0;
+ }
+ XL_LOG(ctx, XL_LOG_ERROR, "Pciback doesn't support do_flr, cannot flr the
device\n");
+ return -1;
+}
+
+int libxl_wait_for_device_model(struct libxl_ctx *ctx, uint32_t domid, char
*state)
+{
+ char path[50];
+ char *p;
+ int watchdog = 100;
+ unsigned int len;
+
+ snprintf(path, sizeof(path), "/local/domain/0/device-model/%d/state",
domid);
+ while (watchdog > 0) {
+ p = xs_read(ctx->xsh, XBT_NULL, path, &len);
+ if (p == NULL) {
+ usleep(100000);
+ watchdog--;
+ } else {
+ if (state == NULL || !strcmp(state, p)) {
+ free(p);
+ return 0;
+ } else {
+ free(p);
+ usleep(100000);
+ watchdog--;
+ }
+ }
+ }
+ XL_LOG(ctx, XL_LOG_ERROR, "Device Model not ready\n");
+ return -1;
+}
+
+int libxl_wait_for_backend(struct libxl_ctx *ctx, char *be_path, char *state)
+{
+ int watchdog = 100;
+ unsigned int len;
+ char *p;
+ char *path = libxl_sprintf(ctx, "%s/state", be_path);
+
+ while (watchdog > 0) {
+ p = xs_read(ctx->xsh, XBT_NULL, path, &len);
+ if (p == NULL) {
+ XL_LOG(ctx, XL_LOG_ERROR, "Backend %s does not exist\n", be_path);
+ return -1;
+ } else {
+ if (!strcmp(p, state)) {
+ return 0;
+ } else {
+ usleep(100000);
+ watchdog--;
+ }
+ }
+ }
+ XL_LOG(ctx, XL_LOG_ERROR, "Backend %s not ready\n", be_path);
+ return -1;
+}
+
diff -r 32a27fe01e07 tools/libxl/libxl_internal.h
--- a/tools/libxl/libxl_internal.h Thu Nov 12 15:34:37 2009 +0000
+++ b/tools/libxl/libxl_internal.h Thu Nov 12 18:04:19 2009 +0000
@@ -72,6 +72,12 @@
libxl_device_kinds kind;
} libxl_device;
+#define XC_PCI_BDF "0x%x, 0x%x, 0x%x, 0x%x"
+#define AUTO_PHP_SLOT 0x100
+#define SYSFS_PCI_DEV /sys/bus/pci/devices
+#define PROC_PCI_NUM_RESOURCES 7
+#define PCI_BAR_IO 0x01
+
#define PRINTF_ATTRIBUTE(x, y) __attribute__((format(printf, x, y)))
/* memory allocation tracking/helpers */
@@ -92,7 +98,7 @@
char *libxl_xs_read(struct libxl_ctx *ctx, xs_transaction_t t, char *path);
char **libxl_xs_directory(struct libxl_ctx *ctx, xs_transaction_t t, char
*path, unsigned int *nb);
-/* from xd_dom */
+/* from xl_dom */
int is_hvm(struct libxl_ctx *ctx, uint32_t domid);
int build_pre(struct libxl_ctx *ctx, uint32_t domid,
libxl_domain_build_info *info, libxl_domain_build_state *state);
@@ -109,7 +115,7 @@
libxl_domain_build_info *info, libxl_domain_build_state
*state, int fd);
int core_suspend(struct libxl_ctx *ctx, uint32_t domid, int fd, int hvm, int
live, int debug);
-/* from xd_device */
+/* from xl_device */
char *device_disk_backend_type_of_phystype(libxl_disk_phystype phystype);
char *device_disk_string_of_phystype(libxl_disk_phystype phystype);
@@ -120,13 +126,17 @@
char **bents, char **fents);
int libxl_device_destroy(struct libxl_ctx *ctx, char *be_path, int force);
int libxl_devices_destroy(struct libxl_ctx *ctx, uint32_t domid, int force);
+int libxl_wait_for_device_model(struct libxl_ctx *ctx, uint32_t domid, char
*state);
+int libxl_wait_for_backend(struct libxl_ctx *ctx, char *be_path, char *state);
+int libxl_device_pci_flr(struct libxl_ctx *ctx, unsigned int domain, unsigned
int bus,
+ unsigned int dev, unsigned int func);
/* from xenguest (helper */
int hvm_build_set_params(int handle, uint32_t domid,
int apic, int acpi, int pae, int nx, int viridian,
int vcpus, int store_evtchn, unsigned long
*store_mfn);
-/* xd_exec */
+/* xl_exec */
int libxl_exec(struct libxl_ctx *ctx, int stdinfd, int stdoutfd, int stderrfd,
char *arg0, char **args);
diff -r 32a27fe01e07 tools/libxl/xl.c
--- a/tools/libxl/xl.c Thu Nov 12 15:34:37 2009 +0000
+++ b/tools/libxl/xl.c Thu Nov 12 18:04:19 2009 +0000
@@ -40,6 +40,8 @@
int num_disks,
libxl_device_nic *vifs,
int num_vifs,
+ libxl_device_pci *pcidevs,
+ int num_pcidevs,
libxl_device_model_info *dm_info)
{
int i;
@@ -104,6 +106,12 @@
printf("model %s\n", vifs[i].model);
printf("mac %02x:%02x:%02x:%02x:%02x:%02x\n", vifs[i].mac[0],
vifs[i].mac[1], vifs[i].mac[2], vifs[i].mac[3], vifs[i].mac[4], vifs[i].mac[5]);
printf("smac %s\n", vifs[i].mac);
+ }
+
+ for (i = 0; i < num_pcidevs; i++) {
+ printf("\n\n\n*** pcidevs_info: %d ***\n", i);
+ printf("pci dev "PCI_BDF_VDEVFN"\n", pcidevs[i].domain,
pcidevs[i].bus, pcidevs[i].dev, pcidevs[i].func, pcidevs[i].vdevfn);
+ printf("opts msitranslate %d power_mgmt %d\n",
pcidevs[i].msitranslate, pcidevs[i].power_mgmt);
}
printf("\n\n\n*** device_model_info ***\n");
@@ -286,13 +294,17 @@
int *num_disks,
libxl_device_nic **vifs,
int *num_vifs,
+ libxl_device_pci **pcidevs,
+ int *num_pcidevs,
libxl_device_model_info *dm_info)
{
const char *buf;
xen_uuid_t uuid[16];
long l;
struct config_t config;
- struct config_setting_t *vbds, *nics;
+ struct config_setting_t *vbds, *nics, *pcis;
+ int pci_power_mgmt = 0;
+ int pci_msitranslate = 1;
config_init (&config);
@@ -482,6 +494,48 @@
}
}
+ if (config_lookup_int (&config, "pci_msitranslate", &l) == CONFIG_TRUE)
+ pci_msitranslate = l;
+
+ if (config_lookup_int (&config, "pci_power_mgmt", &l) == CONFIG_TRUE)
+ pci_power_mgmt = l;
+
+ if ((pcis = config_lookup (&config, "pci")) != NULL) {
+ *num_pcidevs = 0;
+ *pcidevs = NULL;
+ while ((buf = config_setting_get_string_elem (pcis, *num_pcidevs)) !=
NULL) {
+ unsigned int domain = 0, bus = 0, dev = 0, func = 0, vdevfn = 0;
+ char *buf2 = strdup(buf);
+ char *p;
+ *pcidevs = (libxl_device_pci *) realloc(*pcidevs, sizeof
(libxl_device_pci) * ((*num_pcidevs) + 1));
+ memset(*pcidevs + *num_pcidevs, 0x00, sizeof(libxl_device_pci));
+ p = strtok(buf2, ",");
+ if (!p)
+ goto skip_pci;
+ if (!sscanf(p, PCI_BDF_VDEVFN, &domain, &bus, &dev, &func,
&vdevfn)) {
+ sscanf(p, "%02x:%02x.%01x@%02x", &bus, &dev, &func, &vdevfn);
+ domain = 0;
+ }
+ libxl_device_pci_init(*pcidevs + *num_pcidevs, domain, bus, dev,
func, vdevfn);
+ (*pcidevs)[*num_pcidevs].msitranslate = pci_msitranslate;
+ (*pcidevs)[*num_pcidevs].power_mgmt = pci_power_mgmt;
+ while ((p = strtok(NULL, ",=")) != NULL) {
+ while (*p == ' ')
+ p++;
+ if (!strcmp(p, "msitranslate")) {
+ p = strtok(NULL, ",=");
+ (*pcidevs)[*num_pcidevs].msitranslate = atoi(p);
+ } else if (!strcmp(p, "power_mgmt")) {
+ p = strtok(NULL, ",=");
+ (*pcidevs)[*num_pcidevs].power_mgmt = atoi(p);
+ }
+ }
+ *num_pcidevs = (*num_pcidevs) + 1;
+skip_pci:
+ free(buf2);
+ }
+ }
+
/* init dm from c and b */
init_dm_info(dm_info, c_info, b_info);
@@ -527,13 +581,14 @@
libxl_device_model_info dm_info;
libxl_device_disk *disks = NULL;
libxl_device_nic *vifs = NULL;
- int num_disks = 0, num_vifs = 0;
+ libxl_device_pci *pcidevs = NULL;
+ int num_disks = 0, num_vifs = 0, num_pcidevs = 0;
int i;
printf("Parsing config file %s\n", filename);
- parse_config_file(filename, &info1, &info2, &disks, &num_disks, &vifs,
&num_vifs, &dm_info);
+ parse_config_file(filename, &info1, &info2, &disks, &num_disks, &vifs,
&num_vifs, &pcidevs, &num_pcidevs, &dm_info);
if (debug)
- printf_info(&info1, &info2, disks, num_disks, vifs, num_vifs,
&dm_info);
+ printf_info(&info1, &info2, disks, num_disks, vifs, num_vifs, pcidevs,
num_pcidevs, &dm_info);
libxl_ctx_init(&ctx);
libxl_ctx_set_log(&ctx, log_callback, NULL);
@@ -551,6 +606,8 @@
libxl_device_nic_add(&ctx, domid, &vifs[i]);
}
libxl_create_device_model(&ctx, &dm_info, vifs, num_vifs);
+ for (i = 0; i < num_pcidevs; i++)
+ libxl_device_pci_add(&ctx, domid, &pcidevs[i]);
libxl_domain_unpause(&ctx, domid);
}
@@ -560,9 +617,12 @@
if (!command || !strcmp(command, "help")) {
printf("Usage xl <subcommand> [args]\n\n");
printf("xl full list of subcommands:\n\n");
- printf(" create create a domain from
config file <filename>\n\n");
+ printf(" create create a domain from config
file <filename>\n\n");
printf(" list list information about all
domains\n\n");
printf(" destroy terminate a domain
immediately\n\n");
+ printf(" pci-attach insert a new pass-through pci
device\n\n");
+ printf(" pci-detach remove a domain's pass-through
pci device\n\n");
+ printf(" pci-list list pass-through pci devices
for a domain\n\n");
} else if(!strcmp(command, "create")) {
printf("Usage: xl create <ConfigFile> [options] [vars]\n\n");
printf("Create a domain based on <ConfigFile>.\n\n");
@@ -572,11 +632,166 @@
} else if(!strcmp(command, "list")) {
printf("Usage: xl list [Domain]\n\n");
printf("List information about all/some domains.\n\n");
+ } else if(!strcmp(command, "pci-attach")) {
+ printf("Usage: xl pci-attach <Domain> <BDF> [Virtual Slot]\n\n");
+ printf("Insert a new pass-through pci device.\n\n");
+ } else if(!strcmp(command, "pci-detach")) {
+ printf("Usage: xl pci-detach <Domain> <BDF>\n\n");
+ printf("Remove a domain's pass-through pci device.\n\n");
+ } else if(!strcmp(command, "pci-list")) {
+ printf("Usage: xl pci-list <Domain>\n\n");
+ printf("List pass-through pci devices for a domain.\n\n");
} else if(!strcmp(command, "destroy")) {
printf("Usage: xl destroy <Domain>\n\n");
printf("Terminate a domain immediately.\n\n");
}
}
+
+void pcilist(char *dom)
+{
+ struct libxl_ctx ctx;
+ uint32_t domid;
+ libxl_device_pci *pcidevs;
+ int num, i;
+
+ libxl_ctx_init(&ctx);
+ libxl_ctx_set_log(&ctx, log_callback, NULL);
+
+ if (libxl_param_to_domid(&ctx, dom, &domid) < 0) {
+ fprintf(stderr, "%s is an invalid domain identifier\n", dom);
+ exit(2);
+ }
+ pcidevs = libxl_device_pci_list(&ctx, domid, &num);
+ if (!num)
+ return;
+ printf("VFn domain bus slot func\n");
+ for (i = 0; i < num; i++) {
+ printf("0x%02x 0x%04x 0x%02x 0x%02x 0x%01x\n", pcidevs[i].vdevfn,
pcidevs[i].domain, pcidevs[i].bus, pcidevs[i].dev, pcidevs[i].func);
+ }
+}
+
+int main_pcilist(int argc, char **argv)
+{
+ int opt;
+ char *domname = NULL;
+
+ while ((opt = getopt(argc, argv, "h")) != -1) {
+ switch (opt) {
+ case 'h':
+ help("pci-list");
+ exit(0);
+ default:
+ fprintf(stderr, "option not supported\n");
+ break;
+ }
+ }
+ if (optind >= argc) {
+ help("pci-list");
+ exit(2);
+ }
+
+ domname = argv[optind];
+
+ pcilist(domname);
+ exit(0);
+}
+
+void pcidetach(char *dom, char *bdf)
+{
+ struct libxl_ctx ctx;
+ uint32_t domid;
+ libxl_device_pci pcidev;
+ unsigned int domain, bus, dev, func;
+
+ libxl_ctx_init(&ctx);
+ libxl_ctx_set_log(&ctx, log_callback, NULL);
+
+ if (libxl_param_to_domid(&ctx, dom, &domid) < 0) {
+ fprintf(stderr, "%s is an invalid domain identifier\n", dom);
+ exit(2);
+ }
+ memset(&pcidev, 0x00, sizeof(pcidev));
+ sscanf(bdf, PCI_BDF, &domain, &bus, &dev, &func);
+ libxl_device_pci_init(&pcidev, domain, bus, dev, func, 0);
+ libxl_device_pci_remove(&ctx, domid, &pcidev);
+}
+
+int main_pcidetach(int argc, char **argv)
+{
+ int opt;
+ char *domname = NULL, *bdf = NULL;
+
+ while ((opt = getopt(argc, argv, "h")) != -1) {
+ switch (opt) {
+ case 'h':
+ help("pci-attach");
+ exit(0);
+ default:
+ fprintf(stderr, "option not supported\n");
+ break;
+ }
+ }
+ if (optind >= argc - 1) {
+ help("pci-detach");
+ exit(2);
+ }
+
+ domname = argv[optind];
+ bdf = argv[optind + 1];
+
+ pcidetach(domname, bdf);
+ exit(0);
+}
+void pciattach(char *dom, char *bdf, char *vs)
+{
+ struct libxl_ctx ctx;
+ uint32_t domid;
+ libxl_device_pci pcidev;
+ unsigned int domain, bus, dev, func;
+
+ libxl_ctx_init(&ctx);
+ libxl_ctx_set_log(&ctx, log_callback, NULL);
+
+ if (libxl_param_to_domid(&ctx, dom, &domid) < 0) {
+ fprintf(stderr, "%s is an invalid domain identifier\n", dom);
+ exit(2);
+ }
+ memset(&pcidev, 0x00, sizeof(pcidev));
+ sscanf(bdf, PCI_BDF, &domain, &bus, &dev, &func);
+ libxl_device_pci_init(&pcidev, domain, bus, dev, func, 0);
+ libxl_device_pci_add(&ctx, domid, &pcidev);
+}
+
+int main_pciattach(int argc, char **argv)
+{
+ int opt;
+ char *domname = NULL, *bdf = NULL, *vs = NULL;
+
+ while ((opt = getopt(argc, argv, "h")) != -1) {
+ switch (opt) {
+ case 'h':
+ help("pci-attach");
+ exit(0);
+ default:
+ fprintf(stderr, "option not supported\n");
+ break;
+ }
+ }
+ if (optind >= argc - 1) {
+ help("pci-attach");
+ exit(2);
+ }
+
+ domname = argv[optind];
+ bdf = argv[optind + 1];
+
+ if (optind + 1 < argc)
+ vs = argv[optind + 2];
+
+ pciattach(domname, bdf, vs);
+ exit(0);
+}
+
void destroy_domain(char *p)
{
@@ -713,6 +928,12 @@
main_list(argc - 1, argv + 1);
} else if (!strcmp(argv[1], "destroy")) {
main_destroy(argc - 1, argv + 1);
+ } else if (!strcmp(argv[1], "pci-attach")) {
+ main_pciattach(argc - 1, argv + 1);
+ } else if (!strcmp(argv[1], "pci-detach")) {
+ main_pcidetach(argc - 1, argv + 1);
+ } else if (!strcmp(argv[1], "pci-list")) {
+ main_pcilist(argc - 1, argv + 1);
} else if (!strcmp(argv[1], "help")) {
if (argc > 2)
help(argv[2]);
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
|
Home