WARNING - OLD ARCHIVES

This is an archived copy of the Xen.org mailing list, which we have preserved to ensure that existing links to archives are not broken. The live archive, which contains the latest emails, can be found at http://lists.xen.org/
   
 
 
Xen 
 
Home Products Support Community News
 
   
 

xen-devel

[Xen-devel] [PATCH, RFC 6/7] PCI multi-seg: Pass-through adjustments

To: "xen-devel@xxxxxxxxxxxxxxxxxxx" <xen-devel@xxxxxxxxxxxxxxxxxxx>
Subject: [Xen-devel] [PATCH, RFC 6/7] PCI multi-seg: Pass-through adjustments
From: "Jan Beulich" <JBeulich@xxxxxxxxxx>
Date: Thu, 25 Aug 2011 15:58:57 +0100
Delivery-date: Thu, 25 Aug 2011 08:10:20 -0700
Envelope-to: www-data@xxxxxxxxxxxxxxxxxxx
List-help: <mailto:xen-devel-request@lists.xensource.com?subject=help>
List-id: Xen developer discussion <xen-devel.lists.xensource.com>
List-post: <mailto:xen-devel@lists.xensource.com>
List-subscribe: <http://lists.xensource.com/mailman/listinfo/xen-devel>, <mailto:xen-devel-request@lists.xensource.com?subject=subscribe>
List-unsubscribe: <http://lists.xensource.com/mailman/listinfo/xen-devel>, <mailto:xen-devel-request@lists.xensource.com?subject=unsubscribe>
Sender: xen-devel-bounces@xxxxxxxxxxxxxxxxxxx
Signed-off-by: Jan Beulich <jbeulich@xxxxxxxxxx>

--- 2011-08-25.orig/xen/drivers/passthrough/amd/pci_amd_iommu.c 2011-08-25 
15:06:47.000000000 +0200
+++ 2011-08-25/xen/drivers/passthrough/amd/pci_amd_iommu.c      2011-08-25 
15:12:12.000000000 +0200
@@ -123,35 +123,17 @@ static void amd_iommu_setup_domain_devic
     spin_unlock_irqrestore(&iommu->lock, flags);
 }
 
-static void __init amd_iommu_setup_dom0_devices(struct domain *d)
+static void __init amd_iommu_setup_dom0_device(struct pci_dev *pdev)
 {
-    struct amd_iommu *iommu;
-    struct pci_dev *pdev;
-    int bus, devfn, bdf;
+    int bdf = (pdev->bus << 8) | pdev->devfn;
+    struct amd_iommu *iommu = find_iommu_for_device(pdev->seg, bdf);
 
-    spin_lock(&pcidevs_lock);
-    for ( bus = 0; bus < 256; bus++ )
-    {
-        for ( devfn = 0; devfn < 256; devfn++ )
-        {
-            pdev = pci_get_pdev(0, bus, devfn);
-            if ( !pdev )
-                continue;
-
-            pdev->domain = d;
-            list_add(&pdev->domain_list, &d->arch.pdev_list);
-
-            bdf = (bus << 8) | devfn;
-            iommu = find_iommu_for_device(pdev->seg, bdf);
-
-            if ( likely(iommu != NULL) )
-                amd_iommu_setup_domain_device(d, iommu, bdf);
-            else
-                AMD_IOMMU_DEBUG("No iommu for device %02x:%02x.%x\n",
-                                bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
-        }
-    }
-    spin_unlock(&pcidevs_lock);
+    if ( likely(iommu != NULL) )
+        amd_iommu_setup_domain_device(pdev->domain, iommu, bdf);
+    else
+        AMD_IOMMU_DEBUG("No iommu for device %04x:%02x:%02x.%u\n",
+                        pdev->seg, pdev->bus,
+                        PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
 }
 
 int __init amd_iov_detect(void)
@@ -262,7 +244,7 @@ static void __init amd_iommu_dom0_init(s
         }
     }
 
-    amd_iommu_setup_dom0_devices(d);
+    setup_dom0_pci_devices(d, amd_iommu_setup_dom0_device);
 }
 
 static void amd_iommu_disable_domain_device(
--- 2011-08-25.orig/xen/drivers/passthrough/pci.c       2011-08-25 
15:06:40.000000000 +0200
+++ 2011-08-25/xen/drivers/passthrough/pci.c    2011-08-25 15:12:12.000000000 
+0200
@@ -202,9 +202,7 @@ struct pci_dev *pci_get_pdev_by_domain(
 void pci_enable_acs(struct pci_dev *pdev)
 {
     int pos;
-    u16 cap;
-    u16 ctrl;
-
+    u16 cap, ctrl, seg = pdev->seg;
     u8 bus = pdev->bus;
     u8 dev = PCI_SLOT(pdev->devfn);
     u8 func = PCI_FUNC(pdev->devfn);
@@ -212,7 +210,7 @@ void pci_enable_acs(struct pci_dev *pdev
     if ( !iommu_enabled )
         return;
 
-    pos = pci_find_ext_capability(0, bus, pdev->devfn, PCI_EXT_CAP_ID_ACS);
+    pos = pci_find_ext_capability(seg, bus, pdev->devfn, PCI_EXT_CAP_ID_ACS);
     if (!pos)
         return;
 
@@ -453,7 +451,7 @@ void pci_release_devices(struct domain *
 
 #define PCI_CLASS_BRIDGE_PCI     0x0604
 
-int pdev_type(u8 bus, u8 devfn)
+int pdev_type(u16 seg, u8 bus, u8 devfn)
 {
     u16 class_device;
     u16 status, creg;
@@ -488,9 +486,9 @@ int pdev_type(u8 bus, u8 devfn)
  * return 1: find PCIe-to-PCI/PCIX bridge or PCI legacy bridge
  * return -1: fail
  */
-int find_upstream_bridge(u8 *bus, u8 *devfn, u8 *secbus)
+int find_upstream_bridge(u16 seg, u8 *bus, u8 *devfn, u8 *secbus)
 {
-    struct pci_seg *pseg = get_pseg(0);
+    struct pci_seg *pseg = get_pseg(seg);
     int ret = 0;
     int cnt = 0;
 
@@ -525,7 +523,7 @@ out:
 /*
  * detect pci device, return 0 if it exists, or return 0
  */
-int __init pci_device_detect(u8 bus, u8 dev, u8 func)
+int __init pci_device_detect(u16 seg, u8 bus, u8 dev, u8 func)
 {
     u32 vendor;
 
@@ -554,7 +552,7 @@ static int __init _scan_pci_devices(stru
         {
             for ( func = 0; func < 8; func++ )
             {
-                if ( pci_device_detect(bus, dev, func) == 0 )
+                if ( pci_device_detect(pseg->nr, bus, dev, func) == 0 )
                     continue;
 
                 pdev = alloc_pdev(pseg, bus, PCI_DEVFN(dev, func));
@@ -565,7 +563,7 @@ static int __init _scan_pci_devices(stru
                 }
 
                 /* build bus2bridge */
-                type = pdev_type(bus, PCI_DEVFN(dev, func));
+                type = pdev_type(pseg->nr, bus, PCI_DEVFN(dev, func));
                 switch ( type )
                 {
                     case DEV_TYPE_PCIe_BRIDGE:
@@ -594,8 +592,8 @@ static int __init _scan_pci_devices(stru
                         break;
 
                     default:
-                        printk("%s: unknown type: bdf = %x:%x.%x\n",
-                               __func__, bus, dev, func);
+                        printk("%s: unknown type: %04x:%02x:%02x.%u\n",
+                               __func__, pseg->nr, bus, dev, func);
                         return -EINVAL;
                 }
 
@@ -620,6 +618,44 @@ int __init scan_pci_devices(void)
     return ret;
 }
 
+struct setup_dom0 {
+    struct domain *d;
+    void (*handler)(struct pci_dev *);
+};
+
+static int __init _setup_dom0_pci_devices(struct pci_seg *pseg, void *arg)
+{
+    struct setup_dom0 *ctxt = arg;
+    int bus, devfn;
+
+    for ( bus = 0; bus < 256; bus++ )
+    {
+        for ( devfn = 0; devfn < 256; devfn++ )
+        {
+            struct pci_dev *pdev = pci_get_pdev(pseg->nr, bus, devfn);
+
+            if ( !pdev )
+                continue;
+
+            pdev->domain = ctxt->d;
+            list_add(&pdev->domain_list, &ctxt->d->arch.pdev_list);
+            ctxt->handler(pdev);
+        }
+    }
+
+    return 0;
+}
+
+void __init setup_dom0_pci_devices(
+    struct domain *d, void (*handler)(struct pci_dev *))
+{
+    struct setup_dom0 ctxt = { .d = d, .handler = handler };
+
+    spin_lock(&pcidevs_lock);
+    pci_segments_iterate(_setup_dom0_pci_devices, &ctxt);
+    spin_unlock(&pcidevs_lock);
+}
+
 /* Disconnect all PCI devices from the PCI buses. From the PCI spec:
  *   "When a 0 is written to [the COMMAND] register, the device is
  *    logically disconnected from the PCI bus for all accesses except
@@ -654,8 +690,9 @@ static int _dump_pci_devices(struct pci_
 
     list_for_each_entry ( pdev, &pseg->alldevs_list, alldevs_list )
     {
-        printk("%02x:%02x.%x - dom %-3d - MSIs < ",
-               pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
+        printk("%04x:%02x:%02x.%u - dom %-3d - MSIs < ",
+               pseg->nr, pdev->bus,
+               PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
                pdev->domain ? pdev->domain->domain_id : -1);
         list_for_each_entry ( msi, &pdev->msi_list, list )
                printk("%d ", msi->irq);
--- 2011-08-25.orig/xen/drivers/passthrough/vtd/dmar.c  2011-08-25 
15:06:43.000000000 +0200
+++ 2011-08-25/xen/drivers/passthrough/vtd/dmar.c       2011-08-25 
15:12:12.000000000 +0200
@@ -457,11 +457,11 @@ acpi_parse_one_drhd(struct acpi_dmar_ent
             d = PCI_SLOT(dmaru->scope.devices[i]);
             f = PCI_FUNC(dmaru->scope.devices[i]);
 
-            if ( pci_device_detect(b, d, f) == 0 )
+            if ( pci_device_detect(drhd->segment, b, d, f) == 0 )
             {
                 dprintk(XENLOG_WARNING VTDPREFIX,
-                    "  Non-existent device (%x:%x.%x) is reported "
-                    "in this DRHD's scope!\n", b, d, f);
+                        " Non-existent device (%04x:%02x:%02x.%u) is reported"
+                        " in this DRHD's scope!\n", drhd->segment, b, d, f);
                 invalid_cnt++;
             }
         }
@@ -556,12 +556,13 @@ acpi_parse_one_rmrr(struct acpi_dmar_ent
             d = PCI_SLOT(rmrru->scope.devices[i]);
             f = PCI_FUNC(rmrru->scope.devices[i]);
 
-            if ( pci_device_detect(b, d, f) == 0 )
+            if ( pci_device_detect(rmrr->segment, b, d, f) == 0 )
             {
                 dprintk(XENLOG_WARNING VTDPREFIX,
-                    "  Non-existent device (%x:%x.%x) is reported "
-                    "in RMRR (%"PRIx64", %"PRIx64")'s scope!\n",
-                    b, d, f, rmrru->base_address, rmrru->end_address);
+                        " Non-existent device (%04x:%02x:%02x.%u) is reported"
+                        " in RMRR (%"PRIx64", %"PRIx64")'s scope!\n",
+                        rmrr->segment, b, d, f,
+                        rmrru->base_address, rmrru->end_address);
                 ignore = 1;
             }
             else
--- 2011-08-25.orig/xen/drivers/passthrough/vtd/intremap.c      2011-08-19 
17:08:35.000000000 +0200
+++ 2011-08-25/xen/drivers/passthrough/vtd/intremap.c   2011-08-25 
15:12:12.000000000 +0200
@@ -448,15 +448,17 @@ void io_apic_write_remap_rte(
 static void set_msi_source_id(struct pci_dev *pdev, struct iremap_entry *ire)
 {
     int type;
+    u16 seg;
     u8 bus, devfn, secbus;
     int ret;
 
     if ( !pdev || !ire )
         return;
 
+    seg = pdev->seg;
     bus = pdev->bus;
     devfn = pdev->devfn;
-    type = pdev_type(bus, devfn);
+    type = pdev_type(seg, bus, devfn);
     switch ( type )
     {
     case DEV_TYPE_PCIe_BRIDGE:
@@ -469,7 +471,7 @@ static void set_msi_source_id(struct pci
         break;
 
     case DEV_TYPE_PCI:
-        ret = find_upstream_bridge(&bus, &devfn, &secbus);
+        ret = find_upstream_bridge(seg, &bus, &devfn, &secbus);
         if ( ret == 0 ) /* integrated PCI device */
         {
             set_ire_sid(ire, SVT_VERIFY_SID_SQ, SQ_ALL_16,
@@ -477,19 +479,20 @@ static void set_msi_source_id(struct pci
         }
         else if ( ret == 1 ) /* find upstream bridge */
         {
-            if ( pdev_type(bus, devfn) == DEV_TYPE_PCIe2PCI_BRIDGE )
+            if ( pdev_type(seg, bus, devfn) == DEV_TYPE_PCIe2PCI_BRIDGE )
                 set_ire_sid(ire, SVT_VERIFY_BUS, SQ_ALL_16,
                             (bus << 8) | pdev->bus);
-            else if ( pdev_type(bus, devfn) == DEV_TYPE_LEGACY_PCI_BRIDGE )
+            else if ( pdev_type(seg, bus, devfn) == DEV_TYPE_LEGACY_PCI_BRIDGE 
)
                 set_ire_sid(ire, SVT_VERIFY_BUS, SQ_ALL_16,
                             PCI_BDF2(bus, devfn));
         }
         break;
 
     default:
-        dprintk(XENLOG_WARNING VTDPREFIX, "d%d: unknown(%u): bdf = %x:%x.%x\n",
+        dprintk(XENLOG_WARNING VTDPREFIX,
+                "d%d: unknown(%u): %04x:%02x:%02x.%u\n",
                 pdev->domain->domain_id, type,
-                bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
+                seg, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
         break;
    }
 }
--- 2011-08-25.orig/xen/drivers/passthrough/vtd/iommu.c 2011-08-25 
15:06:43.000000000 +0200
+++ 2011-08-25/xen/drivers/passthrough/vtd/iommu.c      2011-08-25 
15:12:12.000000000 +0200
@@ -50,7 +50,7 @@ bool_t __read_mostly untrusted_msi;
 
 int nr_iommus;
 
-static void setup_dom0_devices(struct domain *d);
+static void setup_dom0_device(struct pci_dev *);
 static void setup_dom0_rmrr(struct domain *d);
 
 static int domain_iommu_domid(struct domain *d,
@@ -1219,7 +1219,7 @@ static void __init intel_iommu_dom0_init
         iommu_set_dom0_mapping(d);
     }
 
-    setup_dom0_devices(d);
+    setup_dom0_pci_devices(d, setup_dom0_device);
     setup_dom0_rmrr(d);
 
     iommu_flush_all();
@@ -1355,7 +1355,7 @@ static int domain_context_mapping(
 
     ASSERT(spin_is_locked(&pcidevs_lock));
 
-    type = pdev_type(bus, devfn);
+    type = pdev_type(seg, bus, devfn);
     switch ( type )
     {
     case DEV_TYPE_PCIe_BRIDGE:
@@ -1384,7 +1384,7 @@ static int domain_context_mapping(
         if ( ret )
             break;
 
-        if ( find_upstream_bridge(&bus, &devfn, &secbus) < 1 )
+        if ( find_upstream_bridge(seg, &bus, &devfn, &secbus) < 1 )
             break;
 
         ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn);
@@ -1394,7 +1394,7 @@ static int domain_context_mapping(
          * requester-id. It may originate from devfn=0 on the secondary bus
          * behind the bridge. Map that id as well if we didn't already.
          */
-        if ( !ret && pdev_type(bus, devfn) == DEV_TYPE_PCIe2PCI_BRIDGE &&
+        if ( !ret && pdev_type(seg, bus, devfn) == DEV_TYPE_PCIe2PCI_BRIDGE &&
              (secbus != pdev->bus || pdev->devfn != 0) )
             ret = domain_context_mapping_one(domain, drhd->iommu, secbus, 0);
 
@@ -1486,7 +1486,7 @@ static int domain_context_unmap(
         return -ENODEV;
     iommu = drhd->iommu;
 
-    type = pdev_type(bus, devfn);
+    type = pdev_type(seg, bus, devfn);
     switch ( type )
     {
     case DEV_TYPE_PCIe_BRIDGE:
@@ -1515,11 +1515,11 @@ static int domain_context_unmap(
 
         tmp_bus = bus;
         tmp_devfn = devfn;
-        if ( find_upstream_bridge(&tmp_bus, &tmp_devfn, &secbus) < 1 )
+        if ( find_upstream_bridge(seg, &tmp_bus, &tmp_devfn, &secbus) < 1 )
             break;
 
         /* PCIe to PCI/PCIx bridge */
-        if ( pdev_type(tmp_bus, tmp_devfn) == DEV_TYPE_PCIe2PCI_BRIDGE )
+        if ( pdev_type(seg, tmp_bus, tmp_devfn) == DEV_TYPE_PCIe2PCI_BRIDGE )
         {
             ret = domain_context_unmap_one(domain, iommu, tmp_bus, tmp_devfn);
             if ( ret )
@@ -1892,28 +1892,11 @@ static int intel_iommu_remove_device(str
                                 pdev->devfn);
 }
 
-static void __init setup_dom0_devices(struct domain *d)
+static void __init setup_dom0_device(struct pci_dev *pdev)
 {
-    struct pci_dev *pdev;
-    int bus, devfn;
-
-    spin_lock(&pcidevs_lock);
-    for ( bus = 0; bus < 256; bus++ )
-    {
-        for ( devfn = 0; devfn < 256; devfn++ )
-        {
-            pdev = pci_get_pdev(0, bus, devfn);
-            if ( !pdev )
-                continue;
-
-            pdev->domain = d;
-            list_add(&pdev->domain_list, &d->arch.pdev_list);
-            domain_context_mapping(d, pdev->seg, pdev->bus, pdev->devfn);
-            pci_enable_acs(pdev);
-            pci_vtd_quirk(pdev);
-        }
-    }
-    spin_unlock(&pcidevs_lock);
+    domain_context_mapping(pdev->domain, pdev->seg, pdev->bus, pdev->devfn);
+    pci_enable_acs(pdev);
+    pci_vtd_quirk(pdev);
 }
 
 void clear_fault_bits(struct iommu *iommu)
@@ -2207,7 +2190,7 @@ done:
 static int intel_iommu_group_id(u16 seg, u8 bus, u8 devfn)
 {
     u8 secbus;
-    if ( find_upstream_bridge(&bus, &devfn, &secbus) < 0 )
+    if ( find_upstream_bridge(seg, &bus, &devfn, &secbus) < 0 )
         return -1;
     else
         return PCI_BDF2(bus, devfn);
--- 2011-08-25.orig/xen/include/xen/pci.h       2011-08-25 15:06:35.000000000 
+0200
+++ 2011-08-25/xen/include/xen/pci.h    2011-08-25 15:12:12.000000000 +0200
@@ -82,13 +82,15 @@ enum {
     DEV_TYPE_PCI,
 };
 
-int pci_device_detect(u8 bus, u8 dev, u8 func);
+int pci_device_detect(u16 seg, u8 bus, u8 dev, u8 func);
 int scan_pci_devices(void);
-int pdev_type(u8 bus, u8 devfn);
-int find_upstream_bridge(u8 *bus, u8 *devfn, u8 *secbus);
-struct pci_dev *pci_lock_pdev(int bus, int devfn);
-struct pci_dev *pci_lock_domain_pdev(struct domain *d, int bus, int devfn);
+int pdev_type(u16 seg, u8 bus, u8 devfn);
+int find_upstream_bridge(u16 seg, u8 *bus, u8 *devfn, u8 *secbus);
+struct pci_dev *pci_lock_pdev(int seg, int bus, int devfn);
+struct pci_dev *pci_lock_domain_pdev(
+    struct domain *, int seg, int bus, int devfn);
 
+void setup_dom0_pci_devices(struct domain *, void (*)(struct pci_dev *));
 void pci_release_devices(struct domain *d);
 int pci_add_segment(u16 seg);
 int pci_add_device(u16 seg, u8 bus, u8 devfn, const struct pci_dev_info *);


Attachment: pci-multi-seg-pt.patch
Description: Text document

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
<Prev in Thread] Current Thread [Next in Thread>
  • [Xen-devel] [PATCH, RFC 6/7] PCI multi-seg: Pass-through adjustments, Jan Beulich <=