This patch enables saving/restoring PCI configuration space in
pciback.
The timing of saving/restoring configuration space is like below.
When pciback is bound to devices.
- Pciback saves configuration space.
When pciback is unbound to devices.
- Pciback restores configuration space.
When guest OS boots or a device is hotadded.
- Pciback restores configuration space.
- Pciback changes state of backend device to Initialised/Reconfigured.
- Xend waits for the transition to Initialised/Reconfigured.
When guest OS shutdowns or a device is hotremoved.
- Pciback restores configuration space.
- Xend resets devices.
* If D-state of the device is not D0, the state is changed to D0
before resetting the device.
- Xend deassigns devices.
* Essentially, devices should be reset before configuration space is
restored. But it needs big modifications. Applying these patches,
configuration space is restored when guest OS boots, a device is
hotadded or pciback is unbound. So it has no matter.
The following registers are saved/restored by pciback.
Configuration Space
- Base Address Register set
- Cache-line size Register
- Latency timer Register
- Enable SERR Bit / Enable PERR Bit in Control Register
- Device Control Register (PCI Express Capability)
- Link Control Register (PCI Express Capability)
- Device Control 2 Register (PCI Express Capability)
- Link Control 2 Register (PCI Express Capability)
AER
- Uncorrectable Error Mask Register
- Uncorrectable Error Severity Register
- Correctable Error Mask Register
- Advanced Error Capabilities and Control Register
Thanks,
--
Yuji Shimada
Signed-off-by: Yuji Shimada <shimada-yxb@xxxxxxxxxxxxxxx>
diff -r 366c31f3ab4b drivers/xen/pciback/Makefile
--- a/drivers/xen/pciback/Makefile Tue Apr 14 11:17:47 2009 +0100
+++ b/drivers/xen/pciback/Makefile Tue Apr 21 16:48:25 2009 +0900
@@ -5,6 +5,7 @@
conf_space_capability.o \
conf_space_capability_vpd.o \
conf_space_capability_pm.o \
+ conf_space_capability_exp.o \
conf_space_quirks.o
pciback-$(CONFIG_PCI_MSI) += conf_space_capability_msi.o
pciback-$(CONFIG_XEN_PCIDEV_BACKEND_VPCI) += vpci.o
diff -r 366c31f3ab4b drivers/xen/pciback/conf_space.c
--- a/drivers/xen/pciback/conf_space.c Tue Apr 14 11:17:47 2009 +0100
+++ b/drivers/xen/pciback/conf_space.c Tue Apr 21 16:48:25 2009 +0900
@@ -375,6 +375,18 @@
cfg_entry->field = field;
cfg_entry->base_offset = base_offset;
+ /* When the size of the registers is different, by a version of the PCI,
+ * "is_need" function can prevent addition to the list of an unnecessary
+ * register.
+ */
+ if (field->is_need) {
+ err = field->is_need(dev, base_offset);
+ if (!err) {
+ kfree(cfg_entry);
+ goto out;
+ }
+ }
+
/* silently ignore duplicate fields */
err = pciback_field_is_dup(dev,OFFSET(cfg_entry));
if (err)
@@ -433,3 +445,69 @@
{
return pciback_config_capability_init();
}
+
+/* save AER one register */
+static int pciback_aer_save_one_register(struct pci_dev *dev, int offset)
+{
+ struct pciback_dev_data *dev_data = pci_get_drvdata(dev);
+ return pci_read_config_dword(dev, (dev_data->config.aer_base + offset),
+ (u32*)&dev_data->config.config_aer[offset]);
+}
+
+/* save AER registers */
+int pciback_aer_reg_save(struct pci_dev *dev)
+{
+ int err = 0;
+
+ /* after reset, following register values should be restored.
+ * So, save them.
+ */
+ err = pciback_aer_save_one_register(dev, PCI_ERR_UNCOR_MASK);
+ if (err)
+ goto out;
+ err = pciback_aer_save_one_register(dev, PCI_ERR_UNCOR_SEVER);
+ if (err)
+ goto out;
+ err = pciback_aer_save_one_register(dev, PCI_ERR_COR_MASK);
+ if (err)
+ goto out;
+ err = pciback_aer_save_one_register(dev, PCI_ERR_CAP);
+
+ out:
+ return err;
+}
+
+/* restore AER one register */
+static int pciback_aer_restore_one_register(struct pci_dev *dev, int offset)
+{
+ struct pciback_dev_data *dev_data = pci_get_drvdata(dev);
+ return pci_write_config_dword(dev,
+ (dev_data->config.aer_base + offset),
+ *((u32*)&dev_data->config.config[offset]));
+}
+
+/* restore AER registers */
+int pciback_aer_reg_restore(struct pci_dev *dev)
+{
+ int err = 0;
+
+ /* the following registers should be reconfigured to correct values
+ * after reset. restore them.
+ * other registers should not be reconfigured after reset
+ * if there is no reason
+ */
+ err = pciback_aer_restore_one_register(dev, PCI_ERR_UNCOR_MASK);
+ if (err)
+ goto out;
+ err = pciback_aer_restore_one_register(dev, PCI_ERR_UNCOR_SEVER);
+ if (err)
+ goto out;
+ err = pciback_aer_restore_one_register(dev, PCI_ERR_COR_MASK);
+ if (err)
+ goto out;
+ err = pciback_aer_restore_one_register(dev, PCI_ERR_CAP);
+
+ out:
+ return err;
+}
+
diff -r 366c31f3ab4b drivers/xen/pciback/conf_space.h
--- a/drivers/xen/pciback/conf_space.h Tue Apr 14 11:17:47 2009 +0100
+++ b/drivers/xen/pciback/conf_space.h Tue Apr 21 16:48:25 2009 +0900
@@ -27,6 +27,10 @@
void *data);
typedef int (*conf_byte_read) (struct pci_dev * dev, int offset, u8 * value,
void *data);
+typedef int (*conf_dword_restore)(struct pci_dev *dev, int offset, u32 data);
+typedef int (*conf_word_restore)(struct pci_dev *dev, int offset, u16 data);
+typedef int (*conf_byte_restore)(struct pci_dev *dev, int offset, u8 data);
+typedef int (*conf_field_is_need)(struct pci_dev *dev, int offset);
/* These are the fields within the configuration space which we
* are interested in intercepting reads/writes to and changing their
@@ -44,16 +48,20 @@
struct {
conf_dword_write write;
conf_dword_read read;
+ conf_dword_restore restore;
} dw;
struct {
conf_word_write write;
conf_word_read read;
+ conf_word_restore restore;
} w;
struct {
conf_byte_write write;
conf_byte_read read;
+ conf_byte_restore restore;
} b;
} u;
+ conf_field_is_need is_need;
struct list_head list;
};
@@ -123,4 +131,22 @@
int pciback_config_header_add_fields(struct pci_dev *dev);
int pciback_config_capability_add_fields(struct pci_dev *dev);
+static inline int pciback_restore_config_dword(struct pci_dev *dev, int offset,
+ u32 data)
+{
+ return pci_write_config_dword(dev, offset, data);
+}
+
+static inline int pciback_restore_config_word(struct pci_dev *dev, int offset,
+ u16 data)
+{
+ return pci_write_config_word(dev, offset, data);
+}
+
+static inline int pciback_restore_config_byte(struct pci_dev *dev, int offset,
+ u8 data)
+{
+ return pci_write_config_byte(dev, offset, data);
+}
+
#endif /* __XEN_PCIBACK_CONF_SPACE_H__ */
diff -r 366c31f3ab4b drivers/xen/pciback/conf_space_capability.c
--- a/drivers/xen/pciback/conf_space_capability.c Tue Apr 14 11:17:47
2009 +0100
+++ b/drivers/xen/pciback/conf_space_capability.c Tue Apr 21 16:48:25
2009 +0900
@@ -59,11 +59,13 @@
extern struct pciback_config_capability pciback_config_capability_vpd;
extern struct pciback_config_capability pciback_config_capability_pm;
+extern struct pciback_config_capability pciback_config_capability_exp;
int pciback_config_capability_init(void)
{
register_capability(&pciback_config_capability_vpd);
register_capability(&pciback_config_capability_pm);
+ register_capability(&pciback_config_capability_exp);
return 0;
}
diff -r 366c31f3ab4b drivers/xen/pciback/conf_space_capability_exp.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/drivers/xen/pciback/conf_space_capability_exp.c Tue Apr 21 16:48:25
2009 +0900
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2009, NEC Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ */
+/*
+ * PCI Backend -- Configuration overlay for PCI Express capability
+ */
+#include <linux/pci.h>
+#include "conf_space.h"
+#include "conf_space_capability.h"
+
+/* PCI express version 1.0 does not have the registers over offset 24H */
+static int pciback_exp_is_need_config2(struct pci_dev *dev, int base_offset)
+{
+ u16 cap;
+ pci_read_config_word(dev, base_offset + PCI_EXP_FLAGS, &cap);
+ if ((cap & PCI_EXP_FLAGS_VERS) == 1)
+ return 0;
+ else
+ return 1;
+}
+
+static const struct config_field caplist_exp[] = {
+ {
+ .offset = PCI_EXP_DEVCTL,
+ .size = 2,
+ .u.w.restore = pciback_restore_config_word,
+ },
+ {
+ .offset = PCI_EXP_LNKCTL,
+ .size = 2,
+ .u.w.restore = pciback_restore_config_word,
+ },
+ {
+ .offset = PCI_EXP_DEVCTL2,
+ .size = 2,
+ .u.w.restore = pciback_restore_config_word,
+ .is_need = pciback_exp_is_need_config2,
+ },
+ {
+ .offset = PCI_EXP_LNKCTL2,
+ .size = 2,
+ .u.w.restore = pciback_restore_config_word,
+ .is_need = pciback_exp_is_need_config2,
+ },
+ {}
+};
+
+struct pciback_config_capability pciback_config_capability_exp = {
+ .capability = PCI_CAP_ID_EXP,
+ .fields = caplist_exp,
+};
+
diff -r 366c31f3ab4b drivers/xen/pciback/conf_space_capability_pm.c
--- a/drivers/xen/pciback/conf_space_capability_pm.c Tue Apr 14 11:17:47
2009 +0100
+++ b/drivers/xen/pciback/conf_space_capability_pm.c Tue Apr 21 16:48:25
2009 +0900
@@ -94,6 +94,27 @@
return ERR_PTR(err);
}
+/* restore Power Management Control/Status register */
+static int pm_ctrl_restore(struct pci_dev *dev, int offset, u16 data)
+{
+ int err;
+ u16 value;
+
+ err = pci_read_config_word(dev, offset, &value);
+ if (err)
+ goto out;
+
+ /* No need to restore, just clear PME Enable and PME Status bit
+ * Note: register type of PME Status bit is RW1C, so clear by writing 1b
+ */
+ value = (value & ~PCI_PM_CTRL_PME_ENABLE) | PCI_PM_CTRL_PME_STATUS;
+
+ err = pci_write_config_word(dev, offset, value);
+
+ out:
+ return err;
+}
+
static const struct config_field caplist_pm[] = {
{
.offset = PCI_PM_PMC,
@@ -106,6 +127,7 @@
.init = pm_ctrl_init,
.u.w.read = pciback_read_config_word,
.u.w.write = pm_ctrl_write,
+ .u.w.restore = pm_ctrl_restore,
},
{
.offset = PCI_PM_PPB_EXTENSIONS,
diff -r 366c31f3ab4b drivers/xen/pciback/conf_space_header.c
--- a/drivers/xen/pciback/conf_space_header.c Tue Apr 14 11:17:47 2009 +0100
+++ b/drivers/xen/pciback/conf_space_header.c Tue Apr 21 16:48:25 2009 +0900
@@ -210,12 +210,34 @@
return err;
}
+#define CONF_COMMAND_MASK (PCI_COMMAND_PARITY | PCI_COMMAND_SERR)
+
+static int command_restore(struct pci_dev *dev, int offset, u16 data)
+{
+ int err;
+ u16 tmp_val;
+
+ err = pci_read_config_word(dev, offset, &tmp_val);
+ if (err)
+ goto out;
+
+ tmp_val &= ~CONF_COMMAND_MASK;
+ tmp_val |= (data & CONF_COMMAND_MASK);
+
+ err = pci_write_config_word(dev, offset, tmp_val);
+
+ out:
+ return err;
+}
+
+
static const struct config_field header_common[] = {
{
.offset = PCI_COMMAND,
.size = 2,
.u.w.read = pciback_read_config_word,
.u.w.write = command_write,
+ .u.w.restore = command_restore,
},
{
.offset = PCI_INTERRUPT_LINE,
@@ -233,11 +255,13 @@
.size = 1,
.u.b.read = pciback_read_config_byte,
.u.b.write = pciback_write_config_byte,
+ .u.b.restore = pciback_restore_config_byte,
},
{
.offset = PCI_LATENCY_TIMER,
.size = 1,
.u.b.read = pciback_read_config_byte,
+ .u.b.restore = pciback_restore_config_byte,
},
{
.offset = PCI_BIST,
@@ -257,6 +281,7 @@
.release = bar_release, \
.u.dw.read = bar_read, \
.u.dw.write = bar_write, \
+ .u.dw.restore = pciback_restore_config_dword, \
}
#define CFG_FIELD_ROM(reg_offset) \
@@ -268,6 +293,7 @@
.release = bar_release, \
.u.dw.read = bar_read, \
.u.dw.write = rom_write, \
+ .u.dw.restore = pciback_restore_config_dword, \
}
static const struct config_field header_0[] = {
diff -r 366c31f3ab4b drivers/xen/pciback/pci_stub.c
--- a/drivers/xen/pciback/pci_stub.c Tue Apr 14 11:17:47 2009 +0100
+++ b/drivers/xen/pciback/pci_stub.c Tue Apr 21 16:48:25 2009 +0900
@@ -135,6 +135,91 @@
return psdev;
}
+int pcistub_save_config_space(struct pci_dev *dev)
+{
+ int err = 0;
+ int offset;
+ u8 value;
+ struct pciback_dev_data *dev_data = pci_get_drvdata(dev);
+
+ /* save configuration space */
+ for (offset = 0; offset < 256; offset++) {
+ err = pci_read_config_byte(dev, offset, &value);
+ if (err) {
+ dev_err(&dev->dev,
+ "(%s) Read error in config space[%x]!\n",
+ __func__, offset);
+ goto out;
+ }
+ dev_data->config.config[offset] = value;
+ }
+ dev_dbg(&dev->dev, "Save real configuration space \n");
+
+ dev_data->config.aer_base =
+ (u32)pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+ if (!dev_data->config.aer_base)
+ goto out;
+
+ /* save advanced error reporting registers */
+ err = pciback_aer_reg_save(dev);
+ dev_dbg(&dev->dev, "Save advanced error reporting[%d]\n", err);
+
+ out:
+ return err;
+}
+
+int pcistub_restore_config_space(struct pci_dev *dev)
+{
+ int err = 0;
+ struct pciback_dev_data *dev_data = pci_get_drvdata(dev);
+ struct config_field_entry *cfg_entry;
+ const struct config_field *field;
+ int offset;
+
+ /* Restore configuration space */
+ list_for_each_entry(cfg_entry, &dev_data->config_fields, list) {
+ field = cfg_entry->field;
+ offset = OFFSET(cfg_entry);
+ switch (field->size) {
+ case 1:
+ if (!field->u.b.restore)
+ continue;
+ err = field->u.b.restore(dev,
+ offset, dev_data->config.config[offset]);
+ break;
+ case 2:
+ if (!field->u.w.restore)
+ continue;
+ err = field->u.w.restore(dev, offset,
+ *((u16 *)&dev_data->config.config[offset]));
+ break;
+ case 4:
+ if (!field->u.dw.restore)
+ continue;
+ err = field->u.dw.restore(dev, offset,
+ *((u32 *)&dev_data->config.config[offset]));
+ break;
+ }
+ if (err) {
+ dev_err(&dev->dev,
+ "(%s) Restore error<%d> in config space"
+ "at offset 0x%x!\n", __func__, err, offset);
+ goto out;
+ }
+ }
+ dev_dbg(&dev->dev, "Restore base configuration space \n");
+
+ if (!dev_data->config.aer_base)
+ goto out;
+
+ /* restore advanced error reporting registers */
+ err = pciback_aer_reg_restore(dev);
+ dev_dbg(&dev->dev, "Restore advanced error reporting\n");
+
+ out:
+ return err;
+}
+
static struct pci_dev *pcistub_device_get_pci_dev(struct pciback_device *pdev,
struct pcistub_device *psdev)
{
@@ -292,6 +377,10 @@
}
pci_set_drvdata(dev, dev_data);
+ /* Save configuration space */
+ dev_dbg(&dev->dev, "Save configuration space\n");
+ pcistub_save_config_space(dev);
+
dev_dbg(&dev->dev, "initializing config\n");
init_waitqueue_head(&aer_wait_queue);
diff -r 366c31f3ab4b drivers/xen/pciback/pciback.h
--- a/drivers/xen/pciback/pciback.h Tue Apr 14 11:17:47 2009 +0100
+++ b/drivers/xen/pciback/pciback.h Tue Apr 21 16:48:25 2009 +0900
@@ -44,10 +44,17 @@
struct work_struct op_work;
};
+struct pciback_config {
+ u8 config[256];
+ u32 aer_base;
+ u8 config_aer[72];
+};
+
struct pciback_dev_data {
struct list_head config_fields;
int permissive;
int warned_on_write;
+ struct pciback_config config;
};
/* Get/Put PCI Devices that are hidden from the PCI Backend Domain */
@@ -122,5 +129,12 @@
extern int verbose_request;
void test_and_schedule_op(struct pciback_device *pdev);
+
+extern int pciback_aer_reg_save(struct pci_dev *dev);
+extern int pciback_aer_reg_restore(struct pci_dev *dev);
+
+extern int pcistub_save_config_space(struct pci_dev *dev);
+extern int pcistub_restore_config_space(struct pci_dev *dev);
+
#endif
diff -r 366c31f3ab4b drivers/xen/pciback/pciback_ops.c
--- a/drivers/xen/pciback/pciback_ops.c Tue Apr 14 11:17:47 2009 +0100
+++ b/drivers/xen/pciback/pciback_ops.c Tue Apr 21 16:48:25 2009 +0900
@@ -20,6 +20,9 @@
{
u16 cmd;
+ /* restore configuration space */
+ pcistub_restore_config_space(dev);
+
/* Disable devices (but not bridges) */
if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
pci_disable_device(dev);
diff -r 366c31f3ab4b drivers/xen/pciback/xenbus.c
--- a/drivers/xen/pciback/xenbus.c Tue Apr 14 11:17:47 2009 +0100
+++ b/drivers/xen/pciback/xenbus.c Tue Apr 21 16:48:25 2009 +0900
@@ -43,6 +43,30 @@
return pdev;
}
+static int pciback_reinit_device(struct pciback_device *pdev,
+ int domain, int bus, int slot, int func)
+{
+ int err = 0;
+ struct pci_dev *dev;
+
+ dev_dbg(&pdev->xdev->dev, "Reinitializing dom %x bus %x slot %x"
+ " func %x\n", domain, bus, slot, func);
+
+ dev = pciback_get_pci_dev(pdev, domain, bus, PCI_DEVFN(slot, func));
+ if (!dev) {
+ err = -EINVAL;
+ dev_dbg(&pdev->xdev->dev, "Couldn't locate PCI device "
+ "(%04x:%02x:%02x.%01x)! not owned by this domain\n",
+ domain, bus, slot, func);
+ goto out;
+ }
+
+ pciback_reset_device(dev);
+
+ out:
+ return err;
+}
+
static void pciback_disconnect(struct pciback_device *pdev)
{
spin_lock(&pdev->dev_lock);
@@ -394,6 +418,15 @@
if (err)
goto out;
+ err = pciback_reinit_device(pdev, domain, bus, slot,
+ func);
+ if (err) {
+ xenbus_dev_fatal(pdev->xdev, err,
+ "Error reinitialize pci device"
+ "configuration");
+ goto out;
+ }
+
/* Publish pci roots. */
err = pciback_publish_pci_roots(pdev,
pciback_publish_pci_root);
if (err) {
@@ -575,6 +608,14 @@
if (err)
goto out;
+ err = pciback_reinit_device(pdev, domain, bus, slot, func);
+ if (err) {
+ xenbus_dev_fatal(pdev->xdev, err,
+ "Error reinitialize pci device "
+ "configuration");
+ goto out;
+ }
+
/* Switch substate of this device. */
l = snprintf(state_str, sizeof(state_str), "state-%d", i);
if (unlikely(l >= (sizeof(state_str) - 1))) {
@@ -627,6 +668,10 @@
pciback_setup_backend(pdev);
break;
+ case XenbusStateReconfiguring:
+ pciback_reconfigure(pdev);
+ break;
+
default:
break;
}
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
|
Home