8 files changed, 839 insertions, 88 deletions

diff --git a/drivers/pci/pcie/aer/Kconfig b/drivers/pci/pcie/aer/Kconfig
index c3bde58..50e94e0 100644
--- a/drivers/pci/pcie/aer/Kconfig
+++ b/drivers/pci/pcie/aer/Kconfig
@@ -10,3 +10,18 @@ config PCIEAER
 	  This enables PCI Express Root Port Advanced Error Reporting
 	  (AER) driver support. Error reporting messages sent to Root
 	  Port will be handled by PCI Express AER driver.
+
+
+#
+# PCI Express ECRC
+#
+config PCIE_ECRC
+	bool "PCI Express ECRC settings control"
+	depends on PCIEAER
+	help
+	  Used to override firmware/bios settings for PCI Express ECRC
+	  (transaction layer end-to-end CRC checking).
+
+	  When in doubt, say N.
+
+source "drivers/pci/pcie/aer/Kconfig.debug"
diff --git a/drivers/pci/pcie/aer/Kconfig.debug b/drivers/pci/pcie/aer/Kconfig.debug
new file mode 100644
index 0000000..b8c925c
--- /dev/null
+++ b/drivers/pci/pcie/aer/Kconfig.debug
@@ -0,0 +1,18 @@
+#
+# PCI Express Root Port Device AER Debug Configuration
+#
+
+config PCIEAER_INJECT
+	tristate "PCIE AER error injector support"
+	depends on PCIEAER
+	default n
+	help
+	  This enables PCI Express Root Port Advanced Error Reporting
+	  (AER) software error injector.
+
+	  Debuging PCIE AER code is quite difficult because it is hard
+	  to trigger various real hardware errors. Software based
+	  error injection can fake almost all kinds of errors with the
+	  help of a user space helper tool aer-inject, which can be
+	  gotten from:
+	     http://www.kernel.org/pub/linux/utils/pci/aer-inject/
diff --git a/drivers/pci/pcie/aer/Makefile b/drivers/pci/pcie/aer/Makefile
index 8da3bd8..2cba675 100644
--- a/drivers/pci/pcie/aer/Makefile
+++ b/drivers/pci/pcie/aer/Makefile
@@ -4,6 +4,9 @@
 
 obj-$(CONFIG_PCIEAER) += aerdriver.o
 
+obj-$(CONFIG_PCIE_ECRC)	+= ecrc.o
+
 aerdriver-objs := aerdrv_errprint.o aerdrv_core.o aerdrv.o
 aerdriver-$(CONFIG_ACPI) += aerdrv_acpi.o
 
+obj-$(CONFIG_PCIEAER_INJECT) += aer_inject.o
diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
new file mode 100644
index 0000000..d92ae21
--- /dev/null
+++ b/drivers/pci/pcie/aer/aer_inject.c
@@ -0,0 +1,473 @@
+/*
+ * PCIE AER software error injection support.
+ *
+ * Debuging PCIE AER code is quite difficult because it is hard to
+ * trigger various real hardware errors. Software based error
+ * injection can fake almost all kinds of errors with the help of a
+ * user space helper tool aer-inject, which can be gotten from:
+ *   http://www.kernel.org/pub/linux/utils/pci/aer-inject/
+ *
+ * Copyright 2009 Intel Corporation.
+ *     Huang Ying <ying.huang@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/miscdevice.h>
+#include <linux/pci.h>
+#include <linux/fs.h>
+#include <asm/uaccess.h>
+#include "aerdrv.h"
+
+struct aer_error_inj
+{
+	u8 bus;
+	u8 dev;
+	u8 fn;
+	u32 uncor_status;
+	u32 cor_status;
+	u32 header_log0;
+	u32 header_log1;
+	u32 header_log2;
+	u32 header_log3;
+};
+
+struct aer_error
+{
+	struct list_head list;
+	unsigned int bus;
+	unsigned int devfn;
+	int pos_cap_err;
+
+	u32 uncor_status;
+	u32 cor_status;
+	u32 header_log0;
+	u32 header_log1;
+	u32 header_log2;
+	u32 header_log3;
+	u32 root_status;
+	u32 source_id;
+};
+
+struct pci_bus_ops
+{
+	struct list_head list;
+	struct pci_bus *bus;
+	struct pci_ops *ops;
+};
+
+static LIST_HEAD(einjected);
+
+static LIST_HEAD(pci_bus_ops_list);
+
+/* Protect einjected and pci_bus_ops_list */
+static DEFINE_SPINLOCK(inject_lock);
+
+static void aer_error_init(struct aer_error *err, unsigned int bus,
+			   unsigned int devfn, int pos_cap_err)
+{
+	INIT_LIST_HEAD(&err->list);
+	err->bus = bus;
+	err->devfn = devfn;
+	err->pos_cap_err = pos_cap_err;
+}
+
+/* inject_lock must be held before calling */
+static struct aer_error *__find_aer_error(unsigned int bus, unsigned int devfn)
+{
+	struct aer_error *err;
+
+	list_for_each_entry(err, &einjected, list) {
+		if (bus == err->bus && devfn == err->devfn)
+			return err;
+	}
+	return NULL;
+}
+
+/* inject_lock must be held before calling */
+static struct aer_error *__find_aer_error_by_dev(struct pci_dev *dev)
+{
+	return __find_aer_error(dev->bus->number, dev->devfn);
+}
+
+/* inject_lock must be held before calling */
+static struct pci_ops *__find_pci_bus_ops(struct pci_bus *bus)
+{
+	struct pci_bus_ops *bus_ops;
+
+	list_for_each_entry(bus_ops, &pci_bus_ops_list, list) {
+		if (bus_ops->bus == bus)
+			return bus_ops->ops;
+	}
+	return NULL;
+}
+
+static struct pci_bus_ops *pci_bus_ops_pop(void)
+{
+	unsigned long flags;
+	struct pci_bus_ops *bus_ops = NULL;
+
+	spin_lock_irqsave(&inject_lock, flags);
+	if (list_empty(&pci_bus_ops_list))
+		bus_ops = NULL;
+	else {
+		struct list_head *lh = pci_bus_ops_list.next;
+		list_del(lh);
+		bus_ops = list_entry(lh, struct pci_bus_ops, list);
+	}
+	spin_unlock_irqrestore(&inject_lock, flags);
+	return bus_ops;
+}
+
+static u32 *find_pci_config_dword(struct aer_error *err, int where,
+				  int *prw1cs)
+{
+	int rw1cs = 0;
+	u32 *target = NULL;
+
+	if (err->pos_cap_err == -1)
+		return NULL;
+
+	switch (where - err->pos_cap_err) {
+	case PCI_ERR_UNCOR_STATUS:
+		target = &err->uncor_status;
+		rw1cs = 1;
+		break;
+	case PCI_ERR_COR_STATUS:
+		target = &err->cor_status;
+		rw1cs = 1;
+		break;
+	case PCI_ERR_HEADER_LOG:
+		target = &err->header_log0;
+		break;
+	case PCI_ERR_HEADER_LOG+4:
+		target = &err->header_log1;
+		break;
+	case PCI_ERR_HEADER_LOG+8:
+	        target = &err->header_log2;
+		break;
+	case PCI_ERR_HEADER_LOG+12:
+		target = &err->header_log3;
+		break;
+	case PCI_ERR_ROOT_STATUS:
+		target = &err->root_status;
+		rw1cs = 1;
+		break;
+	case PCI_ERR_ROOT_COR_SRC:
+		target = &err->source_id;
+		break;
+	}
+	if (prw1cs)
+		*prw1cs = rw1cs;
+	return target;
+}
+
+static int pci_read_aer(struct pci_bus *bus, unsigned int devfn, int where,
+			int size, u32 *val)
+{
+	u32 *sim;
+	struct aer_error *err;
+	unsigned long flags;
+	struct pci_ops *ops;
+
+	spin_lock_irqsave(&inject_lock, flags);
+	if (size != sizeof(u32))
+		goto out;
+	err = __find_aer_error(bus->number, devfn);
+	if (!err)
+		goto out;
+
+	sim = find_pci_config_dword(err, where, NULL);
+	if (sim) {
+		*val = *sim;
+		spin_unlock_irqrestore(&inject_lock, flags);
+		return 0;
+	}
+out:
+	ops = __find_pci_bus_ops(bus);
+	spin_unlock_irqrestore(&inject_lock, flags);
+	return ops->read(bus, devfn, where, size, val);
+}
+
+int pci_write_aer(struct pci_bus *bus, unsigned int devfn, int where, int size,
+		  u32 val)
+{
+	u32 *sim;
+	struct aer_error *err;
+	unsigned long flags;
+	int rw1cs;
+	struct pci_ops *ops;
+
+	spin_lock_irqsave(&inject_lock, flags);
+	if (size != sizeof(u32))
+		goto out;
+	err = __find_aer_error(bus->number, devfn);
+	if (!err)
+		goto out;
+
+	sim = find_pci_config_dword(err, where, &rw1cs);
+	if (sim) {
+		if (rw1cs)
+			*sim ^= val;
+		else
+			*sim = val;
+		spin_unlock_irqrestore(&inject_lock, flags);
+		return 0;
+	}
+out:
+	ops = __find_pci_bus_ops(bus);
+	spin_unlock_irqrestore(&inject_lock, flags);
+	return ops->write(bus, devfn, where, size, val);
+}
+
+static struct pci_ops pci_ops_aer = {
+	.read = pci_read_aer,
+	.write = pci_write_aer,
+};
+
+static void pci_bus_ops_init(struct pci_bus_ops *bus_ops,
+			     struct pci_bus *bus,
+			     struct pci_ops *ops)
+{
+	INIT_LIST_HEAD(&bus_ops->list);
+	bus_ops->bus = bus;
+	bus_ops->ops = ops;
+}
+
+static int pci_bus_set_aer_ops(struct pci_bus *bus)
+{
+	struct pci_ops *ops;
+	struct pci_bus_ops *bus_ops;
+	unsigned long flags;
+
+	bus_ops = kmalloc(sizeof(*bus_ops), GFP_KERNEL);
+	if (!bus_ops)
+		return -ENOMEM;
+	ops = pci_bus_set_ops(bus, &pci_ops_aer);
+	spin_lock_irqsave(&inject_lock, flags);
+	if (ops == &pci_ops_aer)
+		goto out;
+	pci_bus_ops_init(bus_ops, bus, ops);
+	list_add(&bus_ops->list, &pci_bus_ops_list);
+	bus_ops = NULL;
+out:
+	spin_unlock_irqrestore(&inject_lock, flags);
+	if (bus_ops)
+		kfree(bus_ops);
+	return 0;
+}
+
+static struct pci_dev *pcie_find_root_port(struct pci_dev *dev)
+{
+	while (1) {
+		if (!dev->is_pcie)
+			break;
+		if (dev->pcie_type == PCI_EXP_TYPE_ROOT_PORT)
+			return dev;
+		if (!dev->bus->self)
+			break;
+		dev = dev->bus->self;
+	}
+	return NULL;
+}
+
+static int find_aer_device_iter(struct device *device, void *data)
+{
+	struct pcie_device **result = data;
+	struct pcie_device *pcie_dev;
+
+	if (device->bus == &pcie_port_bus_type) {
+		pcie_dev = to_pcie_device(device);
+		if (pcie_dev->service & PCIE_PORT_SERVICE_AER) {
+			*result = pcie_dev;
+			return 1;
+		}
+	}
+	return 0;
+}
+
+static int find_aer_device(struct pci_dev *dev, struct pcie_device **result)
+{
+	return device_for_each_child(&dev->dev, result, find_aer_device_iter);
+}
+
+static int aer_inject(struct aer_error_inj *einj)
+{
+	struct aer_error *err, *rperr;
+	struct aer_error *err_alloc = NULL, *rperr_alloc = NULL;
+	struct pci_dev *dev, *rpdev;
+	struct pcie_device *edev;
+	unsigned long flags;
+	unsigned int devfn = PCI_DEVFN(einj->dev, einj->fn);
+	int pos_cap_err, rp_pos_cap_err;
+	u32 sever;
+	int ret = 0;
+
+	dev = pci_get_bus_and_slot(einj->bus, devfn);
+	if (!dev)
+		return -EINVAL;
+	rpdev = pcie_find_root_port(dev);
+	if (!rpdev) {
+		ret = -EINVAL;
+		goto out_put;
+	}
+
+	pos_cap_err = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+	if (!pos_cap_err) {
+		ret = -EIO;
+		goto out_put;
+	}
+	pci_read_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_SEVER, &sever);
+
+	rp_pos_cap_err = pci_find_ext_capability(rpdev, PCI_EXT_CAP_ID_ERR);
+	if (!rp_pos_cap_err) {
+		ret = -EIO;
+		goto out_put;
+	}
+
+	err_alloc =  kzalloc(sizeof(struct aer_error), GFP_KERNEL);
+	if (!err_alloc) {
+		ret = -ENOMEM;
+		goto out_put;
+	}
+	rperr_alloc =  kzalloc(sizeof(struct aer_error), GFP_KERNEL);
+	if (!rperr_alloc) {
+		ret = -ENOMEM;
+		goto out_put;
+	}
+
+	spin_lock_irqsave(&inject_lock, flags);
+
+	err = __find_aer_error_by_dev(dev);
+	if (!err) {
+		err = err_alloc;
+		err_alloc = NULL;
+		aer_error_init(err, einj->bus, devfn, pos_cap_err);
+		list_add(&err->list, &einjected);
+	}
+	err->uncor_status |= einj->uncor_status;
+	err->cor_status |= einj->cor_status;
+	err->header_log0 = einj->header_log0;
+	err->header_log1 = einj->header_log1;
+	err->header_log2 = einj->header_log2;
+	err->header_log3 = einj->header_log3;
+
+	rperr = __find_aer_error_by_dev(rpdev);
+	if (!rperr) {
+		rperr = rperr_alloc;
+		rperr_alloc = NULL;
+		aer_error_init(rperr, rpdev->bus->number, rpdev->devfn,
+			       rp_pos_cap_err);
+		list_add(&rperr->list, &einjected);
+	}
+	if (einj->cor_status) {
+		if (rperr->root_status & PCI_ERR_ROOT_COR_RCV)
+			rperr->root_status |= PCI_ERR_ROOT_MULTI_COR_RCV;
+		else
+			rperr->root_status |= PCI_ERR_ROOT_COR_RCV;
+		rperr->source_id &= 0xffff0000;
+		rperr->source_id |= (einj->bus << 8) | devfn;
+	}
+	if (einj->uncor_status) {
+		if (rperr->root_status & PCI_ERR_ROOT_UNCOR_RCV)
+			rperr->root_status |= PCI_ERR_ROOT_MULTI_UNCOR_RCV;
+		if (sever & einj->uncor_status) {
+			rperr->root_status |= PCI_ERR_ROOT_FATAL_RCV;
+			if (!(rperr->root_status & PCI_ERR_ROOT_UNCOR_RCV))
+				rperr->root_status |= PCI_ERR_ROOT_FIRST_FATAL;
+		} else
+			rperr->root_status |= PCI_ERR_ROOT_NONFATAL_RCV;
+		rperr->root_status |= PCI_ERR_ROOT_UNCOR_RCV;
+		rperr->source_id &= 0x0000ffff;
+		rperr->source_id |= ((einj->bus << 8) | devfn) << 16;
+	}
+	spin_unlock_irqrestore(&inject_lock, flags);
+
+	ret = pci_bus_set_aer_ops(dev->bus);
+	if (ret)
+		goto out_put;
+	ret = pci_bus_set_aer_ops(rpdev->bus);
+	if (ret)
+		goto out_put;
+
+	if (find_aer_device(rpdev, &edev))
+		aer_irq(-1, edev);
+	else
+		ret = -EINVAL;
+out_put:
+	if (err_alloc)
+		kfree(err_alloc);
+	if (rperr_alloc)
+		kfree(rperr_alloc);
+	pci_dev_put(dev);
+	return ret;
+}
+
+static ssize_t aer_inject_write(struct file *filp, const char __user *ubuf,
+				size_t usize, loff_t *off)
+{
+	struct aer_error_inj einj;
+	int ret;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	if (usize != sizeof(struct aer_error_inj))
+		return -EINVAL;
+
+	if (copy_from_user(&einj, ubuf, usize))
+		return -EFAULT;
+
+	ret = aer_inject(&einj);
+	return ret ? ret : usize;
+}
+
+static const struct file_operations aer_inject_fops = {
+	.write = aer_inject_write,
+	.owner = THIS_MODULE,
+};
+
+static struct miscdevice aer_inject_device = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "aer_inject",
+	.fops = &aer_inject_fops,
+};
+
+static int __init aer_inject_init(void)
+{
+	return misc_register(&aer_inject_device);
+}
+
+static void __exit aer_inject_exit(void)
+{
+	struct aer_error *err, *err_next;
+	unsigned long flags;
+	struct pci_bus_ops *bus_ops;
+
+	misc_deregister(&aer_inject_device);
+
+	while ((bus_ops = pci_bus_ops_pop())) {
+		pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
+		kfree(bus_ops);
+	}
+
+	spin_lock_irqsave(&inject_lock, flags);
+	list_for_each_entry_safe(err, err_next,
+				 &pci_bus_ops_list, list) {
+		list_del(&err->list);
+		kfree(err);
+	}
+	spin_unlock_irqrestore(&inject_lock, flags);
+}
+
+module_init(aer_inject_init);
+module_exit(aer_inject_exit);
+
+MODULE_DESCRIPTION("PCIE AER software error injector");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pci/pcie/aer/aerdrv.c b/drivers/pci/pcie/aer/aerdrv.c
index 32ade5a..4770f13 100644
--- a/drivers/pci/pcie/aer/aerdrv.c
+++ b/drivers/pci/pcie/aer/aerdrv.c
@@ -77,7 +77,7 @@ void pci_no_aer(void)
  *
  * Invoked when Root Port detects AER messages.
  **/
-static irqreturn_t aer_irq(int irq, void *context)
+irqreturn_t aer_irq(int irq, void *context)
 {
 	unsigned int status, id;
 	struct pcie_device *pdev = (struct pcie_device *)context;
@@ -126,6 +126,7 @@ static irqreturn_t aer_irq(int irq, void *context)
 
 	return IRQ_HANDLED;
 }
+EXPORT_SYMBOL_GPL(aer_irq);
 
 /**
  * aer_alloc_rpc - allocate Root Port data structure
diff --git a/drivers/pci/pcie/aer/aerdrv.h b/drivers/pci/pcie/aer/aerdrv.h
index aa14482..bbd7428 100644
--- a/drivers/pci/pcie/aer/aerdrv.h
+++ b/drivers/pci/pcie/aer/aerdrv.h
@@ -11,6 +11,7 @@
 #include <linux/workqueue.h>
 #include <linux/pcieport_if.h>
 #include <linux/aer.h>
+#include <linux/interrupt.h>
 
 #define AER_NONFATAL			0
 #define AER_FATAL			1
@@ -56,7 +57,11 @@ struct header_log_regs {
 	unsigned int dw3;
 };
 
+#define AER_MAX_MULTI_ERR_DEVICES	5	/* Not likely to have more */
 struct aer_err_info {
+	struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES];
+	int error_dev_num;
+	u16 id;
 	int severity;			/* 0:NONFATAL | 1:FATAL | 2:COR */
 	int flags;
 	unsigned int status;		/* COR/UNCOR Error Status */
@@ -120,6 +125,7 @@ extern void aer_delete_rootport(struct aer_rpc *rpc);
 extern int aer_init(struct pcie_device *dev);
 extern void aer_isr(struct work_struct *work);
 extern void aer_print_error(struct pci_dev *dev, struct aer_err_info *info);
+extern irqreturn_t aer_irq(int irq, void *context);
 
 #ifdef CONFIG_ACPI
 extern int aer_osc_setup(struct pcie_device *pciedev);
diff --git a/drivers/pci/pcie/aer/aerdrv_core.c b/drivers/pci/pcie/aer/aerdrv_core.c
index 307452f..3d88727 100644
--- a/drivers/pci/pcie/aer/aerdrv_core.c
+++ b/drivers/pci/pcie/aer/aerdrv_core.c
@@ -26,7 +26,9 @@
 #include "aerdrv.h"
 
 static int forceload;
+static int nosourceid;
 module_param(forceload, bool, 0);
+module_param(nosourceid, bool, 0);
 
 int pci_enable_pcie_error_reporting(struct pci_dev *dev)
 {
@@ -109,19 +111,23 @@ int pci_cleanup_aer_correct_error_status(struct pci_dev *dev)
 #endif  /*  0  */
 
 
-static void set_device_error_reporting(struct pci_dev *dev, void *data)
+static int set_device_error_reporting(struct pci_dev *dev, void *data)
 {
 	bool enable = *((bool *)data);
 
-	if (dev->pcie_type != PCIE_RC_PORT &&
-	    dev->pcie_type != PCIE_SW_UPSTREAM_PORT &&
-	    dev->pcie_type != PCIE_SW_DOWNSTREAM_PORT)
-		return;
+	if (dev->pcie_type == PCIE_RC_PORT ||
+	    dev->pcie_type == PCIE_SW_UPSTREAM_PORT ||
+	    dev->pcie_type == PCIE_SW_DOWNSTREAM_PORT) {
+		if (enable)
+			pci_enable_pcie_error_reporting(dev);
+		else
+			pci_disable_pcie_error_reporting(dev);
+	}
 
 	if (enable)
-		pci_enable_pcie_error_reporting(dev);
-	else
-		pci_disable_pcie_error_reporting(dev);
+		pcie_set_ecrc_checking(dev);
+
+	return 0;
 }
 
 /**
@@ -139,73 +145,148 @@ static void set_downstream_devices_error_reporting(struct pci_dev *dev,
 	pci_walk_bus(dev->subordinate, set_device_error_reporting, &enable);
 }
 
-static int find_device_iter(struct device *device, void *data)
+static inline int compare_device_id(struct pci_dev *dev,
+			struct aer_err_info *e_info)
 {
-	struct pci_dev *dev;
-	u16 id = *(unsigned long *)data;
-	u8 secondary, subordinate, d_bus = id >> 8;
+	if (e_info->id == ((dev->bus->number << 8) | dev->devfn)) {
+		/*
+		 * Device ID match
+		 */
+		return 1;
+	}
 
-	if (device->bus == &pci_bus_type) {
-		dev = to_pci_dev(device);
-		if (id == ((dev->bus->number << 8) | dev->devfn)) {
-			/*
-			 * Device ID match
-			 */
-			*(unsigned long*)data = (unsigned long)device;
-			return 1;
-		}
+	return 0;
+}
+
+static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
+{
+	if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) {
+		e_info->dev[e_info->error_dev_num] = dev;
+		e_info->error_dev_num++;
+		return 1;
+	} else
+		return 0;
+}
+
+
+#define	PCI_BUS(x)	(((x) >> 8) & 0xff)
+
+static int find_device_iter(struct pci_dev *dev, void *data)
+{
+	int pos;
+	u32 status;
+	u32 mask;
+	u16 reg16;
+	int result;
+	struct aer_err_info *e_info = (struct aer_err_info *)data;
+
+	/*
+	 * When bus id is equal to 0, it might be a bad id
+	 * reported by root port.
+	 */
+	if (!nosourceid && (PCI_BUS(e_info->id) != 0)) {
+		result = compare_device_id(dev, e_info);
+		if (result)
+			add_error_device(e_info, dev);
 
 		/*
-		 * If device is P2P, check if it is an upstream?
+		 * If there is no multiple error, we stop
+		 * or continue based on the id comparing.
 		 */
-		if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) {
-			pci_read_config_byte(dev, PCI_SECONDARY_BUS,
-				&secondary);
-			pci_read_config_byte(dev, PCI_SUBORDINATE_BUS,
-				&subordinate);
-			if (d_bus >= secondary && d_bus <= subordinate) {
-				*(unsigned long*)data = (unsigned long)device;
-				return 1;
-			}
+		if (!(e_info->flags & AER_MULTI_ERROR_VALID_FLAG))
+			return result;
+
+		/*
+		 * If there are multiple errors and id does match,
+		 * We need continue to search other devices under
+		 * the root port. Return 0 means that.
+		 */
+		if (result)
+			return 0;
+	}
+
+	/*
+	 * When either
+	 *      1) nosourceid==y;
+	 *      2) bus id is equal to 0. Some ports might lose the bus
+	 *              id of error source id;
+	 *      3) There are multiple errors and prior id comparing fails;
+	 * We check AER status registers to find the initial reporter.
+	 */
+	if (atomic_read(&dev->enable_cnt) == 0)
+		return 0;
+	pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
+	if (!pos)
+		return 0;
+	/* Check if AER is enabled */
+	pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, &reg16);
+	if (!(reg16 & (
+		PCI_EXP_DEVCTL_CERE |
+		PCI_EXP_DEVCTL_NFERE |
+		PCI_EXP_DEVCTL_FERE |
+		PCI_EXP_DEVCTL_URRE)))
+		return 0;
+	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+	if (!pos)
+		return 0;
+
+	status = 0;
+	mask = 0;
+	if (e_info->severity == AER_CORRECTABLE) {
+		pci_read_config_dword(dev,
+				pos + PCI_ERR_COR_STATUS,
+				&status);
+		pci_read_config_dword(dev,
+				pos + PCI_ERR_COR_MASK,
+				&mask);
+		if (status & ERR_CORRECTABLE_ERROR_MASK & ~mask) {
+			add_error_device(e_info, dev);
+			goto added;
+		}
+	} else {
+		pci_read_config_dword(dev,
+				pos + PCI_ERR_UNCOR_STATUS,
+				&status);
+		pci_read_config_dword(dev,
+				pos + PCI_ERR_UNCOR_MASK,
+				&mask);
+		if (status & ERR_UNCORRECTABLE_ERROR_MASK & ~mask) {
+			add_error_device(e_info, dev);
+			goto added;
 		}
 	}
 
 	return 0;
+
+added:
+	if (e_info->flags & AER_MULTI_ERROR_VALID_FLAG)
+		return 0;
+	else
+		return 1;
 }
 
 /**
  * find_source_device - search through device hierarchy for source device
  * @parent: pointer to Root Port pci_dev data structure
- * @id: device ID of agent who sends an error message to this Root Port
+ * @err_info: including detailed error information such like id
  *
  * Invoked when error is detected at the Root Port.
  */
-static struct device* find_source_device(struct pci_dev *parent, u16 id)
+static void find_source_device(struct pci_dev *parent,
+		struct aer_err_info *e_info)
 {
 	struct pci_dev *dev = parent;
-	struct device *device;
-	unsigned long device_addr;
-	int status;
+	int result;
 
 	/* Is Root Port an agent that sends error message? */
-	if (id == ((dev->bus->number << 8) | dev->devfn))
-		return &dev->dev;
-
-	do {
-		device_addr = id;
- 		if ((status = device_for_each_child(&dev->dev,
-			&device_addr, find_device_iter))) {
-			device = (struct device*)device_addr;
-			dev = to_pci_dev(device);
-			if (id == ((dev->bus->number << 8) | dev->devfn))
-				return device;
-		}
- 	}while (status);
+	result = find_device_iter(dev, e_info);
+	if (result)
+		return;
 
-	return NULL;
+	pci_walk_bus(parent->subordinate, find_device_iter, e_info);
 }
 
-static void report_error_detected(struct pci_dev *dev, void *data)
+static int report_error_detected(struct pci_dev *dev, void *data)
 {
 	pci_ers_result_t vote;
 	struct pci_error_handlers *err_handler;
@@ -230,16 +311,16 @@ static void report_error_detected(struct pci_dev *dev, void *data)
 				   dev->driver ?
 				   "no AER-aware driver" : "no driver");
 		}
-		return;
+		return 0;
 	}
 
 	err_handler = dev->driver->err_handler;
 	vote = err_handler->error_detected(dev, result_data->state);
 	result_data->result = merge_result(result_data->result, vote);
-	return;
+	return 0;
 }
 
-static void report_mmio_enabled(struct pci_dev *dev, void *data)
+static int report_mmio_enabled(struct pci_dev *dev, void *data)
 {
 	pci_ers_result_t vote;
 	struct pci_error_handlers *err_handler;
@@ -249,15 +330,15 @@ static void report_mmio_enabled(struct pci_dev *dev, void *data)
 	if (!dev->driver ||
 		!dev->driver->err_handler ||
 		!dev->driver->err_handler->mmio_enabled)
-		return;
+		return 0;
 
 	err_handler = dev->driver->err_handler;
 	vote = err_handler->mmio_enabled(dev);
 	result_data->result = merge_result(result_data->result, vote);
-	return;
+	return 0;
 }
 
-static void report_slot_reset(struct pci_dev *dev, void *data)
+static int report_slot_reset(struct pci_dev *dev, void *data)
 {
 	pci_ers_result_t vote;
 	struct pci_error_handlers *err_handler;
@@ -267,15 +348,15 @@ static void report_slot_reset(struct pci_dev *dev, void *data)
 	if (!dev->driver ||
 		!dev->driver->err_handler ||
 		!dev->driver->err_handler->slot_reset)
-		return;
+		return 0;
 
 	err_handler = dev->driver->err_handler;
 	vote = err_handler->slot_reset(dev);
 	result_data->result = merge_result(result_data->result, vote);
-	return;
+	return 0;
 }
 
-static void report_resume(struct pci_dev *dev, void *data)
+static int report_resume(struct pci_dev *dev, void *data)
 {
 	struct pci_error_handlers *err_handler;
 
@@ -284,11 +365,11 @@ static void report_resume(struct pci_dev *dev, void *data)
 	if (!dev->driver ||
 		!dev->driver->err_handler ||
 		!dev->driver->err_handler->resume)
-		return;
+		return 0;
 
 	err_handler = dev->driver->err_handler;
 	err_handler->resume(dev);
-	return;
+	return 0;
 }
 
 /**
@@ -305,7 +386,7 @@ static void report_resume(struct pci_dev *dev, void *data)
 static pci_ers_result_t broadcast_error_message(struct pci_dev *dev,
 	enum pci_channel_state state,
 	char *error_mesg,
-	void (*cb)(struct pci_dev *, void *))
+	int (*cb)(struct pci_dev *, void *))
 {
 	struct aer_broadcast_data result_data;
 
@@ -497,12 +578,12 @@ static pci_ers_result_t do_recovery(struct pcie_device *aerdev,
  */
 static void handle_error_source(struct pcie_device * aerdev,
 	struct pci_dev *dev,
-	struct aer_err_info info)
+	struct aer_err_info *info)
 {
 	pci_ers_result_t status = 0;
 	int pos;
 
-	if (info.severity == AER_CORRECTABLE) {
+	if (info->severity == AER_CORRECTABLE) {
 		/*
 		 * Correctable error does not need software intevention.
 		 * No need to go through error recovery process.
@@ -510,9 +591,9 @@ static void handle_error_source(struct pcie_device * aerdev,
 		pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
 		if (pos)
 			pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS,
-					info.status);
+					info->status);
 	} else {
-		status = do_recovery(aerdev, dev, info.severity);
+		status = do_recovery(aerdev, dev, info->severity);
 		if (status == PCI_ERS_RESULT_RECOVERED) {
 			dev_printk(KERN_DEBUG, &dev->dev, "AER driver "
 				   "successfully recovered\n");
@@ -661,6 +742,28 @@ static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
 	return AER_SUCCESS;
 }
 
+static inline void aer_process_err_devices(struct pcie_device *p_device,
+			struct aer_err_info *e_info)
+{
+	int i;
+
+	if (!e_info->dev[0]) {
+		dev_printk(KERN_DEBUG, &p_device->port->dev,
+				"can't find device of ID%04x\n",
+				e_info->id);
+	}
+
+	for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
+		if (get_device_error_info(e_info->dev[i], e_info) ==
+				AER_SUCCESS) {
+			aer_print_error(e_info->dev[i], e_info);
+			handle_error_source(p_device,
+					e_info->dev[i],
+					e_info);
+		}
+	}
+}
+
 /**
  * aer_isr_one_error - consume an error detected by root port
  * @p_device: pointer to error root port service device
@@ -669,10 +772,16 @@ static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
 static void aer_isr_one_error(struct pcie_device *p_device,
 		struct aer_err_source *e_src)
 {
-	struct device *s_device;
-	struct aer_err_info e_info = {0, 0, 0,};
+	struct aer_err_info *e_info;
 	int i;
-	u16 id;
+
+	/* struct aer_err_info might be big, so we allocate it with slab */
+	e_info = kmalloc(sizeof(struct aer_err_info), GFP_KERNEL);
+	if (e_info == NULL) {
+		dev_printk(KERN_DEBUG, &p_device->port->dev,
+			"Can't allocate mem when processing AER errors\n");
+		return;
+	}
 
 	/*
 	 * There is a possibility that both correctable error and
@@ -684,31 +793,26 @@ static void aer_isr_one_error(struct pcie_device *p_device,
 		if (!(e_src->status & i))
 			continue;
 
+		memset(e_info, 0, sizeof(struct aer_err_info));
+
 		/* Init comprehensive error information */
 		if (i & PCI_ERR_ROOT_COR_RCV) {
-			id = ERR_COR_ID(e_src->id);
-			e_info.severity = AER_CORRECTABLE;
+			e_info->id = ERR_COR_ID(e_src->id);
+			e_info->severity = AER_CORRECTABLE;
 		} else {
-			id = ERR_UNCOR_ID(e_src->id);
-			e_info.severity = ((e_src->status >> 6) & 1);
+			e_info->id = ERR_UNCOR_ID(e_src->id);
+			e_info->severity = ((e_src->status >> 6) & 1);
 		}
 		if (e_src->status &
 			(PCI_ERR_ROOT_MULTI_COR_RCV |
 			 PCI_ERR_ROOT_MULTI_UNCOR_RCV))
-			e_info.flags |= AER_MULTI_ERROR_VALID_FLAG;
-		if (!(s_device = find_source_device(p_device->port, id))) {
-			printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
-				__func__, id);
-			continue;
-		}
-		if (get_device_error_info(to_pci_dev(s_device), &e_info) ==
-				AER_SUCCESS) {
-			aer_print_error(to_pci_dev(s_device), &e_info);
-			handle_error_source(p_device,
-				to_pci_dev(s_device),
-				e_info);
-		}
+			e_info->flags |= AER_MULTI_ERROR_VALID_FLAG;
+
+		find_source_device(p_device->port, e_info);
+		aer_process_err_devices(p_device, e_info);
 	}
+
+	kfree(e_info);
 }
 
 /**
diff --git a/drivers/pci/pcie/aer/ecrc.c b/drivers/pci/pcie/aer/ecrc.c
new file mode 100644
index 0000000..a928d8a
--- /dev/null
+++ b/drivers/pci/pcie/aer/ecrc.c
@@ -0,0 +1,131 @@
+/*
+ *    Enables/disables PCIe ECRC checking.
+ *
+ *    (C) Copyright 2009 Hewlett-Packard Development Company, L.P.
+ *    Andrew Patterson <andrew.patterson@hp.com>
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; version 2 of the License.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ *    General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+ *    02111-1307, USA.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <linux/errno.h>
+#include "../../pci.h"
+
+#define ECRC_POLICY_DEFAULT 0		/* ECRC set by BIOS */
+#define ECRC_POLICY_OFF     1		/* ECRC off for performance */
+#define ECRC_POLICY_ON      2		/* ECRC on for data integrity */
+
+static int ecrc_policy = ECRC_POLICY_DEFAULT;
+
+static const char *ecrc_policy_str[] = {
+	[ECRC_POLICY_DEFAULT] = "bios",
+	[ECRC_POLICY_OFF] = "off",
+	[ECRC_POLICY_ON] = "on"
+};
+
+/**
+ * enable_ercr_checking - enable PCIe ECRC checking for a device
+ * @dev: the PCI device
+ *
+ * Returns 0 on success, or negative on failure.
+ */
+static int enable_ecrc_checking(struct pci_dev *dev)
+{
+	int pos;
+	u32 reg32;
+
+	if (!dev->is_pcie)
+		return -ENODEV;
+
+	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+	if (!pos)
+		return -ENODEV;
+
+	pci_read_config_dword(dev, pos + PCI_ERR_CAP, &reg32);
+	if (reg32 & PCI_ERR_CAP_ECRC_GENC)
+		reg32 |= PCI_ERR_CAP_ECRC_GENE;
+	if (reg32 & PCI_ERR_CAP_ECRC_CHKC)
+		reg32 |= PCI_ERR_CAP_ECRC_CHKE;
+	pci_write_config_dword(dev, pos + PCI_ERR_CAP, reg32);
+
+	return 0;
+}
+
+/**
+ * disable_ercr_checking - disables PCIe ECRC checking for a device
+ * @dev: the PCI device
+ *
+ * Returns 0 on success, or negative on failure.
+ */
+static int disable_ecrc_checking(struct pci_dev *dev)
+{
+	int pos;
+	u32 reg32;
+
+	if (!dev->is_pcie)
+		return -ENODEV;
+
+	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+	if (!pos)
+		return -ENODEV;
+
+	pci_read_config_dword(dev, pos + PCI_ERR_CAP, &reg32);
+	reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);
+	pci_write_config_dword(dev, pos + PCI_ERR_CAP, reg32);
+
+	return 0;
+}
+
+/**
+ * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based on global policy
+ * @dev: the PCI device
+ */
+void pcie_set_ecrc_checking(struct pci_dev *dev)
+{
+	switch (ecrc_policy) {
+	case ECRC_POLICY_DEFAULT:
+		return;
+	case ECRC_POLICY_OFF:
+		disable_ecrc_checking(dev);
+		break;
+	case ECRC_POLICY_ON:
+		enable_ecrc_checking(dev);
+		break;
+	default:
+		return;
+	}
+}
+
+/**
+ * pcie_ecrc_get_policy - parse kernel command-line ecrc option
+ */
+void pcie_ecrc_get_policy(char *str)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ecrc_policy_str); i++)
+		if (!strncmp(str, ecrc_policy_str[i],
+			     strlen(ecrc_policy_str[i])))
+			break;
+	if (i >= ARRAY_SIZE(ecrc_policy_str))
+		return;
+
+	ecrc_policy = i;
+}