powerpc/eeh: Handle EEH error based on PE
authorGavin Shan <shangw@linux.vnet.ibm.com>
Fri, 7 Sep 2012 22:44:19 +0000 (22:44 +0000)
committerBenjamin Herrenschmidt <benh@kernel.crashing.org>
Sun, 9 Sep 2012 23:35:43 +0000 (09:35 +1000)
The patch reworks the current implementation so that the eeh errors
will be handled basing on PE instead of eeh device.

Signed-off-by: Gavin Shan <shangw@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
arch/powerpc/include/asm/eeh.h
arch/powerpc/include/asm/eeh_event.h
arch/powerpc/platforms/pseries/eeh_driver.c
arch/powerpc/platforms/pseries/eeh_event.c
arch/powerpc/platforms/pseries/eeh_pe.c

index 629fb27c093ec5216898bd3b9b94a09b61b5fea4..d25a693595f8223b5b064714bd8c5a1b73baa181 100644 (file)
@@ -174,6 +174,7 @@ int eeh_rmv_from_parent_pe(struct eeh_dev *edev);
 void *eeh_pe_dev_traverse(struct eeh_pe *root,
                eeh_traverse_func fn, void *flag);
 void eeh_pe_restore_bars(struct eeh_pe *pe);
+struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe);
 
 void * __devinit eeh_dev_init(struct device_node *dn, void *data);
 void __devinit eeh_dev_phb_init_dynamic(struct pci_controller *phb);
index dc722b58216c78dcb3a58182ac7b08114ef752ea..de67d830151be7f9cf3067a2a5768eeb48fed6af 100644 (file)
@@ -32,7 +32,7 @@ struct eeh_event {
 };
 
 int eeh_send_failure_event(struct eeh_pe *pe);
-struct eeh_dev *handle_eeh_events(struct eeh_event *);
+void eeh_handle_event(struct eeh_pe *pe);
 
 #endif /* __KERNEL__ */
 #endif /* ASM_POWERPC_EEH_EVENT_H */
index baf92cd9dfab624c1fa37af234eb9da6f45bacf5..343c807e608a51cfabe5988da0dd5554ef556d1d 100644 (file)
@@ -116,28 +116,35 @@ static void eeh_enable_irq(struct pci_dev *dev)
 
 /**
  * eeh_report_error - Report pci error to each device driver
- * @dev: PCI device
+ * @data: eeh device
  * @userdata: return value
  * 
  * Report an EEH error to each device driver, collect up and 
  * merge the device driver responses. Cumulative response 
  * passed back in "userdata".
  */
-static int eeh_report_error(struct pci_dev *dev, void *userdata)
+static void *eeh_report_error(void *data, void *userdata)
 {
+       struct eeh_dev *edev = (struct eeh_dev *)data;
+       struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
        enum pci_ers_result rc, *res = userdata;
        struct pci_driver *driver = dev->driver;
 
+       /* We might not have the associated PCI device,
+        * then we should continue for next one.
+        */
+       if (!dev) return NULL;
+
        dev->error_state = pci_channel_io_frozen;
 
        if (!driver)
-               return 0;
+               return NULL;
 
        eeh_disable_irq(dev);
 
        if (!driver->err_handler ||
            !driver->err_handler->error_detected)
-               return 0;
+               return NULL;
 
        rc = driver->err_handler->error_detected(dev, pci_channel_io_frozen);
 
@@ -145,27 +152,31 @@ static int eeh_report_error(struct pci_dev *dev, void *userdata)
        if (rc == PCI_ERS_RESULT_NEED_RESET) *res = rc;
        if (*res == PCI_ERS_RESULT_NONE) *res = rc;
 
-       return 0;
+       return NULL;
 }
 
 /**
  * eeh_report_mmio_enabled - Tell drivers that MMIO has been enabled
- * @dev: PCI device
+ * @data: eeh device
  * @userdata: return value
  *
  * Tells each device driver that IO ports, MMIO and config space I/O
  * are now enabled. Collects up and merges the device driver responses.
  * Cumulative response passed back in "userdata".
  */
-static int eeh_report_mmio_enabled(struct pci_dev *dev, void *userdata)
+static void *eeh_report_mmio_enabled(void *data, void *userdata)
 {
+       struct eeh_dev *edev = (struct eeh_dev *)data;
+       struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
        enum pci_ers_result rc, *res = userdata;
-       struct pci_driver *driver = dev->driver;
+       struct pci_driver *driver;
 
-       if (!driver ||
+       if (!dev) return NULL;
+
+       if (!(driver = dev->driver) ||
            !driver->err_handler ||
            !driver->err_handler->mmio_enabled)
-               return 0;
+               return NULL;
 
        rc = driver->err_handler->mmio_enabled(dev);
 
@@ -173,12 +184,12 @@ static int eeh_report_mmio_enabled(struct pci_dev *dev, void *userdata)
        if (rc == PCI_ERS_RESULT_NEED_RESET) *res = rc;
        if (*res == PCI_ERS_RESULT_NONE) *res = rc;
 
-       return 0;
+       return NULL;
 }
 
 /**
  * eeh_report_reset - Tell device that slot has been reset
- * @dev: PCI device
+ * @data: eeh device
  * @userdata: return value
  *
  * This routine must be called while EEH tries to reset particular
@@ -186,13 +197,15 @@ static int eeh_report_mmio_enabled(struct pci_dev *dev, void *userdata)
  * some actions, usually to save data the driver needs so that the
  * driver can work again while the device is recovered.
  */
-static int eeh_report_reset(struct pci_dev *dev, void *userdata)
+static void *eeh_report_reset(void *data, void *userdata)
 {
+       struct eeh_dev *edev = (struct eeh_dev *)data;
+       struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
        enum pci_ers_result rc, *res = userdata;
-       struct pci_driver *driver = dev->driver;
+       struct pci_driver *driver;
 
-       if (!driver)
-               return 0;
+       if (!dev || !(driver = dev->driver))
+               return NULL;
 
        dev->error_state = pci_channel_io_normal;
 
@@ -200,7 +213,7 @@ static int eeh_report_reset(struct pci_dev *dev, void *userdata)
 
        if (!driver->err_handler ||
            !driver->err_handler->slot_reset)
-               return 0;
+               return NULL;
 
        rc = driver->err_handler->slot_reset(dev);
        if ((*res == PCI_ERS_RESULT_NONE) ||
@@ -208,82 +221,89 @@ static int eeh_report_reset(struct pci_dev *dev, void *userdata)
        if (*res == PCI_ERS_RESULT_DISCONNECT &&
             rc == PCI_ERS_RESULT_NEED_RESET) *res = rc;
 
-       return 0;
+       return NULL;
 }
 
 /**
  * eeh_report_resume - Tell device to resume normal operations
- * @dev: PCI device
+ * @data: eeh device
  * @userdata: return value
  *
  * This routine must be called to notify the device driver that it
  * could resume so that the device driver can do some initialization
  * to make the recovered device work again.
  */
-static int eeh_report_resume(struct pci_dev *dev, void *userdata)
+static void *eeh_report_resume(void *data, void *userdata)
 {
-       struct pci_driver *driver = dev->driver;
+       struct eeh_dev *edev = (struct eeh_dev *)data;
+       struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
+       struct pci_driver *driver;
+
+       if (!dev) return NULL;
 
        dev->error_state = pci_channel_io_normal;
 
-       if (!driver)
-               return 0;
+       if (!(driver = dev->driver))
+               return NULL;
 
        eeh_enable_irq(dev);
 
        if (!driver->err_handler ||
            !driver->err_handler->resume)
-               return 0;
+               return NULL;
 
        driver->err_handler->resume(dev);
 
-       return 0;
+       return NULL;
 }
 
 /**
  * eeh_report_failure - Tell device driver that device is dead.
- * @dev: PCI device
+ * @data: eeh device
  * @userdata: return value
  *
  * This informs the device driver that the device is permanently
  * dead, and that no further recovery attempts will be made on it.
  */
-static int eeh_report_failure(struct pci_dev *dev, void *userdata)
+static void *eeh_report_failure(void *data, void *userdata)
 {
-       struct pci_driver *driver = dev->driver;
+       struct eeh_dev *edev = (struct eeh_dev *)data;
+       struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
+       struct pci_driver *driver;
+
+       if (!dev) return NULL;
 
        dev->error_state = pci_channel_io_perm_failure;
 
-       if (!driver)
-               return 0;
+       if (!(driver = dev->driver))
+               return NULL;
 
        eeh_disable_irq(dev);
 
        if (!driver->err_handler ||
            !driver->err_handler->error_detected)
-               return 0;
+               return NULL;
 
        driver->err_handler->error_detected(dev, pci_channel_io_perm_failure);
 
-       return 0;
+       return NULL;
 }
 
 /**
  * eeh_reset_device - Perform actual reset of a pci slot
- * @edev: PE associated EEH device
+ * @pe: EEH PE
  * @bus: PCI bus corresponding to the isolcated slot
  *
  * This routine must be called to do reset on the indicated PE.
  * During the reset, udev might be invoked because those affected
  * PCI devices will be removed and then added.
  */
-static int eeh_reset_device(struct eeh_dev *edev, struct pci_bus *bus)
+static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus)
 {
-       struct device_node *dn;
        int cnt, rc;
 
        /* pcibios will clear the counter; save the value */
-       cnt = edev->freeze_count;
+       cnt = pe->freeze_count;
 
        if (bus)
                pcibios_remove_pci_devices(bus);
@@ -292,25 +312,13 @@ static int eeh_reset_device(struct eeh_dev *edev, struct pci_bus *bus)
         * Reconfigure bridges and devices. Don't try to bring the system
         * up if the reset failed for some reason.
         */
-       rc = eeh_reset_pe(edev);
+       rc = eeh_reset_pe(pe);
        if (rc)
                return rc;
 
-       /* Walk over all functions on this device. */
-       dn = eeh_dev_to_of_node(edev);
-       if (!pcibios_find_pci_bus(dn) && of_node_to_eeh_dev(dn->parent))
-               dn = dn->parent->child;
-
-       while (dn) {
-               struct eeh_dev *pedev = of_node_to_eeh_dev(dn);
-
-               /* On Power4, always true because eeh_pe_config_addr=0 */
-               if (edev->pe_config_addr == pedev->pe_config_addr) {
-                       eeh_ops->configure_bridge(dn);
-                       eeh_restore_bars(pedev);
-               }
-               dn = dn->sibling;
-       }
+       /* Restore PE */
+       eeh_ops->configure_bridge(pe);
+       eeh_pe_restore_bars(pe);
 
        /* Give the system 5 seconds to finish running the user-space
         * hotplug shutdown scripts, e.g. ifdown for ethernet.  Yes, 
@@ -322,7 +330,7 @@ static int eeh_reset_device(struct eeh_dev *edev, struct pci_bus *bus)
                ssleep(5);
                pcibios_add_pci_devices(bus);
        }
-       edev->freeze_count = cnt;
+       pe->freeze_count = cnt;
 
        return 0;
 }
@@ -334,7 +342,7 @@ static int eeh_reset_device(struct eeh_dev *edev, struct pci_bus *bus)
 
 /**
  * eeh_handle_event - Reset a PCI device after hard lockup.
- * @event: EEH event
+ * @pe: EEH PE
  *
  * While PHB detects address or data parity errors on particular PCI
  * slot, the associated PE will be frozen. Besides, DMA's occurring
@@ -349,69 +357,24 @@ static int eeh_reset_device(struct eeh_dev *edev, struct pci_bus *bus)
  * drivers (which cause a second set of hotplug events to go out to
  * userspace).
  */
-struct eeh_dev *handle_eeh_events(struct eeh_event *event)
+void eeh_handle_event(struct eeh_pe *pe)
 {
-       struct device_node *frozen_dn;
-       struct eeh_dev *frozen_edev;
        struct pci_bus *frozen_bus;
        int rc = 0;
        enum pci_ers_result result = PCI_ERS_RESULT_NONE;
-       const char *location, *pci_str, *drv_str, *bus_pci_str, *bus_drv_str;
-
-       frozen_dn = eeh_find_device_pe(eeh_dev_to_of_node(event->edev));
-       if (!frozen_dn) {
-               location = of_get_property(eeh_dev_to_of_node(event->edev), "ibm,loc-code", NULL);
-               location = location ? location : "unknown";
-               printk(KERN_ERR "EEH: Error: Cannot find partition endpoint "
-                               "for location=%s pci addr=%s\n",
-                       location, eeh_pci_name(eeh_dev_to_pci_dev(event->edev)));
-               return NULL;
-       }
-
-       frozen_bus = pcibios_find_pci_bus(frozen_dn);
-       location = of_get_property(frozen_dn, "ibm,loc-code", NULL);
-       location = location ? location : "unknown";
-
-       /* There are two different styles for coming up with the PE.
-        * In the old style, it was the highest EEH-capable device
-        * which was always an EADS pci bridge.  In the new style,
-        * there might not be any EADS bridges, and even when there are,
-        * the firmware marks them as "EEH incapable". So another
-        * two-step is needed to find the pci bus..
-        */
-       if (!frozen_bus)
-               frozen_bus = pcibios_find_pci_bus(frozen_dn->parent);
 
+       frozen_bus = eeh_pe_bus_get(pe);
        if (!frozen_bus) {
-               printk(KERN_ERR "EEH: Cannot find PCI bus "
-                       "for location=%s dn=%s\n",
-                       location, frozen_dn->full_name);
-               return NULL;
+               pr_err("%s: Cannot find PCI bus for PHB#%d-PE#%x\n",
+                       __func__, pe->phb->global_number, pe->addr);
+               return;
        }
 
-       frozen_edev = of_node_to_eeh_dev(frozen_dn);
-       frozen_edev->freeze_count++;
-       pci_str = eeh_pci_name(eeh_dev_to_pci_dev(event->edev));
-       drv_str = eeh_pcid_name(eeh_dev_to_pci_dev(event->edev));
-
-       if (frozen_edev->freeze_count > EEH_MAX_ALLOWED_FREEZES)
+       pe->freeze_count++;
+       if (pe->freeze_count > EEH_MAX_ALLOWED_FREEZES)
                goto excess_failures;
-
-       printk(KERN_WARNING
-          "EEH: This PCI device has failed %d times in the last hour:\n",
-               frozen_edev->freeze_count);
-
-       if (frozen_edev->pdev) {
-               bus_pci_str = pci_name(frozen_edev->pdev);
-               bus_drv_str = eeh_pcid_name(frozen_edev->pdev);
-               printk(KERN_WARNING
-                       "EEH: Bus location=%s driver=%s pci addr=%s\n",
-                       location, bus_drv_str, bus_pci_str);
-       }
-
-       printk(KERN_WARNING
-               "EEH: Device location=%s driver=%s pci addr=%s\n",
-               location, drv_str, pci_str);
+       pr_warning("EEH: This PCI device has failed %d times in the last hour\n",
+               pe->freeze_count);
 
        /* Walk the various device drivers attached to this slot through
         * a reset sequence, giving each an opportunity to do what it needs
@@ -419,12 +382,12 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event)
         * status ... if any child can't handle the reset, then the entire
         * slot is dlpar removed and added.
         */
-       pci_walk_bus(frozen_bus, eeh_report_error, &result);
+       eeh_pe_dev_traverse(pe, eeh_report_error, &result);
 
        /* Get the current PCI slot state. This can take a long time,
         * sometimes over 3 seconds for certain systems.
         */
-       rc = eeh_ops->wait_state(eeh_dev_to_of_node(frozen_edev), MAX_WAIT_FOR_RECOVERY*1000);
+       rc = eeh_ops->wait_state(pe, MAX_WAIT_FOR_RECOVERY*1000);
        if (rc < 0 || rc == EEH_STATE_NOT_SUPPORT) {
                printk(KERN_WARNING "EEH: Permanent failure\n");
                goto hard_fail;
@@ -434,14 +397,14 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event)
         * don't post the error log until after all dev drivers
         * have been informed.
         */
-       eeh_slot_error_detail(frozen_edev, EEH_LOG_TEMP);
+       eeh_slot_error_detail(pe, EEH_LOG_TEMP);
 
        /* If all device drivers were EEH-unaware, then shut
         * down all of the device drivers, and hope they
         * go down willingly, without panicing the system.
         */
        if (result == PCI_ERS_RESULT_NONE) {
-               rc = eeh_reset_device(frozen_edev, frozen_bus);
+               rc = eeh_reset_device(pe, frozen_bus);
                if (rc) {
                        printk(KERN_WARNING "EEH: Unable to reset, rc=%d\n", rc);
                        goto hard_fail;
@@ -450,7 +413,7 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event)
 
        /* If all devices reported they can proceed, then re-enable MMIO */
        if (result == PCI_ERS_RESULT_CAN_RECOVER) {
-               rc = eeh_pci_enable(frozen_edev, EEH_OPT_THAW_MMIO);
+               rc = eeh_pci_enable(pe, EEH_OPT_THAW_MMIO);
 
                if (rc < 0)
                        goto hard_fail;
@@ -458,13 +421,13 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event)
                        result = PCI_ERS_RESULT_NEED_RESET;
                } else {
                        result = PCI_ERS_RESULT_NONE;
-                       pci_walk_bus(frozen_bus, eeh_report_mmio_enabled, &result);
+                       eeh_pe_dev_traverse(pe, eeh_report_mmio_enabled, &result);
                }
        }
 
        /* If all devices reported they can proceed, then re-enable DMA */
        if (result == PCI_ERS_RESULT_CAN_RECOVER) {
-               rc = eeh_pci_enable(frozen_edev, EEH_OPT_THAW_DMA);
+               rc = eeh_pci_enable(pe, EEH_OPT_THAW_DMA);
 
                if (rc < 0)
                        goto hard_fail;
@@ -482,13 +445,13 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event)
 
        /* If any device called out for a reset, then reset the slot */
        if (result == PCI_ERS_RESULT_NEED_RESET) {
-               rc = eeh_reset_device(frozen_edev, NULL);
+               rc = eeh_reset_device(pe, NULL);
                if (rc) {
                        printk(KERN_WARNING "EEH: Cannot reset, rc=%d\n", rc);
                        goto hard_fail;
                }
                result = PCI_ERS_RESULT_NONE;
-               pci_walk_bus(frozen_bus, eeh_report_reset, &result);
+               eeh_pe_dev_traverse(pe, eeh_report_reset, &result);
        }
 
        /* All devices should claim they have recovered by now. */
@@ -499,9 +462,9 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event)
        }
 
        /* Tell all device drivers that they can resume operations */
-       pci_walk_bus(frozen_bus, eeh_report_resume, NULL);
+       eeh_pe_dev_traverse(pe, eeh_report_resume, NULL);
 
-       return frozen_edev;
+       return;
        
 excess_failures:
        /*
@@ -509,30 +472,26 @@ excess_failures:
         * are due to poorly seated PCI cards. Only 10% or so are
         * due to actual, failed cards.
         */
-       printk(KERN_ERR
-          "EEH: PCI device at location=%s driver=%s pci addr=%s\n"
-               "has failed %d times in the last hour "
-               "and has been permanently disabled.\n"
-               "Please try reseating this device or replacing it.\n",
-               location, drv_str, pci_str, frozen_edev->freeze_count);
+       pr_err("EEH: PHB#%d-PE#%x has failed %d times in the\n"
+              "last hour and has been permanently disabled.\n"
+              "Please try reseating or replacing it.\n",
+               pe->phb->global_number, pe->addr,
+               pe->freeze_count);
        goto perm_error;
 
 hard_fail:
-       printk(KERN_ERR
-          "EEH: Unable to recover from failure of PCI device "
-          "at location=%s driver=%s pci addr=%s\n"
-          "Please try reseating this device or replacing it.\n",
-               location, drv_str, pci_str);
+       pr_err("EEH: Unable to recover from failure from PHB#%d-PE#%x.\n"
+              "Please try reseating or replacing it\n",
+               pe->phb->global_number, pe->addr);
 
 perm_error:
-       eeh_slot_error_detail(frozen_edev, EEH_LOG_PERM);
+       eeh_slot_error_detail(pe, EEH_LOG_PERM);
 
        /* Notify all devices that they're about to go down. */
-       pci_walk_bus(frozen_bus, eeh_report_failure, NULL);
+       eeh_pe_dev_traverse(pe, eeh_report_failure, NULL);
 
        /* Shut down the device drivers for good. */
-       pcibios_remove_pci_devices(frozen_bus);
-
-       return NULL;
+       if (frozen_bus)
+               pcibios_remove_pci_devices(frozen_bus);
 }
 
index ba7005a5e2feb3e810a9fe2588568bb4517407ac..51faaac8abe6c401636326deebd59f4d12457b28 100644 (file)
@@ -82,7 +82,7 @@ static int eeh_event_handler(void * dummy)
                pe->phb->global_number, pe->addr);
 
        set_current_state(TASK_INTERRUPTIBLE);  /* Don't add to load average */
-       handle_eeh_events(event);
+       eeh_handle_event(pe);
        eeh_pe_state_clear(pe, EEH_PE_RECOVERING);
 
        kfree(event);
index 2b7e85b804a406114288a5d2988eb218cb892300..904123c7657b796113c54d2436db82970bbae569 100644 (file)
@@ -561,3 +561,31 @@ void eeh_pe_restore_bars(struct eeh_pe *pe)
 {
        eeh_pe_dev_traverse(pe, eeh_restore_one_device_bars, NULL);
 }
+
+/**
+ * eeh_pe_bus_get - Retrieve PCI bus according to the given PE
+ * @pe: EEH PE
+ *
+ * Retrieve the PCI bus according to the given PE. Basically,
+ * there're 3 types of PEs: PHB/Bus/Device. For PHB PE, the
+ * primary PCI bus will be retrieved. The parent bus will be
+ * returned for BUS PE. However, we don't have associated PCI
+ * bus for DEVICE PE.
+ */
+struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe)
+{
+       struct pci_bus *bus = NULL;
+       struct eeh_dev *edev;
+       struct pci_dev *pdev;
+
+       if (pe->type == EEH_PE_PHB) {
+               bus = pe->phb->bus;
+       } else if (pe->type == EEH_PE_BUS) {
+               edev = list_first_entry(&pe->edevs, struct eeh_dev, list);
+               pdev = eeh_dev_to_pci_dev(edev);
+               if (pdev)
+                       bus = pdev->bus;
+       }
+
+       return bus;
+}