[SCSI] libsas: prevent domain rediscovery competing with ata error handling

libata error handling provides for a timeout for link recovery.  libsas
must not rescan for previously known devices in this interval otherwise
it may remove a device that is simply waiting for its link to recover.
Let libata-eh make the determination of when the link is stable and
prevent libsas (host workqueue) from taking action while this
determination is pending.

Using a mutex (ha->disco_mutex) to flush and disable revalidation while
eh is running requires any discovery action that may block on eh be
moved to its own context outside the lock.  Probing ATA devices
explicitly waits on ata-eh and the cache-flush-io issued during device
removal may also pend awaiting eh completion.  Essentially any rphy
add/remove activity needs to run outside the lock.

This adds two new cleanup states for sas_unregister_domain_devices()
'allocated-but-not-probed', and 'flagged-for-destruction'.  In the
'allocated-but-not-probed' state  dev->rphy points to a rphy that is
known to have not been through a sas_rphy_add() event.  At domain
teardown check if this device is still pending probe and cleanup
accordingly.  Similarly if a device has already been queued for removal
then sas_unregister_domain_devices has nothing to do.

Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: James Bottomley <JBottomley@Parallels.com>

diff --git a/drivers/scsi/libsas/sas_ata.c b/drivers/scsi/libsas/sas_ata.c
index 2fc5a3961ca69d8b4d1878d193f701fe1b903583..4b6365c6410fff6450e80223f910a2092f9ddf41 100644 (file)
--- a/drivers/scsi/libsas/sas_ata.c
+++ b/drivers/scsi/libsas/sas_ata.c
@@ -758,6 +758,35 @@ static int sas_discover_sata_pm(struct domain_device *dev)
        return -ENODEV;
 }
 
+void sas_probe_sata(struct work_struct *work)
+{
+       struct domain_device *dev, *n;
+       struct sas_discovery_event *ev =
+               container_of(work, struct sas_discovery_event, work);
+       struct asd_sas_port *port = ev->port;
+
+       clear_bit(DISCE_PROBE, &port->disc.pending);
+
+       list_for_each_entry_safe(dev, n, &port->disco_list, disco_list_node) {
+               int err;
+
+               spin_lock_irq(&port->dev_list_lock);
+               list_add_tail(&dev->dev_list_node, &port->dev_list);
+               spin_unlock_irq(&port->dev_list_lock);
+
+               err = sas_rphy_add(dev->rphy);
+
+               if (err) {
+                       SAS_DPRINTK("%s: for %s device %16llx returned %d\n",
+                                   __func__, dev->parent ? "exp-attached" :
+                                                           "direct-attached",
+                                   SAS_ADDR(dev->sas_addr), err);
+                       sas_unregister_dev(port, dev);
+               } else
+                       list_del_init(&dev->disco_list_node);
+       }
+}
+
 /**
  * sas_discover_sata -- discover an STP/SATA domain device
  * @dev: pointer to struct domain_device of interest
@@ -794,10 +823,15 @@ int sas_discover_sata(struct domain_device *dev)
                break;
        }
        sas_notify_lldd_dev_gone(dev);
-       if (!res) {
-               sas_notify_lldd_dev_found(dev);
-               res = sas_rphy_add(dev->rphy);
-       }
+
+       if (res)
+               return res;
+
+       res = sas_notify_lldd_dev_found(dev);
+       if (res)
+               return res;
+
+       sas_discover_event(dev->port, DISCE_PROBE);
 
        return res;
 }
@@ -805,6 +839,17 @@ int sas_discover_sata(struct domain_device *dev)
 void sas_ata_strategy_handler(struct Scsi_Host *shost)
 {
        struct scsi_device *sdev;
+       struct sas_ha_struct *sas_ha = SHOST_TO_SAS_HA(shost);
+
+       /* it's ok to defer revalidation events during ata eh, these
+        * disks are in one of three states:
+        * 1/ present for initial domain discovery, and these
+        *    resets will cause bcn flutters
+        * 2/ hot removed, we'll discover that after eh fails
+        * 3/ hot added after initial discovery, lost the race, and need
+        *    to catch the next train.
+        */
+       sas_disable_revalidation(sas_ha);
 
        shost_for_each_device(sdev, shost) {
                struct domain_device *ddev = sdev_to_domain_dev(sdev);
@@ -816,6 +861,8 @@ void sas_ata_strategy_handler(struct Scsi_Host *shost)
                ata_port_printk(ap, KERN_DEBUG, "sas eh calling libata port error handler");
                ata_scsi_port_error_handler(shost, ap);
        }
+
+       sas_enable_revalidation(sas_ha);
 }
 
 int sas_ata_timed_out(struct scsi_cmnd *cmd, struct sas_task *task,

diff --git a/drivers/scsi/libsas/sas_discover.c b/drivers/scsi/libsas/sas_discover.c
index 32e01176604689574ee5ab56571c71d011c56654..7e8fdcb202b7ee4aa05d5c3eb7e3fad7ed975086 100644 (file)
--- a/drivers/scsi/libsas/sas_discover.c
+++ b/drivers/scsi/libsas/sas_discover.c
@@ -148,9 +148,14 @@ static int sas_get_port_device(struct asd_sas_port *port)
        port->disc.max_level = 0;
 
        dev->rphy = rphy;
-       spin_lock_irq(&port->dev_list_lock);
-       list_add_tail(&dev->dev_list_node, &port->dev_list);
-       spin_unlock_irq(&port->dev_list_lock);
+
+       if (dev_is_sata(dev))
+               list_add_tail(&dev->disco_list_node, &port->disco_list);
+       else {
+               spin_lock_irq(&port->dev_list_lock);
+               list_add_tail(&dev->dev_list_node, &port->dev_list);
+               spin_unlock_irq(&port->dev_list_lock);
+       }
 
        return 0;
 }
@@ -255,14 +260,43 @@ static void sas_unregister_common_dev(struct asd_sas_port *port, struct domain_d
        sas_put_device(dev);
 }
 
-void sas_unregister_dev(struct asd_sas_port *port, struct domain_device *dev)
+static void sas_destruct_devices(struct work_struct *work)
 {
-       if (dev->rphy) {
+       struct domain_device *dev, *n;
+       struct sas_discovery_event *ev =
+               container_of(work, struct sas_discovery_event, work);
+       struct asd_sas_port *port = ev->port;
+
+       clear_bit(DISCE_DESTRUCT, &port->disc.pending);
+
+       list_for_each_entry_safe(dev, n, &port->destroy_list, disco_list_node) {
+               list_del_init(&dev->disco_list_node);
+
                sas_remove_children(&dev->rphy->dev);
                sas_rphy_delete(dev->rphy);
                dev->rphy = NULL;
+               sas_unregister_common_dev(port, dev);
+
+               sas_put_device(dev);
+       }
+}
+
+void sas_unregister_dev(struct asd_sas_port *port, struct domain_device *dev)
+{
+       if (!test_bit(SAS_DEV_DESTROY, &dev->state) &&
+           !list_empty(&dev->disco_list_node)) {
+               /* this rphy never saw sas_rphy_add */
+               list_del_init(&dev->disco_list_node);
+               sas_rphy_free(dev->rphy);
+               dev->rphy = NULL;
+               sas_unregister_common_dev(port, dev);
+       }
+
+       if (dev->rphy && !test_and_set_bit(SAS_DEV_DESTROY, &dev->state)) {
+               sas_rphy_unlink(dev->rphy);
+               list_move_tail(&dev->disco_list_node, &port->destroy_list);
+               sas_discover_event(dev->port, DISCE_DESTRUCT);
        }
-       sas_unregister_common_dev(port, dev);
 }
 
 void sas_unregister_domain_devices(struct asd_sas_port *port)
@@ -271,6 +305,8 @@ void sas_unregister_domain_devices(struct asd_sas_port *port)
 
        list_for_each_entry_safe_reverse(dev, n, &port->dev_list, dev_list_node)
                sas_unregister_dev(port, dev);
+       list_for_each_entry_safe(dev, n, &port->disco_list, disco_list_node)
+               sas_unregister_dev(port, dev);
 
        port->port->rphy = NULL;
 
@@ -335,6 +371,7 @@ static void sas_discover_domain(struct work_struct *work)
                sas_rphy_free(dev->rphy);
                dev->rphy = NULL;
 
+               list_del_init(&dev->disco_list_node);
                spin_lock_irq(&port->dev_list_lock);
                list_del_init(&dev->dev_list_node);
                spin_unlock_irq(&port->dev_list_lock);
@@ -353,16 +390,28 @@ static void sas_revalidate_domain(struct work_struct *work)
        struct sas_discovery_event *ev =
                container_of(work, struct sas_discovery_event, work);
        struct asd_sas_port *port = ev->port;
+       struct sas_ha_struct *ha = port->ha;
+
+       /* prevent revalidation from finding sata links in recovery */
+       mutex_lock(&ha->disco_mutex);
+       if (test_bit(SAS_HA_ATA_EH_ACTIVE, &ha->state)) {
+               SAS_DPRINTK("REVALIDATION DEFERRED on port %d, pid:%d\n",
+                           port->id, task_pid_nr(current));
+               goto out;
+       }
 
        clear_bit(DISCE_REVALIDATE_DOMAIN, &port->disc.pending);
 
        SAS_DPRINTK("REVALIDATING DOMAIN on port %d, pid:%d\n", port->id,
                    task_pid_nr(current));
+
        if (port->port_dev)
                res = sas_ex_revalidate_domain(port->port_dev);
 
        SAS_DPRINTK("done REVALIDATING DOMAIN on port %d, pid:%d, res 0x%x\n",
                    port->id, task_pid_nr(current), res);
+ out:
+       mutex_unlock(&ha->disco_mutex);
 }
 
 /* ---------- Events ---------- */
@@ -414,6 +463,8 @@ void sas_init_disc(struct sas_discovery *disc, struct asd_sas_port *port)
        static const work_func_t sas_event_fns[DISC_NUM_EVENTS] = {
                [DISCE_DISCOVER_DOMAIN] = sas_discover_domain,
                [DISCE_REVALIDATE_DOMAIN] = sas_revalidate_domain,
+               [DISCE_PROBE] = sas_probe_sata,
+               [DISCE_DESTRUCT] = sas_destruct_devices,
        };
 
        disc->pending = 0;

diff --git a/drivers/scsi/libsas/sas_event.c b/drivers/scsi/libsas/sas_event.c
index e5035aa4c2a6a3b9449596c676ffc439a2e0391b..933d757499b5c053eae9c235ecbedbc1e80166d0 100644 (file)
--- a/drivers/scsi/libsas/sas_event.c
+++ b/drivers/scsi/libsas/sas_event.c
@@ -81,6 +81,32 @@ int sas_drain_work(struct sas_ha_struct *ha)
 }
 EXPORT_SYMBOL_GPL(sas_drain_work);
 
+void sas_disable_revalidation(struct sas_ha_struct *ha)
+{
+       mutex_lock(&ha->disco_mutex);
+       set_bit(SAS_HA_ATA_EH_ACTIVE, &ha->state);
+       mutex_unlock(&ha->disco_mutex);
+}
+
+void sas_enable_revalidation(struct sas_ha_struct *ha)
+{
+       int i;
+
+       mutex_lock(&ha->disco_mutex);
+       clear_bit(SAS_HA_ATA_EH_ACTIVE, &ha->state);
+       for (i = 0; i < ha->num_phys; i++) {
+               struct asd_sas_port *port = ha->sas_port[i];
+               const int ev = DISCE_REVALIDATE_DOMAIN;
+               struct sas_discovery *d = &port->disc;
+
+               if (!test_and_clear_bit(ev, &d->pending))
+                       continue;
+
+               sas_queue_event(ev, &d->pending, &d->disc_work[ev].work, ha);
+       }
+       mutex_unlock(&ha->disco_mutex);
+}
+
 static void notify_ha_event(struct sas_ha_struct *sas_ha, enum ha_event event)
 {
        BUG_ON(event >= HA_NUM_EVENTS);

diff --git a/drivers/scsi/libsas/sas_expander.c b/drivers/scsi/libsas/sas_expander.c
index f33d0c9911c4ac8614c44c84ccf7a39032f375b0..e45b259dac4cb6b09324b7434db4d563f289791d 100644 (file)
--- a/drivers/scsi/libsas/sas_expander.c
+++ b/drivers/scsi/libsas/sas_expander.c
@@ -704,9 +704,7 @@ static struct domain_device *sas_ex_discover_end_dev(
 
                child->rphy = rphy;
 
-               spin_lock_irq(&parent->port->dev_list_lock);
-               list_add_tail(&child->dev_list_node, &parent->port->dev_list);
-               spin_unlock_irq(&parent->port->dev_list_lock);
+               list_add_tail(&child->disco_list_node, &parent->port->disco_list);
 
                res = sas_discover_sata(child);
                if (res) {
@@ -756,6 +754,7 @@ static struct domain_device *sas_ex_discover_end_dev(
        sas_rphy_free(child->rphy);
        child->rphy = NULL;
 
+       list_del(&child->disco_list_node);
        spin_lock_irq(&parent->port->dev_list_lock);
        list_del(&child->dev_list_node);
        spin_unlock_irq(&parent->port->dev_list_lock);

diff --git a/drivers/scsi/libsas/sas_init.c b/drivers/scsi/libsas/sas_init.c
index 572b943d7603c25d1759d0fdd6d38115a26be544..52cd11d766644b7e29aacfd1a219c4abd336424a 100644 (file)
--- a/drivers/scsi/libsas/sas_init.c
+++ b/drivers/scsi/libsas/sas_init.c
@@ -104,6 +104,7 @@ int sas_register_ha(struct sas_ha_struct *sas_ha)
 {
        int error = 0;
 
+       mutex_init(&sas_ha->disco_mutex);
        spin_lock_init(&sas_ha->phy_port_lock);
        sas_hash_addr(sas_ha->hashed_sas_addr, sas_ha->sas_addr);
 
@@ -168,6 +169,7 @@ int sas_unregister_ha(struct sas_ha_struct *sas_ha)
        sas_drain_work(sas_ha);
 
        sas_unregister_ports(sas_ha);
+       sas_drain_work(sas_ha);
 
        if (sas_ha->lldd_max_execute_num > 1) {
                sas_shutdown_queue(sas_ha);

diff --git a/drivers/scsi/libsas/sas_internal.h b/drivers/scsi/libsas/sas_internal.h
index 948ea64cc2eb2e11a3aa4633e7b2249473071533..ebe9b81ddef57659ff96e4046728570c2e248315 100644 (file)
--- a/drivers/scsi/libsas/sas_internal.h
+++ b/drivers/scsi/libsas/sas_internal.h
@@ -56,6 +56,8 @@ enum blk_eh_timer_return sas_scsi_timed_out(struct scsi_cmnd *);
 int  sas_init_queue(struct sas_ha_struct *sas_ha);
 int  sas_init_events(struct sas_ha_struct *sas_ha);
 void sas_shutdown_queue(struct sas_ha_struct *sas_ha);
+void sas_disable_revalidation(struct sas_ha_struct *ha);
+void sas_enable_revalidation(struct sas_ha_struct *ha);
 
 void sas_deform_port(struct asd_sas_phy *phy, int gone);
 
@@ -138,6 +140,7 @@ static inline struct domain_device *sas_alloc_device(void)
        if (dev) {
                INIT_LIST_HEAD(&dev->siblings);
                INIT_LIST_HEAD(&dev->dev_list_node);
+               INIT_LIST_HEAD(&dev->disco_list_node);
                kref_init(&dev->kref);
        }
        return dev;

diff --git a/drivers/scsi/libsas/sas_port.c b/drivers/scsi/libsas/sas_port.c
index d88e55f9732b44f5e3252fc9869403275d7c865e..2980bde4e34a6cb86602253e2b5e11f371d95669 100644 (file)
--- a/drivers/scsi/libsas/sas_port.c
+++ b/drivers/scsi/libsas/sas_port.c
@@ -277,6 +277,8 @@ static void sas_init_port(struct asd_sas_port *port,
        memset(port, 0, sizeof(*port));
        port->id = i;
        INIT_LIST_HEAD(&port->dev_list);
+       INIT_LIST_HEAD(&port->disco_list);
+       INIT_LIST_HEAD(&port->destroy_list);
        spin_lock_init(&port->phy_list_lock);
        INIT_LIST_HEAD(&port->phy_list);
        port->ha = sas_ha;

diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index 9d9330ae4213206e5c300e5bbeeb4c3835170b9e..9421bae8af1a6ce4401a39e58135a3342f560e8d 100644 (file)
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -1602,6 +1602,20 @@ sas_rphy_delete(struct sas_rphy *rphy)
 }
 EXPORT_SYMBOL(sas_rphy_delete);
 
+/**
+ * sas_rphy_unlink  -  unlink SAS remote PHY
+ * @rphy:      SAS remote phy to unlink from its parent port
+ *
+ * Removes port reference to an rphy
+ */
+void sas_rphy_unlink(struct sas_rphy *rphy)
+{
+       struct sas_port *parent = dev_to_sas_port(rphy->dev.parent);
+
+       parent->rphy = NULL;
+}
+EXPORT_SYMBOL(sas_rphy_unlink);
+
 /**
  * sas_rphy_remove  -  remove SAS remote PHY
  * @rphy:      SAS remote phy to remove
@@ -1612,7 +1626,6 @@ void
 sas_rphy_remove(struct sas_rphy *rphy)
 {
        struct device *dev = &rphy->dev;
-       struct sas_port *parent = dev_to_sas_port(dev->parent);
 
        switch (rphy->identify.device_type) {
        case SAS_END_DEVICE:
@@ -1626,10 +1639,9 @@ sas_rphy_remove(struct sas_rphy *rphy)
                break;
        }
 
+       sas_rphy_unlink(rphy);
        transport_remove_device(dev);
        device_del(dev);
-
-       parent->rphy = NULL;
 }
 EXPORT_SYMBOL(sas_rphy_remove);
 

diff --git a/include/scsi/libsas.h b/include/scsi/libsas.h
index d792b13cfcf5426c3e9c9b9b59305a343d404359..bd6e89ece2ab7edf42a7db0f66fd6b8e977a0c15 100644 (file)
--- a/include/scsi/libsas.h
+++ b/include/scsi/libsas.h
@@ -86,7 +86,9 @@ enum discover_event {
        DISCE_DISCOVER_DOMAIN   = 0U,
        DISCE_REVALIDATE_DOMAIN = 1,
        DISCE_PORT_GONE         = 2,
-       DISC_NUM_EVENTS         = 3,
+       DISCE_PROBE             = 3,
+       DISCE_DESTRUCT          = 4,
+       DISC_NUM_EVENTS         = 5,
 };
 
 /* ---------- Expander Devices ---------- */
@@ -175,6 +177,7 @@ struct sata_device {
 
 enum {
        SAS_DEV_GONE,
+       SAS_DEV_DESTROY,
 };
 
 struct domain_device {
@@ -191,6 +194,7 @@ struct domain_device {
         struct asd_sas_port *port;        /* shortcut to root of the tree */
 
         struct list_head dev_list_node;
+       struct list_head disco_list_node; /* awaiting probe or destruct */
 
         enum sas_protocol    iproto;
         enum sas_protocol    tproto;
@@ -226,7 +230,6 @@ struct sas_discovery {
        int    max_level;
 };
 
-
 /* The port struct is Class:RW, driver:RO */
 struct asd_sas_port {
 /* private: */
@@ -236,6 +239,8 @@ struct asd_sas_port {
        struct domain_device *port_dev;
        spinlock_t dev_list_lock;
        struct list_head dev_list;
+       struct list_head disco_list;
+       struct list_head destroy_list;
        enum   sas_linkrate linkrate;
 
        struct sas_phy *phy;
@@ -334,6 +339,7 @@ struct sas_ha_event {
 enum sas_ha_state {
        SAS_HA_REGISTERED,
        SAS_HA_DRAINING,
+       SAS_HA_ATA_EH_ACTIVE,
 };
 
 struct sas_ha_struct {
@@ -346,6 +352,8 @@ struct sas_ha_struct {
        unsigned long     state;
        spinlock_t        state_lock;
 
+       struct mutex disco_mutex;
+
        struct scsi_core core;
 
 /* public: */

diff --git a/include/scsi/sas_ata.h b/include/scsi/sas_ata.h
index 7d5013f8653d4baac394bb79ee63df8219e6f299..557fc9a8559bc7251a0a60976735bb40c9c650a3 100644 (file)
--- a/include/scsi/sas_ata.h
+++ b/include/scsi/sas_ata.h
@@ -45,6 +45,7 @@ int sas_ata_timed_out(struct scsi_cmnd *cmd, struct sas_task *task,
                      enum blk_eh_timer_return *rtn);
 int sas_ata_eh(struct Scsi_Host *shost, struct list_head *work_q,
               struct list_head *done_q);
+void sas_probe_sata(struct work_struct *work);
 
 #else
 
@@ -78,6 +79,10 @@ static inline int sas_ata_eh(struct Scsi_Host *shost, struct list_head *work_q,
        return 0;
 }
 
+static inline void sas_probe_sata(struct work_struct *work)
+{
+}
+
 #endif
 
 #endif /* _SAS_ATA_H_ */

diff --git a/include/scsi/scsi_transport_sas.h b/include/scsi/scsi_transport_sas.h
index ffeebc34a4f7b27ce3ab3cbd75d8f41d0f4edf54..6d14daac75899ab64b98398595f0c5f456aec815 100644 (file)
--- a/include/scsi/scsi_transport_sas.h
+++ b/include/scsi/scsi_transport_sas.h
@@ -194,6 +194,7 @@ void sas_rphy_free(struct sas_rphy *);
 extern int sas_rphy_add(struct sas_rphy *);
 extern void sas_rphy_remove(struct sas_rphy *);
 extern void sas_rphy_delete(struct sas_rphy *);
+extern void sas_rphy_unlink(struct sas_rphy *);
 extern int scsi_is_sas_rphy(const struct device *);
 
 struct sas_port *sas_port_alloc(struct device *, int);