From ec8a142327f85ae9aa5ad784520e669a34839941 Mon Sep 17 00:00:00 2001 From: Dean Luick Date: Mon, 20 Mar 2017 17:24:39 -0700 Subject: [PATCH] IB/hfi1: Force logical link down If the logical link state does not read as down when the physical link state is offline, force it to down. Reviewed-by: Dennis Dalessandro Reviewed-by: Jakub Byczkowski Signed-off-by: Dean Luick Signed-off-by: Easwar Hariharan Signed-off-by: Dennis Dalessandro Signed-off-by: Doug Ledford --- drivers/infiniband/hw/hfi1/chip.c | 86 ++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index 121a4c920f1b..44322c645469 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -1045,6 +1045,7 @@ static void dc_start(struct hfi1_devdata *); static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, unsigned int *np); static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd); +static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms); /* * Error interrupt table entry. This is used as input to the interrupt @@ -8891,8 +8892,6 @@ int send_idle_sma(struct hfi1_devdata *dd, u64 message) */ static int do_quick_linkup(struct hfi1_devdata *dd) { - u64 reg; - unsigned long timeout; int ret; lcb_shutdown(dd, 0); @@ -8915,19 +8914,9 @@ static int do_quick_linkup(struct hfi1_devdata *dd) write_csr(dd, DC_LCB_CFG_RUN, 1ull << DC_LCB_CFG_RUN_EN_SHIFT); - /* watch LCB_STS_LINK_TRANSFER_ACTIVE */ - timeout = jiffies + msecs_to_jiffies(10); - while (1) { - reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE); - if (reg) - break; - if (time_after(jiffies, timeout)) { - dd_dev_err(dd, - "timeout waiting for LINK_TRANSFER_ACTIVE\n"); - return -ETIMEDOUT; - } - udelay(2); - } + ret = wait_link_transfer_active(dd, 10); + if (ret) + return ret; write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT); @@ -10082,6 +10071,64 @@ static void check_lni_states(struct hfi1_pportdata *ppd) decode_state_complete(ppd, last_remote_state, "received"); } +/* wait for wait_ms for LINK_TRANSFER_ACTIVE to go to 1 */ +static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms) +{ + u64 reg; + unsigned long timeout; + + /* watch LCB_STS_LINK_TRANSFER_ACTIVE */ + timeout = jiffies + msecs_to_jiffies(wait_ms); + while (1) { + reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE); + if (reg) + break; + if (time_after(jiffies, timeout)) { + dd_dev_err(dd, + "timeout waiting for LINK_TRANSFER_ACTIVE\n"); + return -ETIMEDOUT; + } + udelay(2); + } + return 0; +} + +/* called when the logical link state is not down as it should be */ +static void force_logical_link_state_down(struct hfi1_pportdata *ppd) +{ + struct hfi1_devdata *dd = ppd->dd; + + /* + * Bring link up in LCB loopback + */ + write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1); + write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, + DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK); + + write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0); + write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0); + write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110); + write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x2); + + write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); + (void)read_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET); + udelay(3); + write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 1); + write_csr(dd, DC_LCB_CFG_RUN, 1ull << DC_LCB_CFG_RUN_EN_SHIFT); + + wait_link_transfer_active(dd, 100); + + /* + * Bring the link down again. + */ + write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1); + write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 0); + write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 0); + + /* call again to adjust ppd->statusp, if needed */ + get_logical_state(ppd); +} + /* * Helper for set_link_state(). Do not call except from that routine. * Expects ppd->hls_mutex to be held. @@ -10135,15 +10182,18 @@ static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason) return ret; } - /* make sure the logical state is also down */ - wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000); - /* * Now in charge of LCB - must be after the physical state is * offline.quiet and before host_link_state is changed. */ set_host_lcb_access(dd); write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */ + + /* make sure the logical state is also down */ + ret = wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000); + if (ret) + force_logical_link_state_down(ppd); + ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */ if (ppd->port_type == PORT_TYPE_QSFP && -- 2.20.1