[PATCH] IB/ipath: fix shared receive queues for RC

Signed-off-by: Ralph Campbell <ralph.campbell@qlogic.com>
Signed-off-by: Bryan O'Sullivan <bryan.osullivan@qlogic.com>
Cc: "Michael S. Tsirkin" <mst@mellanox.co.il>
Cc: Roland Dreier <rolandd@cisco.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/drivers/infiniband/hw/ipath/ipath_rc.c b/drivers/infiniband/hw/ipath/ipath_rc.c
index 720cb3ae1fc3323bd216ef37dde722f3829d3aee..bd2c405c4bf09ea5c878eba4aedff70c4330613a 100644 (file)
--- a/drivers/infiniband/hw/ipath/ipath_rc.c
+++ b/drivers/infiniband/hw/ipath/ipath_rc.c
@@ -257,7 +257,7 @@ int ipath_make_rc_req(struct ipath_qp *qp,
                        break;
 
                case IB_WR_RDMA_WRITE:
-                       if (newreq)
+                       if (newreq && qp->s_lsn != (u32) -1)
                                qp->s_lsn++;
                        /* FALLTHROUGH */
                case IB_WR_RDMA_WRITE_WITH_IMM:
@@ -283,8 +283,7 @@ int ipath_make_rc_req(struct ipath_qp *qp,
                        else {
                                qp->s_state =
                                        OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
-                               /* Immediate data comes
-                                * after RETH */
+                               /* Immediate data comes after RETH */
                                ohdr->u.rc.imm_data = wqe->wr.imm_data;
                                hwords += 1;
                                if (wqe->wr.send_flags & IB_SEND_SOLICITED)
@@ -304,7 +303,8 @@ int ipath_make_rc_req(struct ipath_qp *qp,
                        qp->s_state = OP(RDMA_READ_REQUEST);
                        hwords += sizeof(ohdr->u.rc.reth) / 4;
                        if (newreq) {
-                               qp->s_lsn++;
+                               if (qp->s_lsn != (u32) -1)
+                                       qp->s_lsn++;
                                /*
                                 * Adjust s_next_psn to count the
                                 * expected number of responses.
@@ -335,7 +335,8 @@ int ipath_make_rc_req(struct ipath_qp *qp,
                                wqe->wr.wr.atomic.compare_add);
                        hwords += sizeof(struct ib_atomic_eth) / 4;
                        if (newreq) {
-                               qp->s_lsn++;
+                               if (qp->s_lsn != (u32) -1)
+                                       qp->s_lsn++;
                                wqe->lpsn = wqe->psn;
                        }
                        if (++qp->s_cur == qp->s_size)
@@ -552,6 +553,88 @@ static void send_rc_ack(struct ipath_qp *qp)
        }
 }
 
+/**
+ * reset_psn - reset the QP state to send starting from PSN
+ * @qp: the QP
+ * @psn: the packet sequence number to restart at
+ *
+ * This is called from ipath_rc_rcv() to process an incoming RC ACK
+ * for the given QP.
+ * Called at interrupt level with the QP s_lock held.
+ */
+static void reset_psn(struct ipath_qp *qp, u32 psn)
+{
+       u32 n = qp->s_last;
+       struct ipath_swqe *wqe = get_swqe_ptr(qp, n);
+       u32 opcode;
+
+       qp->s_cur = n;
+
+       /*
+        * If we are starting the request from the beginning,
+        * let the normal send code handle initialization.
+        */
+       if (ipath_cmp24(psn, wqe->psn) <= 0) {
+               qp->s_state = OP(SEND_LAST);
+               goto done;
+       }
+
+       /* Find the work request opcode corresponding to the given PSN. */
+       opcode = wqe->wr.opcode;
+       for (;;) {
+               int diff;
+
+               if (++n == qp->s_size)
+                       n = 0;
+               if (n == qp->s_tail)
+                       break;
+               wqe = get_swqe_ptr(qp, n);
+               diff = ipath_cmp24(psn, wqe->psn);
+               if (diff < 0)
+                       break;
+               qp->s_cur = n;
+               /*
+                * If we are starting the request from the beginning,
+                * let the normal send code handle initialization.
+                */
+               if (diff == 0) {
+                       qp->s_state = OP(SEND_LAST);
+                       goto done;
+               }
+               opcode = wqe->wr.opcode;
+       }
+
+       /*
+        * Set the state to restart in the middle of a request.
+        * Don't change the s_sge, s_cur_sge, or s_cur_size.
+        * See ipath_do_rc_send().
+        */
+       switch (opcode) {
+       case IB_WR_SEND:
+       case IB_WR_SEND_WITH_IMM:
+               qp->s_state = OP(RDMA_READ_RESPONSE_FIRST);
+               break;
+
+       case IB_WR_RDMA_WRITE:
+       case IB_WR_RDMA_WRITE_WITH_IMM:
+               qp->s_state = OP(RDMA_READ_RESPONSE_LAST);
+               break;
+
+       case IB_WR_RDMA_READ:
+               qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE);
+               break;
+
+       default:
+               /*
+                * This case shouldn't happen since its only
+                * one PSN per req.
+                */
+               qp->s_state = OP(SEND_LAST);
+       }
+done:
+       qp->s_psn = psn;
+}
+
 /**
  * ipath_restart_rc - back up requester to resend the last un-ACKed request
  * @qp: the QP to restart
@@ -564,7 +647,6 @@ void ipath_restart_rc(struct ipath_qp *qp, u32 psn, struct ib_wc *wc)
 {
        struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
        struct ipath_ibdev *dev;
-       u32 n;
 
        /*
         * If there are no requests pending, we are done.
@@ -606,62 +688,7 @@ void ipath_restart_rc(struct ipath_qp *qp, u32 psn, struct ib_wc *wc)
        else
                dev->n_rc_resends += (int)qp->s_psn - (int)psn;
 
-       /*
-        * If we are starting the request from the beginning, let the normal
-        * send code handle initialization.
-        */
-       qp->s_cur = qp->s_last;
-       if (ipath_cmp24(psn, wqe->psn) <= 0) {
-               qp->s_state = OP(SEND_LAST);
-               qp->s_psn = wqe->psn;
-       } else {
-               n = qp->s_cur;
-               for (;;) {
-                       if (++n == qp->s_size)
-                               n = 0;
-                       if (n == qp->s_tail) {
-                               if (ipath_cmp24(psn, qp->s_next_psn) >= 0) {
-                                       qp->s_cur = n;
-                                       wqe = get_swqe_ptr(qp, n);
-                               }
-                               break;
-                       }
-                       wqe = get_swqe_ptr(qp, n);
-                       if (ipath_cmp24(psn, wqe->psn) < 0)
-                               break;
-                       qp->s_cur = n;
-               }
-               qp->s_psn = psn;
-
-               /*
-                * Reset the state to restart in the middle of a request.
-                * Don't change the s_sge, s_cur_sge, or s_cur_size.
-                * See ipath_do_rc_send().
-                */
-               switch (wqe->wr.opcode) {
-               case IB_WR_SEND:
-               case IB_WR_SEND_WITH_IMM:
-                       qp->s_state = OP(RDMA_READ_RESPONSE_FIRST);
-                       break;
-
-               case IB_WR_RDMA_WRITE:
-               case IB_WR_RDMA_WRITE_WITH_IMM:
-                       qp->s_state = OP(RDMA_READ_RESPONSE_LAST);
-                       break;
-
-               case IB_WR_RDMA_READ:
-                       qp->s_state =
-                               OP(RDMA_READ_RESPONSE_MIDDLE);
-                       break;
-
-               default:
-                       /*
-                        * This case shouldn't happen since its only
-                        * one PSN per req.
-                        */
-                       qp->s_state = OP(SEND_LAST);
-               }
-       }
+       reset_psn(qp, psn);
 
 done:
        tasklet_hi_schedule(&qp->s_task);
@@ -670,75 +697,13 @@ bail:
        return;
 }
 
-/**
- * reset_psn - reset the QP state to send starting from PSN
- * @qp: the QP
- * @psn: the packet sequence number to restart at
- *
- * This is called from ipath_rc_rcv_resp() to process an incoming RC ACK
- * for the given QP.
- * Called at interrupt level with the QP s_lock held.
- */
-static void reset_psn(struct ipath_qp *qp, u32 psn)
-{
-       struct ipath_swqe *wqe;
-       u32 n;
-
-       n = qp->s_cur;
-       wqe = get_swqe_ptr(qp, n);
-       for (;;) {
-               if (++n == qp->s_size)
-                       n = 0;
-               if (n == qp->s_tail) {
-                       if (ipath_cmp24(psn, qp->s_next_psn) >= 0) {
-                               qp->s_cur = n;
-                               wqe = get_swqe_ptr(qp, n);
-                       }
-                       break;
-               }
-               wqe = get_swqe_ptr(qp, n);
-               if (ipath_cmp24(psn, wqe->psn) < 0)
-                       break;
-               qp->s_cur = n;
-       }
-       qp->s_psn = psn;
-
-       /*
-        * Set the state to restart in the middle of a
-        * request.  Don't change the s_sge, s_cur_sge, or
-        * s_cur_size.  See ipath_do_rc_send().
-        */
-       switch (wqe->wr.opcode) {
-       case IB_WR_SEND:
-       case IB_WR_SEND_WITH_IMM:
-               qp->s_state = OP(RDMA_READ_RESPONSE_FIRST);
-               break;
-
-       case IB_WR_RDMA_WRITE:
-       case IB_WR_RDMA_WRITE_WITH_IMM:
-               qp->s_state = OP(RDMA_READ_RESPONSE_LAST);
-               break;
-
-       case IB_WR_RDMA_READ:
-               qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE);
-               break;
-
-       default:
-               /*
-                * This case shouldn't happen since its only
-                * one PSN per req.
-                */
-               qp->s_state = OP(SEND_LAST);
-       }
-}
-
 /**
  * do_rc_ack - process an incoming RC ACK
  * @qp: the QP the ACK came in on
  * @psn: the packet sequence number of the ACK
  * @opcode: the opcode of the request that resulted in the ACK
  *
- * This is called from ipath_rc_rcv() to process an incoming RC ACK
+ * This is called from ipath_rc_rcv_resp() to process an incoming RC ACK
  * for the given QP.
  * Called at interrupt level with the QP s_lock held.
  * Returns 1 if OK, 0 if current operation should be aborted (NAK).
@@ -877,22 +842,12 @@ static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode)
                if (qp->s_last == qp->s_tail)
                        goto bail;
 
-               /* The last valid PSN seen is the previous request's. */
-               qp->s_last_psn = wqe->psn - 1;
+               /* The last valid PSN is the previous PSN. */
+               qp->s_last_psn = psn - 1;
 
                dev->n_rc_resends += (int)qp->s_psn - (int)psn;
 
-               /*
-                * If we are starting the request from the beginning, let
-                * the normal send code handle initialization.
-                */
-               qp->s_cur = qp->s_last;
-               wqe = get_swqe_ptr(qp, qp->s_cur);
-               if (ipath_cmp24(psn, wqe->psn) <= 0) {
-                       qp->s_state = OP(SEND_LAST);
-                       qp->s_psn = wqe->psn;
-               } else
-                       reset_psn(qp, psn);
+               reset_psn(qp, psn);
 
                qp->s_rnr_timeout =
                        ib_ipath_rnr_table[(aeth >> IPS_AETH_CREDIT_SHIFT) &
@@ -1070,9 +1025,10 @@ static inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev,
                                       &dev->pending[dev->pending_index]);
                spin_unlock(&dev->pending_lock);
                /*
-                * Update the RDMA receive state but do the copy w/o holding the
-                * locks and blocking interrupts.  XXX Yet another place that
-                * affects relaxed RDMA order since we don't want s_sge modified.
+                * Update the RDMA receive state but do the copy w/o
+                * holding the locks and blocking interrupts.
+                * XXX Yet another place that affects relaxed RDMA order
+                * since we don't want s_sge modified.
                 */
                qp->s_len -= pmtu;
                qp->s_last_psn = psn;
@@ -1119,9 +1075,12 @@ static inline void ipath_rc_rcv_resp(struct ipath_ibdev *dev,
                if (do_rc_ack(qp, aeth, psn, OP(RDMA_READ_RESPONSE_LAST))) {
                        /*
                         * Change the state so we contimue
-                        * processing new requests.
+                        * processing new requests and wake up the
+                        * tasklet if there are posted sends.
                         */
                        qp->s_state = OP(SEND_LAST);
+                       if (qp->s_tail != qp->s_head)
+                               tasklet_hi_schedule(&qp->s_task);
                }
                goto ack_done;
        }