net/smc: synchronize buffer usage with device

Usage of send buffer "sndbuf" is synced
(a) before filling sndbuf for cpu access
(b) after filling sndbuf for device access

Usage of receive buffer "RMB" is synced
(a) before reading RMB content for cpu access
(b) after reading RMB content for device access

Signed-off-by: Ursula Braun <ubraun@linux.vnet.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index 75518879b68a6b3d9dc5e1a6c5a6cbe7838824ea..8c6d24b2995dd5ced548275acce04d3a6d70f043 100644 (file)
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -474,6 +474,7 @@ static int smc_connect_rdma(struct smc_sock *smc)
                        }
                }
        }
+       smc_rmb_sync_sg_for_device(&smc->conn);
 
        rc = smc_clc_send_confirm(smc);
        if (rc)
@@ -832,6 +833,7 @@ static void smc_listen_work(struct work_struct *work)
                        }
                }
        }
+       smc_rmb_sync_sg_for_device(&new_smc->conn);
 
        rc = smc_clc_send_accept(new_smc, local_contact);
        if (rc)

diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index 447bd52da0e2454948ea07c1ccf7e2d656e092c5..1a16d51e2330bb43de8bdd67c18f8b03c1e3ee92 100644 (file)
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -615,6 +615,38 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_rmb)
        return 0;
 }
 
+void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn)
+{
+       struct smc_link_group *lgr = conn->lgr;
+
+       smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
+                              conn->sndbuf_desc, DMA_TO_DEVICE);
+}
+
+void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn)
+{
+       struct smc_link_group *lgr = conn->lgr;
+
+       smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
+                                 conn->sndbuf_desc, DMA_TO_DEVICE);
+}
+
+void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn)
+{
+       struct smc_link_group *lgr = conn->lgr;
+
+       smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
+                              conn->rmb_desc, DMA_FROM_DEVICE);
+}
+
+void smc_rmb_sync_sg_for_device(struct smc_connection *conn)
+{
+       struct smc_link_group *lgr = conn->lgr;
+
+       smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
+                                 conn->rmb_desc, DMA_FROM_DEVICE);
+}
+
 /* create the send and receive buffer for an SMC socket;
  * receive buffers are called RMBs;
  * (even though the SMC protocol allows more than one RMB-element per RMB,

diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h
index 1d713e8e067cbe9baa0032a4dc4fefab17a6e0f5..19c44bf4e39160deeaea80cb71f5ca781d022ab3 100644 (file)
--- a/net/smc/smc_core.h
+++ b/net/smc/smc_core.h
@@ -189,4 +189,8 @@ void smc_lgr_terminate(struct smc_link_group *lgr);
 int smc_buf_create(struct smc_sock *smc);
 int smc_rmb_rtoken_handling(struct smc_connection *conn,
                            struct smc_clc_msg_accept_confirm *clc);
+void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn);
+void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn);
+void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn);
+void smc_rmb_sync_sg_for_device(struct smc_connection *conn);
 #endif

diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c
index 021f061609f523bb455c83e3c75c6eb4467766d9..547e0e113b17b19dc93e2606987a3dc0ff4c9c47 100644 (file)
--- a/net/smc/smc_ib.c
+++ b/net/smc/smc_ib.c
@@ -13,6 +13,7 @@
 
 #include <linux/random.h>
 #include <linux/workqueue.h>
+#include <linux/scatterlist.h>
 #include <rdma/ib_verbs.h>
 
 #include "smc_pnet.h"
@@ -295,6 +296,46 @@ int smc_ib_get_memory_region(struct ib_pd *pd, int access_flags,
        return 0;
 }
 
+/* synchronize buffer usage for cpu access */
+void smc_ib_sync_sg_for_cpu(struct smc_ib_device *smcibdev,
+                           struct smc_buf_desc *buf_slot,
+                           enum dma_data_direction data_direction)
+{
+       struct scatterlist *sg;
+       unsigned int i;
+
+       /* for now there is just one DMA address */
+       for_each_sg(buf_slot->sgt[SMC_SINGLE_LINK].sgl, sg,
+                   buf_slot->sgt[SMC_SINGLE_LINK].nents, i) {
+               if (!sg_dma_len(sg))
+                       break;
+               ib_dma_sync_single_for_cpu(smcibdev->ibdev,
+                                          sg_dma_address(sg),
+                                          sg_dma_len(sg),
+                                          data_direction);
+       }
+}
+
+/* synchronize buffer usage for device access */
+void smc_ib_sync_sg_for_device(struct smc_ib_device *smcibdev,
+                              struct smc_buf_desc *buf_slot,
+                              enum dma_data_direction data_direction)
+{
+       struct scatterlist *sg;
+       unsigned int i;
+
+       /* for now there is just one DMA address */
+       for_each_sg(buf_slot->sgt[SMC_SINGLE_LINK].sgl, sg,
+                   buf_slot->sgt[SMC_SINGLE_LINK].nents, i) {
+               if (!sg_dma_len(sg))
+                       break;
+               ib_dma_sync_single_for_device(smcibdev->ibdev,
+                                             sg_dma_address(sg),
+                                             sg_dma_len(sg),
+                                             data_direction);
+       }
+}
+
 /* Map a new TX or RX buffer SG-table to DMA */
 int smc_ib_buf_map_sg(struct smc_ib_device *smcibdev,
                      struct smc_buf_desc *buf_slot,

diff --git a/net/smc/smc_ib.h b/net/smc/smc_ib.h
index 72acb19ffc67508569ca152e4c5012f51e72a57c..9b927a33d5e6236564aab11d7b1b004958c32f52 100644 (file)
--- a/net/smc/smc_ib.h
+++ b/net/smc/smc_ib.h
@@ -68,4 +68,10 @@ long smc_ib_setup_per_ibdev(struct smc_ib_device *smcibdev);
 int smc_ib_get_memory_region(struct ib_pd *pd, int access_flags,
                             struct smc_buf_desc *buf_slot);
 void smc_ib_put_memory_region(struct ib_mr *mr);
+void smc_ib_sync_sg_for_cpu(struct smc_ib_device *smcibdev,
+                           struct smc_buf_desc *buf_slot,
+                           enum dma_data_direction data_direction);
+void smc_ib_sync_sg_for_device(struct smc_ib_device *smcibdev,
+                              struct smc_buf_desc *buf_slot,
+                              enum dma_data_direction data_direction);
 #endif

diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c
index f0c8b089f770a229b7c805fe5b2d59da40a5c1e3..b17a333e9bb0733893d9e44d92fc7ddf2f75ffbf 100644 (file)
--- a/net/smc/smc_rx.c
+++ b/net/smc/smc_rx.c
@@ -170,6 +170,7 @@ copy:
                                  copylen, conn->rmbe_size - cons.count);
                chunk_len_sum = chunk_len;
                chunk_off = cons.count;
+               smc_rmb_sync_sg_for_cpu(conn);
                for (chunk = 0; chunk < 2; chunk++) {
                        if (!(flags & MSG_TRUNC)) {
                                rc = memcpy_to_msg(msg, rcvbuf_base + chunk_off,
@@ -177,6 +178,7 @@ copy:
                                if (rc) {
                                        if (!read_done)
                                                read_done = -EFAULT;
+                                       smc_rmb_sync_sg_for_device(conn);
                                        goto out;
                                }
                        }
@@ -190,6 +192,7 @@ copy:
                        chunk_len_sum += chunk_len;
                        chunk_off = 0; /* modulo offset in recv ring buffer */
                }
+               smc_rmb_sync_sg_for_device(conn);
 
                /* update cursors */
                if (!(flags & MSG_PEEK)) {

diff --git a/net/smc/smc_tx.c b/net/smc/smc_tx.c
index f4d58e2dd559433a11260cabb8d5b2cab7b71220..3c656beb8820cb16a28f379ee32f2f39a8ce6631 100644 (file)
--- a/net/smc/smc_tx.c
+++ b/net/smc/smc_tx.c
@@ -174,10 +174,12 @@ int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len)
                                  copylen, conn->sndbuf_size - tx_cnt_prep);
                chunk_len_sum = chunk_len;
                chunk_off = tx_cnt_prep;
+               smc_sndbuf_sync_sg_for_cpu(conn);
                for (chunk = 0; chunk < 2; chunk++) {
                        rc = memcpy_from_msg(sndbuf_base + chunk_off,
                                             msg, chunk_len);
                        if (rc) {
+                               smc_sndbuf_sync_sg_for_device(conn);
                                if (send_done)
                                        return send_done;
                                goto out_err;
@@ -192,6 +194,7 @@ int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len)
                        chunk_len_sum += chunk_len;
                        chunk_off = 0; /* modulo offset in send ring buffer */
                }
+               smc_sndbuf_sync_sg_for_device(conn);
                /* update cursors */
                smc_curs_add(conn->sndbuf_size, &prep, copylen);
                smc_curs_write(&conn->tx_curs_prep,