smc: introduce SMC as an IB-client

* create a list of SMC IB-devices

Signed-off-by: Ursula Braun <ubraun@linux.vnet.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/smc/Makefile b/net/smc/Makefile
index c285c868dd8228941da241a2c989d8f120680d17..56f2170e6f64b5d5641cd31b3692885263d1396f 100644 (file)
--- a/net/smc/Makefile
+++ b/net/smc/Makefile
@@ -1,2 +1,2 @@
 obj-$(CONFIG_SMC)      += smc.o
-smc-y := af_smc.o
+smc-y := af_smc.o smc_ib.o

diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index 7fd773fc6238918fbe1f14d5f0b17ab3f923e902..50492ee495ce2b52880fa27343219347a1c54302 100644 (file)
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -20,6 +20,7 @@
 #include <net/sock.h>
 
 #include "smc.h"
+#include "smc_ib.h"
 
 static void smc_set_keepalive(struct sock *sk, int val)
 {
@@ -597,8 +598,16 @@ static int __init smc_init(void)
                goto out_proto;
        }
 
+       rc = smc_ib_register_client();
+       if (rc) {
+               pr_err("%s: ib_register fails with %d\n", __func__, rc);
+               goto out_sock;
+       }
+
        return 0;
 
+out_sock:
+       sock_unregister(PF_SMC);
 out_proto:
        proto_unregister(&smc_proto);
 out:
@@ -607,6 +616,7 @@ out:
 
 static void __exit smc_exit(void)
 {
+       smc_ib_unregister_client();
        sock_unregister(PF_SMC);
        proto_unregister(&smc_proto);
 }

diff --git a/net/smc/smc.h b/net/smc/smc.h
index fcea7399e2eb9bcf914cbcf026708781e77760ca..e87d7986709941dbb35e365c7054e98c3eb28f11 100644 (file)
--- a/net/smc/smc.h
+++ b/net/smc/smc.h
@@ -16,6 +16,8 @@
 
 #define SMCPROTO_SMC           0       /* SMC protocol */
 
+#define SMC_MAX_PORTS          2       /* Max # of ports */
+
 enum smc_state {               /* possible states of an SMC socket */
        SMC_ACTIVE      = 1,
        SMC_INIT        = 2,
@@ -34,4 +36,8 @@ static inline struct smc_sock *smc_sk(const struct sock *sk)
        return (struct smc_sock *)sk;
 }
 
+#define SMC_SYSTEMID_LEN               8
+
+extern u8      local_systemid[SMC_SYSTEMID_LEN]; /* unique system identifier */
+
 #endif /* __SMC_H */

diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c
new file mode 100644 (file)
index 0000000..7222b7e
--- /dev/null
+++ b/net/smc/smc_ib.c
@@ -0,0 +1,143 @@
+/*
+ *  Shared Memory Communications over RDMA (SMC-R) and RoCE
+ *
+ *  IB infrastructure:
+ *  Establish SMC-R as an Infiniband Client to be notified about added and
+ *  removed IB devices of type RDMA.
+ *  Determine device and port characteristics for these IB devices.
+ *
+ *  Copyright IBM Corp. 2016
+ *
+ *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
+ */
+
+#include <linux/random.h>
+#include <rdma/ib_verbs.h>
+
+#include "smc_ib.h"
+#include "smc.h"
+
+struct smc_ib_devices smc_ib_devices = {       /* smc-registered ib devices */
+       .lock = __SPIN_LOCK_UNLOCKED(smc_ib_devices.lock),
+       .list = LIST_HEAD_INIT(smc_ib_devices.list),
+};
+
+#define SMC_LOCAL_SYSTEMID_RESET       "%%%%%%%"
+
+u8 local_systemid[SMC_SYSTEMID_LEN] = SMC_LOCAL_SYSTEMID_RESET;        /* unique system
+                                                                * identifier
+                                                                */
+
+static int smc_ib_fill_gid_and_mac(struct smc_ib_device *smcibdev, u8 ibport)
+{
+       struct net_device *ndev;
+       int rc;
+
+       rc = ib_query_gid(smcibdev->ibdev, ibport, 0,
+                         &smcibdev->gid[ibport - 1], NULL);
+       /* the SMC protocol requires specification of the roce MAC address;
+        * if net_device cannot be determined, it can be derived from gid 0
+        */
+       ndev = smcibdev->ibdev->get_netdev(smcibdev->ibdev, ibport);
+       if (ndev) {
+               memcpy(&smcibdev->mac, ndev->dev_addr, ETH_ALEN);
+       } else if (!rc) {
+               memcpy(&smcibdev->mac[ibport - 1][0],
+                      &smcibdev->gid[ibport - 1].raw[8], 3);
+               memcpy(&smcibdev->mac[ibport - 1][3],
+                      &smcibdev->gid[ibport - 1].raw[13], 3);
+               smcibdev->mac[ibport - 1][0] &= ~0x02;
+       }
+       return rc;
+}
+
+/* Create an identifier unique for this instance of SMC-R.
+ * The MAC-address of the first active registered IB device
+ * plus a random 2-byte number is used to create this identifier.
+ * This name is delivered to the peer during connection initialization.
+ */
+static inline void smc_ib_define_local_systemid(struct smc_ib_device *smcibdev,
+                                               u8 ibport)
+{
+       memcpy(&local_systemid[2], &smcibdev->mac[ibport - 1],
+              sizeof(smcibdev->mac[ibport - 1]));
+       get_random_bytes(&local_systemid[0], 2);
+}
+
+bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport)
+{
+       return smcibdev->pattr[ibport - 1].state == IB_PORT_ACTIVE;
+}
+
+int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport)
+{
+       int rc;
+
+       memset(&smcibdev->pattr[ibport - 1], 0,
+              sizeof(smcibdev->pattr[ibport - 1]));
+       rc = ib_query_port(smcibdev->ibdev, ibport,
+                          &smcibdev->pattr[ibport - 1]);
+       if (rc)
+               goto out;
+       rc = smc_ib_fill_gid_and_mac(smcibdev, ibport);
+       if (rc)
+               goto out;
+       if (!strncmp(local_systemid, SMC_LOCAL_SYSTEMID_RESET,
+                    sizeof(local_systemid)) &&
+           smc_ib_port_active(smcibdev, ibport))
+               /* create unique system identifier */
+               smc_ib_define_local_systemid(smcibdev, ibport);
+out:
+       return rc;
+}
+
+static struct ib_client smc_ib_client;
+
+/* callback function for ib_register_client() */
+static void smc_ib_add_dev(struct ib_device *ibdev)
+{
+       struct smc_ib_device *smcibdev;
+
+       if (ibdev->node_type != RDMA_NODE_IB_CA)
+               return;
+
+       smcibdev = kzalloc(sizeof(*smcibdev), GFP_KERNEL);
+       if (!smcibdev)
+               return;
+
+       smcibdev->ibdev = ibdev;
+
+       spin_lock(&smc_ib_devices.lock);
+       list_add_tail(&smcibdev->list, &smc_ib_devices.list);
+       spin_unlock(&smc_ib_devices.lock);
+       ib_set_client_data(ibdev, &smc_ib_client, smcibdev);
+}
+
+/* callback function for ib_register_client() */
+static void smc_ib_remove_dev(struct ib_device *ibdev, void *client_data)
+{
+       struct smc_ib_device *smcibdev;
+
+       smcibdev = ib_get_client_data(ibdev, &smc_ib_client);
+       ib_set_client_data(ibdev, &smc_ib_client, NULL);
+       spin_lock(&smc_ib_devices.lock);
+       list_del_init(&smcibdev->list); /* remove from smc_ib_devices */
+       spin_unlock(&smc_ib_devices.lock);
+       kfree(smcibdev);
+}
+
+static struct ib_client smc_ib_client = {
+       .name   = "smc_ib",
+       .add    = smc_ib_add_dev,
+       .remove = smc_ib_remove_dev,
+};
+
+int __init smc_ib_register_client(void)
+{
+       return ib_register_client(&smc_ib_client);
+}
+
+void smc_ib_unregister_client(void)
+{
+       ib_unregister_client(&smc_ib_client);
+}

diff --git a/net/smc/smc_ib.h b/net/smc/smc_ib.h
new file mode 100644 (file)
index 0000000..63613e7
--- /dev/null
+++ b/net/smc/smc_ib.h
@@ -0,0 +1,40 @@
+/*
+ * Shared Memory Communications over RDMA (SMC-R) and RoCE
+ *
+ *  Definitions for IB environment
+ *
+ *  Copyright IBM Corp. 2016
+ *
+ *  Author(s):  Ursula Braun <Ursula Braun@linux.vnet.ibm.com>
+ */
+
+#ifndef _SMC_IB_H
+#define _SMC_IB_H
+
+#include <rdma/ib_verbs.h>
+
+#define SMC_MAX_PORTS                  2       /* Max # of ports */
+#define SMC_GID_SIZE                   sizeof(union ib_gid)
+
+struct smc_ib_devices {                        /* list of smc ib devices definition */
+       struct list_head        list;
+       spinlock_t              lock;   /* protects list of smc ib devices */
+};
+
+extern struct smc_ib_devices   smc_ib_devices; /* list of smc ib devices */
+
+struct smc_ib_device {                         /* ib-device infos for smc */
+       struct list_head        list;
+       struct ib_device        *ibdev;
+       struct ib_port_attr     pattr[SMC_MAX_PORTS];   /* ib dev. port attrs */
+       char                    mac[SMC_MAX_PORTS][6]; /* mac address per port*/
+       union ib_gid            gid[SMC_MAX_PORTS]; /* gid per port */
+       u8                      initialized : 1; /* ib dev CQ, evthdl done */
+};
+
+int smc_ib_register_client(void) __init;
+void smc_ib_unregister_client(void);
+bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport);
+int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport);
+
+#endif