nfs41: pull data server cache from file layout to generic pnfs
authorPeng Tao <tao.peng@primarydata.com>
Thu, 29 May 2014 13:06:57 +0000 (21:06 +0800)
committerTom Haynes <loghyr@primarydata.com>
Tue, 3 Feb 2015 19:06:32 +0000 (11:06 -0800)
Also pull nfs4_pnfs_ds_addr and nfs4_pnfs_ds to generic pnfs.

They can all be reused by flexfile layout as well.

Reviewed-by: Jeff Layton <jlayton@primarydata.com>
Signed-off-by: Peng Tao <tao.peng@primarydata.com>
Signed-off-by: Tom Haynes <Thomas.Haynes@primarydata.com>
fs/nfs/filelayout/filelayout.h
fs/nfs/filelayout/filelayoutdev.c
fs/nfs/pnfs.h
fs/nfs/pnfs_nfs.c

index a5ce9b4bf2f8e113fbf7550dc8458b03509e6820..f97eea627c4f90c3b856e67bc65d03d429b44bfc 100644 (file)
@@ -56,24 +56,6 @@ enum stripetype4 {
        STRIPE_DENSE = 2
 };
 
-/* Individual ip address */
-struct nfs4_pnfs_ds_addr {
-       struct sockaddr_storage da_addr;
-       size_t                  da_addrlen;
-       struct list_head        da_node;  /* nfs4_pnfs_dev_hlist dev_dslist */
-       char                    *da_remotestr;  /* human readable addr+port */
-};
-
-struct nfs4_pnfs_ds {
-       struct list_head        ds_node;  /* nfs4_pnfs_dev_hlist dev_dslist */
-       char                    *ds_remotestr;  /* comma sep list of addrs */
-       struct list_head        ds_addrs;
-       struct nfs_client       *ds_clp;
-       atomic_t                ds_count;
-       unsigned long           ds_state;
-#define NFS4DS_CONNECTING      0       /* ds is establishing connection */
-};
-
 struct nfs4_file_layout_dsaddr {
        struct nfs4_deviceid_node       id_node;
        u32                             stripe_count;
@@ -131,7 +113,6 @@ filelayout_test_devid_unavailable(struct nfs4_deviceid_node *node);
 extern struct nfs_fh *
 nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, u32 j);
 
-extern void print_ds(struct nfs4_pnfs_ds *ds);
 u32 nfs4_fl_calc_j_index(struct pnfs_layout_segment *lseg, loff_t offset);
 u32 nfs4_fl_calc_ds_index(struct pnfs_layout_segment *lseg, u32 j);
 struct nfs4_pnfs_ds *nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg,
index d21080aed9b200625fbfb32208d66bb3fed08ef5..fbfbb701159d7969fb3be9fb264b6cc8c4d58f9c 100644 (file)
 static unsigned int dataserver_timeo = NFS4_DEF_DS_TIMEO;
 static unsigned int dataserver_retrans = NFS4_DEF_DS_RETRANS;
 
-/*
- * Data server cache
- *
- * Data servers can be mapped to different device ids.
- * nfs4_pnfs_ds reference counting
- *   - set to 1 on allocation
- *   - incremented when a device id maps a data server already in the cache.
- *   - decremented when deviceid is removed from the cache.
- */
-static DEFINE_SPINLOCK(nfs4_ds_cache_lock);
-static LIST_HEAD(nfs4_data_server_cache);
-
-/* Debug routines */
-void
-print_ds(struct nfs4_pnfs_ds *ds)
-{
-       if (ds == NULL) {
-               printk("%s NULL device\n", __func__);
-               return;
-       }
-       printk("        ds %s\n"
-               "        ref count %d\n"
-               "        client %p\n"
-               "        cl_exchange_flags %x\n",
-               ds->ds_remotestr,
-               atomic_read(&ds->ds_count), ds->ds_clp,
-               ds->ds_clp ? ds->ds_clp->cl_exchange_flags : 0);
-}
-
-static bool
-same_sockaddr(struct sockaddr *addr1, struct sockaddr *addr2)
-{
-       struct sockaddr_in *a, *b;
-       struct sockaddr_in6 *a6, *b6;
-
-       if (addr1->sa_family != addr2->sa_family)
-               return false;
-
-       switch (addr1->sa_family) {
-       case AF_INET:
-               a = (struct sockaddr_in *)addr1;
-               b = (struct sockaddr_in *)addr2;
-
-               if (a->sin_addr.s_addr == b->sin_addr.s_addr &&
-                   a->sin_port == b->sin_port)
-                       return true;
-               break;
-
-       case AF_INET6:
-               a6 = (struct sockaddr_in6 *)addr1;
-               b6 = (struct sockaddr_in6 *)addr2;
-
-               /* LINKLOCAL addresses must have matching scope_id */
-               if (ipv6_addr_src_scope(&a6->sin6_addr) ==
-                   IPV6_ADDR_SCOPE_LINKLOCAL &&
-                   a6->sin6_scope_id != b6->sin6_scope_id)
-                       return false;
-
-               if (ipv6_addr_equal(&a6->sin6_addr, &b6->sin6_addr) &&
-                   a6->sin6_port == b6->sin6_port)
-                       return true;
-               break;
-
-       default:
-               dprintk("%s: unhandled address family: %u\n",
-                       __func__, addr1->sa_family);
-               return false;
-       }
-
-       return false;
-}
-
-static bool
-_same_data_server_addrs_locked(const struct list_head *dsaddrs1,
-                              const struct list_head *dsaddrs2)
-{
-       struct nfs4_pnfs_ds_addr *da1, *da2;
-
-       /* step through both lists, comparing as we go */
-       for (da1 = list_first_entry(dsaddrs1, typeof(*da1), da_node),
-            da2 = list_first_entry(dsaddrs2, typeof(*da2), da_node);
-            da1 != NULL && da2 != NULL;
-            da1 = list_entry(da1->da_node.next, typeof(*da1), da_node),
-            da2 = list_entry(da2->da_node.next, typeof(*da2), da_node)) {
-               if (!same_sockaddr((struct sockaddr *)&da1->da_addr,
-                                  (struct sockaddr *)&da2->da_addr))
-                       return false;
-       }
-       if (da1 == NULL && da2 == NULL)
-               return true;
-
-       return false;
-}
-
-/*
- * Lookup DS by addresses.  nfs4_ds_cache_lock is held
- */
-static struct nfs4_pnfs_ds *
-_data_server_lookup_locked(const struct list_head *dsaddrs)
-{
-       struct nfs4_pnfs_ds *ds;
-
-       list_for_each_entry(ds, &nfs4_data_server_cache, ds_node)
-               if (_same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs))
-                       return ds;
-       return NULL;
-}
-
 /*
  * Create an rpc connection to the nfs4_pnfs_ds data server
  * Currently only supports IPv4 and IPv6 addresses
@@ -195,30 +87,6 @@ out_put:
        goto out;
 }
 
-static void
-destroy_ds(struct nfs4_pnfs_ds *ds)
-{
-       struct nfs4_pnfs_ds_addr *da;
-
-       dprintk("--> %s\n", __func__);
-       ifdebug(FACILITY)
-               print_ds(ds);
-
-       nfs_put_client(ds->ds_clp);
-
-       while (!list_empty(&ds->ds_addrs)) {
-               da = list_first_entry(&ds->ds_addrs,
-                                     struct nfs4_pnfs_ds_addr,
-                                     da_node);
-               list_del_init(&da->da_node);
-               kfree(da->da_remotestr);
-               kfree(da);
-       }
-
-       kfree(ds->ds_remotestr);
-       kfree(ds);
-}
-
 void
 nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr)
 {
@@ -229,112 +97,13 @@ nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr)
 
        for (i = 0; i < dsaddr->ds_num; i++) {
                ds = dsaddr->ds_list[i];
-               if (ds != NULL) {
-                       if (atomic_dec_and_lock(&ds->ds_count,
-                                               &nfs4_ds_cache_lock)) {
-                               list_del_init(&ds->ds_node);
-                               spin_unlock(&nfs4_ds_cache_lock);
-                               destroy_ds(ds);
-                       }
-               }
+               if (ds != NULL)
+                       nfs4_pnfs_ds_put(ds);
        }
        kfree(dsaddr->stripe_indices);
        kfree(dsaddr);
 }
 
-/*
- * Create a string with a human readable address and port to avoid
- * complicated setup around many dprinks.
- */
-static char *
-nfs4_pnfs_remotestr(struct list_head *dsaddrs, gfp_t gfp_flags)
-{
-       struct nfs4_pnfs_ds_addr *da;
-       char *remotestr;
-       size_t len;
-       char *p;
-
-       len = 3;        /* '{', '}' and eol */
-       list_for_each_entry(da, dsaddrs, da_node) {
-               len += strlen(da->da_remotestr) + 1;    /* string plus comma */
-       }
-
-       remotestr = kzalloc(len, gfp_flags);
-       if (!remotestr)
-               return NULL;
-
-       p = remotestr;
-       *(p++) = '{';
-       len--;
-       list_for_each_entry(da, dsaddrs, da_node) {
-               size_t ll = strlen(da->da_remotestr);
-
-               if (ll > len)
-                       goto out_err;
-
-               memcpy(p, da->da_remotestr, ll);
-               p += ll;
-               len -= ll;
-
-               if (len < 1)
-                       goto out_err;
-               (*p++) = ',';
-               len--;
-       }
-       if (len < 2)
-               goto out_err;
-       *(p++) = '}';
-       *p = '\0';
-       return remotestr;
-out_err:
-       kfree(remotestr);
-       return NULL;
-}
-
-static struct nfs4_pnfs_ds *
-nfs4_pnfs_ds_add(struct list_head *dsaddrs, gfp_t gfp_flags)
-{
-       struct nfs4_pnfs_ds *tmp_ds, *ds = NULL;
-       char *remotestr;
-
-       if (list_empty(dsaddrs)) {
-               dprintk("%s: no addresses defined\n", __func__);
-               goto out;
-       }
-
-       ds = kzalloc(sizeof(*ds), gfp_flags);
-       if (!ds)
-               goto out;
-
-       /* this is only used for debugging, so it's ok if its NULL */
-       remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags);
-
-       spin_lock(&nfs4_ds_cache_lock);
-       tmp_ds = _data_server_lookup_locked(dsaddrs);
-       if (tmp_ds == NULL) {
-               INIT_LIST_HEAD(&ds->ds_addrs);
-               list_splice_init(dsaddrs, &ds->ds_addrs);
-               ds->ds_remotestr = remotestr;
-               atomic_set(&ds->ds_count, 1);
-               INIT_LIST_HEAD(&ds->ds_node);
-               ds->ds_clp = NULL;
-               list_add(&ds->ds_node, &nfs4_data_server_cache);
-               dprintk("%s add new data server %s\n", __func__,
-                       ds->ds_remotestr);
-       } else {
-               kfree(remotestr);
-               kfree(ds);
-               atomic_inc(&tmp_ds->ds_count);
-               dprintk("%s data server %s found, inc'ed ds_count to %d\n",
-                       __func__, tmp_ds->ds_remotestr,
-                       atomic_read(&tmp_ds->ds_count));
-               ds = tmp_ds;
-       }
-       spin_unlock(&nfs4_ds_cache_lock);
-out:
-       return ds;
-}
-
 /*
  * Currently only supports ipv4, ipv6 and one multi-path address.
  */
index e94f6050e9b1768865dc9fafb34835185056efcd..b0168f1dd0728d3c7bb5b2eabcfd0ba17bcc3640 100644 (file)
@@ -40,6 +40,24 @@ enum {
        NFS_LSEG_LAYOUTCOMMIT,  /* layoutcommit bit set for layoutcommit */
 };
 
+/* Individual ip address */
+struct nfs4_pnfs_ds_addr {
+       struct sockaddr_storage da_addr;
+       size_t                  da_addrlen;
+       struct list_head        da_node;  /* nfs4_pnfs_dev_hlist dev_dslist */
+       char                    *da_remotestr;  /* human readable addr+port */
+};
+
+struct nfs4_pnfs_ds {
+       struct list_head        ds_node;  /* nfs4_pnfs_dev_hlist dev_dslist */
+       char                    *ds_remotestr;  /* comma sep list of addrs */
+       struct list_head        ds_addrs;
+       struct nfs_client       *ds_clp;
+       atomic_t                ds_count;
+       unsigned long           ds_state;
+#define NFS4DS_CONNECTING      0       /* ds is establishing connection */
+};
+
 struct pnfs_layout_segment {
        struct list_head pls_list;
        struct list_head pls_lc_list;
@@ -291,6 +309,9 @@ int pnfs_generic_commit_pagelist(struct inode *inode,
                                                        int how));
 int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max);
 void pnfs_generic_write_commit_done(struct rpc_task *task, void *data);
+void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds);
+struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(struct list_head *dsaddrs,
+                                     gfp_t gfp_flags);
 
 static inline struct nfs4_deviceid_node *
 nfs4_get_deviceid(struct nfs4_deviceid_node *d)
index fd2a2f0e8cbbb40a07fa9367186a1418e20475b3..3bb2b74cf600414be7d70a3c8a345a669ee04dcc 100644 (file)
@@ -13,6 +13,8 @@
 #include "internal.h"
 #include "pnfs.h"
 
+#define NFSDBG_FACILITY                NFSDBG_PNFS
+
 static void pnfs_generic_fenceme(struct inode *inode,
                                 struct pnfs_layout_hdr *lo)
 {
@@ -290,3 +292,243 @@ out:
        return PNFS_ATTEMPTED;
 }
 EXPORT_SYMBOL_GPL(pnfs_generic_commit_pagelist);
+
+/*
+ * Data server cache
+ *
+ * Data servers can be mapped to different device ids.
+ * nfs4_pnfs_ds reference counting
+ *   - set to 1 on allocation
+ *   - incremented when a device id maps a data server already in the cache.
+ *   - decremented when deviceid is removed from the cache.
+ */
+static DEFINE_SPINLOCK(nfs4_ds_cache_lock);
+static LIST_HEAD(nfs4_data_server_cache);
+
+/* Debug routines */
+static void
+print_ds(struct nfs4_pnfs_ds *ds)
+{
+       if (ds == NULL) {
+               printk(KERN_WARNING "%s NULL device\n", __func__);
+               return;
+       }
+       printk(KERN_WARNING "        ds %s\n"
+               "        ref count %d\n"
+               "        client %p\n"
+               "        cl_exchange_flags %x\n",
+               ds->ds_remotestr,
+               atomic_read(&ds->ds_count), ds->ds_clp,
+               ds->ds_clp ? ds->ds_clp->cl_exchange_flags : 0);
+}
+
+static bool
+same_sockaddr(struct sockaddr *addr1, struct sockaddr *addr2)
+{
+       struct sockaddr_in *a, *b;
+       struct sockaddr_in6 *a6, *b6;
+
+       if (addr1->sa_family != addr2->sa_family)
+               return false;
+
+       switch (addr1->sa_family) {
+       case AF_INET:
+               a = (struct sockaddr_in *)addr1;
+               b = (struct sockaddr_in *)addr2;
+
+               if (a->sin_addr.s_addr == b->sin_addr.s_addr &&
+                   a->sin_port == b->sin_port)
+                       return true;
+               break;
+
+       case AF_INET6:
+               a6 = (struct sockaddr_in6 *)addr1;
+               b6 = (struct sockaddr_in6 *)addr2;
+
+               /* LINKLOCAL addresses must have matching scope_id */
+               if (ipv6_addr_src_scope(&a6->sin6_addr) ==
+                   IPV6_ADDR_SCOPE_LINKLOCAL &&
+                   a6->sin6_scope_id != b6->sin6_scope_id)
+                       return false;
+
+               if (ipv6_addr_equal(&a6->sin6_addr, &b6->sin6_addr) &&
+                   a6->sin6_port == b6->sin6_port)
+                       return true;
+               break;
+
+       default:
+               dprintk("%s: unhandled address family: %u\n",
+                       __func__, addr1->sa_family);
+               return false;
+       }
+
+       return false;
+}
+
+static bool
+_same_data_server_addrs_locked(const struct list_head *dsaddrs1,
+                              const struct list_head *dsaddrs2)
+{
+       struct nfs4_pnfs_ds_addr *da1, *da2;
+
+       /* step through both lists, comparing as we go */
+       for (da1 = list_first_entry(dsaddrs1, typeof(*da1), da_node),
+            da2 = list_first_entry(dsaddrs2, typeof(*da2), da_node);
+            da1 != NULL && da2 != NULL;
+            da1 = list_entry(da1->da_node.next, typeof(*da1), da_node),
+            da2 = list_entry(da2->da_node.next, typeof(*da2), da_node)) {
+               if (!same_sockaddr((struct sockaddr *)&da1->da_addr,
+                                  (struct sockaddr *)&da2->da_addr))
+                       return false;
+       }
+       if (da1 == NULL && da2 == NULL)
+               return true;
+
+       return false;
+}
+
+/*
+ * Lookup DS by addresses.  nfs4_ds_cache_lock is held
+ */
+static struct nfs4_pnfs_ds *
+_data_server_lookup_locked(const struct list_head *dsaddrs)
+{
+       struct nfs4_pnfs_ds *ds;
+
+       list_for_each_entry(ds, &nfs4_data_server_cache, ds_node)
+               if (_same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs))
+                       return ds;
+       return NULL;
+}
+
+static void destroy_ds(struct nfs4_pnfs_ds *ds)
+{
+       struct nfs4_pnfs_ds_addr *da;
+
+       dprintk("--> %s\n", __func__);
+       ifdebug(FACILITY)
+               print_ds(ds);
+
+       nfs_put_client(ds->ds_clp);
+
+       while (!list_empty(&ds->ds_addrs)) {
+               da = list_first_entry(&ds->ds_addrs,
+                                     struct nfs4_pnfs_ds_addr,
+                                     da_node);
+               list_del_init(&da->da_node);
+               kfree(da->da_remotestr);
+               kfree(da);
+       }
+
+       kfree(ds->ds_remotestr);
+       kfree(ds);
+}
+
+void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds)
+{
+       if (atomic_dec_and_lock(&ds->ds_count,
+                               &nfs4_ds_cache_lock)) {
+               list_del_init(&ds->ds_node);
+               spin_unlock(&nfs4_ds_cache_lock);
+               destroy_ds(ds);
+       }
+}
+EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_put);
+
+/*
+ * Create a string with a human readable address and port to avoid
+ * complicated setup around many dprinks.
+ */
+static char *
+nfs4_pnfs_remotestr(struct list_head *dsaddrs, gfp_t gfp_flags)
+{
+       struct nfs4_pnfs_ds_addr *da;
+       char *remotestr;
+       size_t len;
+       char *p;
+
+       len = 3;        /* '{', '}' and eol */
+       list_for_each_entry(da, dsaddrs, da_node) {
+               len += strlen(da->da_remotestr) + 1;    /* string plus comma */
+       }
+
+       remotestr = kzalloc(len, gfp_flags);
+       if (!remotestr)
+               return NULL;
+
+       p = remotestr;
+       *(p++) = '{';
+       len--;
+       list_for_each_entry(da, dsaddrs, da_node) {
+               size_t ll = strlen(da->da_remotestr);
+
+               if (ll > len)
+                       goto out_err;
+
+               memcpy(p, da->da_remotestr, ll);
+               p += ll;
+               len -= ll;
+
+               if (len < 1)
+                       goto out_err;
+               (*p++) = ',';
+               len--;
+       }
+       if (len < 2)
+               goto out_err;
+       *(p++) = '}';
+       *p = '\0';
+       return remotestr;
+out_err:
+       kfree(remotestr);
+       return NULL;
+}
+
+/*
+ * Given a list of multipath struct nfs4_pnfs_ds_addr, add it to ds cache if
+ * uncached and return cached struct nfs4_pnfs_ds.
+ */
+struct nfs4_pnfs_ds *
+nfs4_pnfs_ds_add(struct list_head *dsaddrs, gfp_t gfp_flags)
+{
+       struct nfs4_pnfs_ds *tmp_ds, *ds = NULL;
+       char *remotestr;
+
+       if (list_empty(dsaddrs)) {
+               dprintk("%s: no addresses defined\n", __func__);
+               goto out;
+       }
+
+       ds = kzalloc(sizeof(*ds), gfp_flags);
+       if (!ds)
+               goto out;
+
+       /* this is only used for debugging, so it's ok if its NULL */
+       remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags);
+
+       spin_lock(&nfs4_ds_cache_lock);
+       tmp_ds = _data_server_lookup_locked(dsaddrs);
+       if (tmp_ds == NULL) {
+               INIT_LIST_HEAD(&ds->ds_addrs);
+               list_splice_init(dsaddrs, &ds->ds_addrs);
+               ds->ds_remotestr = remotestr;
+               atomic_set(&ds->ds_count, 1);
+               INIT_LIST_HEAD(&ds->ds_node);
+               ds->ds_clp = NULL;
+               list_add(&ds->ds_node, &nfs4_data_server_cache);
+               dprintk("%s add new data server %s\n", __func__,
+                       ds->ds_remotestr);
+       } else {
+               kfree(remotestr);
+               kfree(ds);
+               atomic_inc(&tmp_ds->ds_count);
+               dprintk("%s data server %s found, inc'ed ds_count to %d\n",
+                       __func__, tmp_ds->ds_remotestr,
+                       atomic_read(&tmp_ds->ds_count));
+               ds = tmp_ds;
+       }
+       spin_unlock(&nfs4_ds_cache_lock);
+out:
+       return ds;
+}
+EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_add);