IB/mlx5: Expose MR cache for mlx5_ib

Allow other parts of mlx5_ib to use MR cache mechanism.
* Add new functions mlx5_mr_cache_alloc and mlx5_mr_cache_free
* Traditional MTT MKey buckets are limited by MAX_UMR_CACHE_ENTRY
  Additinal buckets may be added above.

Signed-off-by: Artemy Kovalyov <artemyko@mellanox.com>
Signed-off-by: Leon Romanovsky <leon@kernel.org>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 13bef19649f2dfa8ded4ed2821fe2b40f2809c71..efc44de3c7d77b3bd6194e04aebda0419f9e7335 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -541,6 +541,10 @@ struct mlx5_cache_ent {
        struct dentry          *dir;
        char                    name[4];
        u32                     order;
+       u32                     xlt;
+       u32                     access_mode;
+       u32                     page;
+
        u32                     size;
        u32                     cur;
        u32                     miss;
@@ -555,6 +559,7 @@ struct mlx5_cache_ent {
        struct work_struct      work;
        struct delayed_work     dwork;
        int                     pending;
+       struct completion       compl;
 };
 
 struct mlx5_mr_cache {
@@ -837,7 +842,9 @@ void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq);
 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
-int mlx5_mr_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift);
+
+struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry);
+void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
                            struct ib_mr_status *mr_status);
 struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,

diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 8cf2a67f9fb0bad7bdbd201e7f359a859c782b80..8f5b94d483e46c5c958b4dcf5a0b23720ff9148b 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -49,6 +49,7 @@ enum {
 
 static int clean_mr(struct mlx5_ib_mr *mr);
 static int use_umr(struct mlx5_ib_dev *dev, int order);
+static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
 
 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
 {
@@ -149,6 +150,9 @@ static void reg_mr_callback(int status, void *context)
        if (err)
                pr_err("Error inserting to mkey tree. 0x%x\n", -err);
        write_unlock_irqrestore(&table->lock, flags);
+
+       if (!completion_done(&ent->compl))
+               complete(&ent->compl);
 }
 
 static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
@@ -157,7 +161,6 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
        struct mlx5_cache_ent *ent = &cache->ent[c];
        int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
        struct mlx5_ib_mr *mr;
-       int npages = 1 << ent->order;
        void *mkc;
        u32 *in;
        int err = 0;
@@ -185,11 +188,11 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
 
                MLX5_SET(mkc, mkc, free, 1);
                MLX5_SET(mkc, mkc, umr_en, 1);
-               MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
+               MLX5_SET(mkc, mkc, access_mode, ent->access_mode);
 
                MLX5_SET(mkc, mkc, qpn, 0xffffff);
-               MLX5_SET(mkc, mkc, translations_octword_size, (npages + 1) / 2);
-               MLX5_SET(mkc, mkc, log_page_size, 12);
+               MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
+               MLX5_SET(mkc, mkc, log_page_size, ent->page);
 
                spin_lock_irq(&ent->lock);
                ent->pending++;
@@ -447,6 +450,42 @@ static void cache_work_func(struct work_struct *work)
        __cache_work_func(ent);
 }
 
+struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry)
+{
+       struct mlx5_mr_cache *cache = &dev->cache;
+       struct mlx5_cache_ent *ent;
+       struct mlx5_ib_mr *mr;
+       int err;
+
+       if (entry < 0 || entry >= MAX_MR_CACHE_ENTRIES) {
+               mlx5_ib_err(dev, "cache entry %d is out of range\n", entry);
+               return NULL;
+       }
+
+       ent = &cache->ent[entry];
+       while (1) {
+               spin_lock_irq(&ent->lock);
+               if (list_empty(&ent->head)) {
+                       spin_unlock_irq(&ent->lock);
+
+                       err = add_keys(dev, entry, 1);
+                       if (err)
+                               return ERR_PTR(err);
+
+                       wait_for_completion(&ent->compl);
+               } else {
+                       mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
+                                             list);
+                       list_del(&mr->list);
+                       ent->cur--;
+                       spin_unlock_irq(&ent->lock);
+                       if (ent->cur < ent->limit)
+                               queue_work(cache->wq, &ent->work);
+                       return mr;
+               }
+       }
+}
+
 static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
 {
        struct mlx5_mr_cache *cache = &dev->cache;
@@ -456,12 +495,12 @@ static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
        int i;
 
        c = order2idx(dev, order);
-       if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) {
+       if (c < 0 || c > MAX_UMR_CACHE_ENTRY) {
                mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c);
                return NULL;
        }
 
-       for (i = c; i < MAX_MR_CACHE_ENTRIES; i++) {
+       for (i = c; i < MAX_UMR_CACHE_ENTRY; i++) {
                ent = &cache->ent[i];
 
                mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i);
@@ -488,7 +527,7 @@ static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
        return mr;
 }
 
-static void free_cached_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
+void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
 {
        struct mlx5_mr_cache *cache = &dev->cache;
        struct mlx5_cache_ent *ent;
@@ -500,6 +539,10 @@ static void free_cached_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
                mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c);
                return;
        }
+
+       if (unreg_umr(dev, mr))
+               return;
+
        ent = &cache->ent[c];
        spin_lock_irq(&ent->lock);
        list_add_tail(&mr->list, &ent->head);
@@ -602,7 +645,6 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
 {
        struct mlx5_mr_cache *cache = &dev->cache;
        struct mlx5_cache_ent *ent;
-       int limit;
        int err;
        int i;
 
@@ -615,26 +657,33 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
 
        setup_timer(&dev->delay_timer, delay_time_func, (unsigned long)dev);
        for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
-               INIT_LIST_HEAD(&cache->ent[i].head);
-               spin_lock_init(&cache->ent[i].lock);
-
                ent = &cache->ent[i];
                INIT_LIST_HEAD(&ent->head);
                spin_lock_init(&ent->lock);
                ent->order = i + 2;
                ent->dev = dev;
+               ent->limit = 0;
 
-               if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) &&
-                   mlx5_core_is_pf(dev->mdev) &&
-                   use_umr(dev, ent->order))
-                       limit = dev->mdev->profile->mr_cache[i].limit;
-               else
-                       limit = 0;
-
+               init_completion(&ent->compl);
                INIT_WORK(&ent->work, cache_work_func);
                INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
-               ent->limit = limit;
                queue_work(cache->wq, &ent->work);
+
+               if (i > MAX_UMR_CACHE_ENTRY)
+                       continue;
+
+               if (!use_umr(dev, ent->order))
+                       continue;
+
+               ent->page = PAGE_SHIFT;
+               ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) /
+                          MLX5_IB_UMR_OCTOWORD;
+               ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
+               if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) &&
+                   mlx5_core_is_pf(dev->mdev))
+                       ent->limit = dev->mdev->profile->mr_cache[i].limit;
+               else
+                       ent->limit = 0;
        }
 
        err = mlx5_mr_cache_debugfs_init(dev);
@@ -758,7 +807,7 @@ static int get_octo_len(u64 addr, u64 len, int page_size)
 static int use_umr(struct mlx5_ib_dev *dev, int order)
 {
        if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
-               return order < MAX_MR_CACHE_ENTRIES + 2;
+               return order <= MAX_UMR_CACHE_ENTRY + 2;
        return order <= MLX5_MAX_UMR_SHIFT;
 }
 
@@ -871,7 +920,7 @@ static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
                                 MLX5_IB_UPD_XLT_ENABLE);
 
        if (err) {
-               free_cached_mr(dev, mr);
+               mlx5_mr_cache_free(dev, mr);
                return ERR_PTR(err);
        }
 
@@ -1091,6 +1140,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
                goto err_2;
        }
        mr->mmkey.type = MLX5_MKEY_MR;
+       mr->desc_size = sizeof(struct mlx5_mtt);
        mr->umem = umem;
        mr->dev = dev;
        mr->live = 1;
@@ -1398,12 +1448,7 @@ static int clean_mr(struct mlx5_ib_mr *mr)
                        return err;
                }
        } else {
-               err = unreg_umr(dev, mr);
-               if (err) {
-                       mlx5_ib_warn(dev, "failed unregister\n");
-                       return err;
-               }
-               free_cached_mr(dev, mr);
+               mlx5_mr_cache_free(dev, mr);
        }
 
        if (!umred)

diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index b8d69aeb178417385414eff2bd01d15a0422e4a4..2534b8a0fd7b3e35aa93f7a8510b8f7a5ddf526b 100644 (file)
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -1052,7 +1052,8 @@ enum {
 };
 
 enum {
-       MAX_MR_CACHE_ENTRIES    = 21,
+       MAX_UMR_CACHE_ENTRY = 20,
+       MAX_MR_CACHE_ENTRIES
 };
 
 enum {