Btrfs: add the ability to cache a pointer into the eb

This cuts down on the CPU time used by map_private_extent_buffer

Signed-off-by: Chris Mason <chris.mason@oracle.com>

diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c
index 74c03fb0ca1dd91f0723c54acb18645b1153318e..8af24da983b1156e888ac0a95f58e7aa0a5bb9bb 100644 (file)
--- a/fs/btrfs/ctree.c
+++ b/fs/btrfs/ctree.c
@@ -2333,6 +2333,7 @@ static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
 {
        struct extent_buffer *left = path->nodes[0];
        struct extent_buffer *upper = path->nodes[1];
+       struct btrfs_map_token token;
        struct btrfs_disk_key disk_key;
        int slot;
        u32 i;
@@ -2344,6 +2345,8 @@ static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
        u32 data_end;
        u32 this_item_size;
 
+       btrfs_init_map_token(&token);
+
        if (empty)
                nr = 0;
        else
@@ -2421,8 +2424,8 @@ static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
        push_space = BTRFS_LEAF_DATA_SIZE(root);
        for (i = 0; i < right_nritems; i++) {
                item = btrfs_item_nr(right, i);
-               push_space -= btrfs_item_size(right, item);
-               btrfs_set_item_offset(right, item, push_space);
+               push_space -= btrfs_token_item_size(right, item, &token);
+               btrfs_set_token_item_offset(right, item, push_space, &token);
        }
 
        left_nritems -= push_items;
@@ -2553,6 +2556,9 @@ static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
        int wret;
        u32 this_item_size;
        u32 old_left_item_size;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        if (empty)
                nr = min(right_nritems, max_slot);
@@ -2613,9 +2619,10 @@ static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
 
                item = btrfs_item_nr(left, i);
 
-               ioff = btrfs_item_offset(left, item);
-               btrfs_set_item_offset(left, item,
-                     ioff - (BTRFS_LEAF_DATA_SIZE(root) - old_left_item_size));
+               ioff = btrfs_token_item_offset(left, item, &token);
+               btrfs_set_token_item_offset(left, item,
+                     ioff - (BTRFS_LEAF_DATA_SIZE(root) - old_left_item_size),
+                     &token);
        }
        btrfs_set_header_nritems(left, old_left_nritems + push_items);
 
@@ -2645,8 +2652,9 @@ static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
        for (i = 0; i < right_nritems; i++) {
                item = btrfs_item_nr(right, i);
 
-               push_space = push_space - btrfs_item_size(right, item);
-               btrfs_set_item_offset(right, item, push_space);
+               push_space = push_space - btrfs_token_item_size(right,
+                                                               item, &token);
+               btrfs_set_token_item_offset(right, item, push_space, &token);
        }
 
        btrfs_mark_buffer_dirty(left);
@@ -2767,6 +2775,9 @@ static noinline int copy_for_split(struct btrfs_trans_handle *trans,
        int ret = 0;
        int wret;
        struct btrfs_disk_key disk_key;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        nritems = nritems - mid;
        btrfs_set_header_nritems(right, nritems);
@@ -2788,8 +2799,9 @@ static noinline int copy_for_split(struct btrfs_trans_handle *trans,
                struct btrfs_item *item = btrfs_item_nr(right, i);
                u32 ioff;
 
-               ioff = btrfs_item_offset(right, item);
-               btrfs_set_item_offset(right, item, ioff + rt_data_off);
+               ioff = btrfs_token_item_offset(right, item, &token);
+               btrfs_set_token_item_offset(right, item,
+                                           ioff + rt_data_off, &token);
        }
 
        btrfs_set_header_nritems(l, mid);
@@ -3284,6 +3296,9 @@ int btrfs_truncate_item(struct btrfs_trans_handle *trans,
        unsigned int old_size;
        unsigned int size_diff;
        int i;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        leaf = path->nodes[0];
        slot = path->slots[0];
@@ -3310,8 +3325,9 @@ int btrfs_truncate_item(struct btrfs_trans_handle *trans,
                u32 ioff;
                item = btrfs_item_nr(leaf, i);
 
-               ioff = btrfs_item_offset(leaf, item);
-               btrfs_set_item_offset(leaf, item, ioff + size_diff);
+               ioff = btrfs_token_item_offset(leaf, item, &token);
+               btrfs_set_token_item_offset(leaf, item,
+                                           ioff + size_diff, &token);
        }
 
        /* shift the data */
@@ -3381,6 +3397,9 @@ int btrfs_extend_item(struct btrfs_trans_handle *trans,
        unsigned int old_data;
        unsigned int old_size;
        int i;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        leaf = path->nodes[0];
 
@@ -3410,8 +3429,9 @@ int btrfs_extend_item(struct btrfs_trans_handle *trans,
                u32 ioff;
                item = btrfs_item_nr(leaf, i);
 
-               ioff = btrfs_item_offset(leaf, item);
-               btrfs_set_item_offset(leaf, item, ioff - data_size);
+               ioff = btrfs_token_item_offset(leaf, item, &token);
+               btrfs_set_token_item_offset(leaf, item,
+                                           ioff - data_size, &token);
        }
 
        /* shift the data */
@@ -3454,6 +3474,9 @@ int btrfs_insert_some_items(struct btrfs_trans_handle *trans,
        unsigned int data_end;
        struct btrfs_disk_key disk_key;
        struct btrfs_key found_key;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        for (i = 0; i < nr; i++) {
                if (total_size + data_size[i] + sizeof(struct btrfs_item) >
@@ -3519,8 +3542,9 @@ int btrfs_insert_some_items(struct btrfs_trans_handle *trans,
                        u32 ioff;
 
                        item = btrfs_item_nr(leaf, i);
-                       ioff = btrfs_item_offset(leaf, item);
-                       btrfs_set_item_offset(leaf, item, ioff - total_data);
+                       ioff = btrfs_token_item_offset(leaf, item, &token);
+                       btrfs_set_token_item_offset(leaf, item,
+                                                   ioff - total_data, &token);
                }
                /* shift the items */
                memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr),
@@ -3547,9 +3571,10 @@ int btrfs_insert_some_items(struct btrfs_trans_handle *trans,
                btrfs_cpu_key_to_disk(&disk_key, cpu_key + i);
                btrfs_set_item_key(leaf, &disk_key, slot + i);
                item = btrfs_item_nr(leaf, slot + i);
-               btrfs_set_item_offset(leaf, item, data_end - data_size[i]);
+               btrfs_set_token_item_offset(leaf, item,
+                                           data_end - data_size[i], &token);
                data_end -= data_size[i];
-               btrfs_set_item_size(leaf, item, data_size[i]);
+               btrfs_set_token_item_size(leaf, item, data_size[i], &token);
        }
        btrfs_set_header_nritems(leaf, nritems + nr);
        btrfs_mark_buffer_dirty(leaf);
@@ -3588,6 +3613,9 @@ int setup_items_for_insert(struct btrfs_trans_handle *trans,
        int ret;
        struct extent_buffer *leaf;
        int slot;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        leaf = path->nodes[0];
        slot = path->slots[0];
@@ -3619,8 +3647,9 @@ int setup_items_for_insert(struct btrfs_trans_handle *trans,
                        u32 ioff;
 
                        item = btrfs_item_nr(leaf, i);
-                       ioff = btrfs_item_offset(leaf, item);
-                       btrfs_set_item_offset(leaf, item, ioff - total_data);
+                       ioff = btrfs_token_item_offset(leaf, item, &token);
+                       btrfs_set_token_item_offset(leaf, item,
+                                                   ioff - total_data, &token);
                }
                /* shift the items */
                memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr),
@@ -3639,9 +3668,10 @@ int setup_items_for_insert(struct btrfs_trans_handle *trans,
                btrfs_cpu_key_to_disk(&disk_key, cpu_key + i);
                btrfs_set_item_key(leaf, &disk_key, slot + i);
                item = btrfs_item_nr(leaf, slot + i);
-               btrfs_set_item_offset(leaf, item, data_end - data_size[i]);
+               btrfs_set_token_item_offset(leaf, item,
+                                           data_end - data_size[i], &token);
                data_end -= data_size[i];
-               btrfs_set_item_size(leaf, item, data_size[i]);
+               btrfs_set_token_item_size(leaf, item, data_size[i], &token);
        }
 
        btrfs_set_header_nritems(leaf, nritems + nr);
@@ -3814,6 +3844,9 @@ int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
        int wret;
        int i;
        u32 nritems;
+       struct btrfs_map_token token;
+
+       btrfs_init_map_token(&token);
 
        leaf = path->nodes[0];
        last_off = btrfs_item_offset_nr(leaf, slot + nr - 1);
@@ -3835,8 +3868,9 @@ int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
                        u32 ioff;
 
                        item = btrfs_item_nr(leaf, i);
-                       ioff = btrfs_item_offset(leaf, item);
-                       btrfs_set_item_offset(leaf, item, ioff + dsize);
+                       ioff = btrfs_token_item_offset(leaf, item, &token);
+                       btrfs_set_token_item_offset(leaf, item,
+                                                   ioff + dsize, &token);
                }
 
                memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot),

diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index 85ab1c5844a26af67819d95fa4fed174bb87068d..c2e17cd299b74d9f0c112a9acd8975519aa7974c 100644 (file)
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -1546,6 +1546,17 @@ struct btrfs_ioctl_defrag_range_args {
 
 #define BTRFS_INODE_ROOT_ITEM_INIT     (1 << 31)
 
+struct btrfs_map_token {
+       struct extent_buffer *eb;
+       char *kaddr;
+       unsigned long offset;
+};
+
+static inline void btrfs_init_map_token (struct btrfs_map_token *token)
+{
+       memset(token, 0, sizeof(*token));
+}
+
 /* some macros to generate set/get funcs for the struct fields.  This
  * assumes there is a lefoo_to_cpu for every type, so lets make a simple
  * one for u8:
@@ -1569,6 +1580,8 @@ struct btrfs_ioctl_defrag_range_args {
 #ifndef BTRFS_SETGET_FUNCS
 #define BTRFS_SETGET_FUNCS(name, type, member, bits)                   \
 u##bits btrfs_##name(struct extent_buffer *eb, type *s);               \
+u##bits btrfs_token_##name(struct extent_buffer *eb, type *s, struct btrfs_map_token *token);          \
+void btrfs_set_token_##name(struct extent_buffer *eb, type *s, u##bits val, struct btrfs_map_token *token);\
 void btrfs_set_##name(struct extent_buffer *eb, type *s, u##bits val);
 #endif
 

diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c
index bc1f6ad18442bc728a10e9919b91162af1b60b4d..c6ffa58124192a9e069b1464d863b33300bcf6e9 100644 (file)
--- a/fs/btrfs/struct-funcs.c
+++ b/fs/btrfs/struct-funcs.c
@@ -44,8 +44,9 @@
 #define BTRFS_SETGET_FUNCS(name, type, member, bits)                   \
 u##bits btrfs_##name(struct extent_buffer *eb, type *s);               \
 void btrfs_set_##name(struct extent_buffer *eb, type *s, u##bits val); \
-u##bits btrfs_##name(struct extent_buffer *eb,                         \
-                                  type *s)                             \
+void btrfs_set_token_##name(struct extent_buffer *eb, type *s, u##bits val, struct btrfs_map_token *token);    \
+u##bits btrfs_token_##name(struct extent_buffer *eb,                           \
+                          type *s, struct btrfs_map_token *token)      \
 {                                                                      \
        unsigned long part_offset = (unsigned long)s;                   \
        unsigned long offset = part_offset + offsetof(type, member);    \
@@ -54,9 +55,18 @@ u##bits btrfs_##name(struct extent_buffer *eb,                               \
        char *kaddr;                                            \
        unsigned long map_start;                                \
        unsigned long map_len;                                  \
+       unsigned long mem_len = sizeof(((type *)0)->member);    \
        u##bits res;                                            \
+       if (token && token->kaddr && token->offset <= offset && \
+           token->eb == eb &&                                  \
+          (token->offset + PAGE_CACHE_SIZE >= offset + mem_len)) { \
+               kaddr = token->kaddr;                           \
+               p = (type *)(kaddr + part_offset - token->offset);  \
+               res = le##bits##_to_cpu(p->member);             \
+               return res;                                     \
+       }                                                       \
        err = map_private_extent_buffer(eb, offset,             \
-                       sizeof(((type *)0)->member),            \
+                       mem_len,                                \
                        &kaddr, &map_start, &map_len);          \
        if (err) {                                              \
                __le##bits leres;                               \
@@ -65,10 +75,15 @@ u##bits btrfs_##name(struct extent_buffer *eb,                              \
        }                                                       \
        p = (type *)(kaddr + part_offset - map_start);          \
        res = le##bits##_to_cpu(p->member);                     \
+       if (token) {                                            \
+               token->kaddr = kaddr;                           \
+               token->offset = map_start;                      \
+               token->eb = eb;                                 \
+       }                                                       \
        return res;                                             \
 }                                                                      \
-void btrfs_set_##name(struct extent_buffer *eb,                                \
-                                   type *s, u##bits val)               \
+void btrfs_set_token_##name(struct extent_buffer *eb,                          \
+                           type *s, u##bits val, struct btrfs_map_token *token)                \
 {                                                                      \
        unsigned long part_offset = (unsigned long)s;                   \
        unsigned long offset = part_offset + offsetof(type, member);    \
@@ -77,8 +92,17 @@ void btrfs_set_##name(struct extent_buffer *eb,                              \
        char *kaddr;                                            \
        unsigned long map_start;                                \
        unsigned long map_len;                                  \
+       unsigned long mem_len = sizeof(((type *)0)->member);    \
+       if (token && token->kaddr && token->offset <= offset && \
+           token->eb == eb &&                                  \
+          (token->offset + PAGE_CACHE_SIZE >= offset + mem_len)) { \
+               kaddr = token->kaddr;                           \
+               p = (type *)(kaddr + part_offset - token->offset);  \
+               p->member = cpu_to_le##bits(val);               \
+               return;                                         \
+       }                                                       \
        err = map_private_extent_buffer(eb, offset,             \
-                       sizeof(((type *)0)->member),            \
+                       mem_len,                                \
                        &kaddr, &map_start, &map_len);          \
        if (err) {                                              \
                __le##bits val2;                                \
@@ -88,7 +112,22 @@ void btrfs_set_##name(struct extent_buffer *eb,                             \
        }                                                       \
        p = (type *)(kaddr + part_offset - map_start);          \
        p->member = cpu_to_le##bits(val);                       \
-}
+       if (token) {                                            \
+               token->kaddr = kaddr;                           \
+               token->offset = map_start;                      \
+               token->eb = eb;                                 \
+       }                                                       \
+}                                                              \
+void btrfs_set_##name(struct extent_buffer *eb,                        \
+                     type *s, u##bits val)                     \
+{                                                              \
+       btrfs_set_token_##name(eb, s, val, NULL);               \
+}                                                              \
+u##bits btrfs_##name(struct extent_buffer *eb,                 \
+                     type *s)                                  \
+{                                                              \
+       return btrfs_token_##name(eb, s, NULL);                 \
+}                                                              \
 
 #include "ctree.h"