fs: dcache scale lru

Add a new lock, dcache_lru_lock, to protect the dcache LRU list from concurrent
modification. d_lru is also protected by d_lock, which allows LRU lists to be
accessed without the lru lock, using RCU in future patches.

Signed-off-by: Nick Piggin <npiggin@kernel.dk>

diff --git a/fs/dcache.c b/fs/dcache.c
index 1e124d4673c65bd3b235f41632f69a6148390394..3d3c843c36edb07a180b95c7f026b5900609cb3a 100644 (file)
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -37,11 +37,19 @@
 
 /*
  * Usage:
- * dcache_hash_lock protects dcache hash table, s_anon lists
+ * dcache_hash_lock protects:
+ *   - the dcache hash table, s_anon lists
+ * dcache_lru_lock protects:
+ *   - the dcache lru lists and counters
+ * d_lock protects:
+ *   - d_flags
+ *   - d_name
+ *   - d_lru
  *
  * Ordering:
  * dcache_lock
  *   dentry->d_lock
+ *     dcache_lru_lock
  *     dcache_hash_lock
  *
  * if (dentry1 < dentry2)
@@ -52,6 +60,7 @@ int sysctl_vfs_cache_pressure __read_mostly = 100;
 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
 
 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_hash_lock);
+static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock);
 __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock);
 __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
 
@@ -154,28 +163,38 @@ static void dentry_iput(struct dentry * dentry)
 }
 
 /*
- * dentry_lru_(add|del|move_tail) must be called with dcache_lock held.
+ * dentry_lru_(add|del|move_tail) must be called with d_lock held.
  */
 static void dentry_lru_add(struct dentry *dentry)
 {
        if (list_empty(&dentry->d_lru)) {
+               spin_lock(&dcache_lru_lock);
                list_add(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
                dentry->d_sb->s_nr_dentry_unused++;
                dentry_stat.nr_unused++;
+               spin_unlock(&dcache_lru_lock);
        }
 }
 
+static void __dentry_lru_del(struct dentry *dentry)
+{
+       list_del_init(&dentry->d_lru);
+       dentry->d_sb->s_nr_dentry_unused--;
+       dentry_stat.nr_unused--;
+}
+
 static void dentry_lru_del(struct dentry *dentry)
 {
        if (!list_empty(&dentry->d_lru)) {
-               list_del_init(&dentry->d_lru);
-               dentry->d_sb->s_nr_dentry_unused--;
-               dentry_stat.nr_unused--;
+               spin_lock(&dcache_lru_lock);
+               __dentry_lru_del(dentry);
+               spin_unlock(&dcache_lru_lock);
        }
 }
 
 static void dentry_lru_move_tail(struct dentry *dentry)
 {
+       spin_lock(&dcache_lru_lock);
        if (list_empty(&dentry->d_lru)) {
                list_add_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
                dentry->d_sb->s_nr_dentry_unused++;
@@ -183,6 +202,7 @@ static void dentry_lru_move_tail(struct dentry *dentry)
        } else {
                list_move_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
        }
+       spin_unlock(&dcache_lru_lock);
 }
 
 /**
@@ -192,6 +212,8 @@ static void dentry_lru_move_tail(struct dentry *dentry)
  * The dentry must already be unhashed and removed from the LRU.
  *
  * If this is the root of the dentry tree, return NULL.
+ *
+ * dcache_lock and d_lock must be held by caller, are dropped by d_kill.
  */
 static struct dentry *d_kill(struct dentry *dentry)
        __releases(dentry->d_lock)
@@ -383,10 +405,19 @@ int d_invalidate(struct dentry * dentry)
 EXPORT_SYMBOL(d_invalidate);
 
 /* This should be called _only_ with dcache_lock held */
+static inline struct dentry * __dget_locked_dlock(struct dentry *dentry)
+{
+       atomic_inc(&dentry->d_count);
+       dentry_lru_del(dentry);
+       return dentry;
+}
+
 static inline struct dentry * __dget_locked(struct dentry *dentry)
 {
        atomic_inc(&dentry->d_count);
+       spin_lock(&dentry->d_lock);
        dentry_lru_del(dentry);
+       spin_unlock(&dentry->d_lock);
        return dentry;
 }
 
@@ -465,7 +496,7 @@ restart:
        list_for_each_entry(dentry, &inode->i_dentry, d_alias) {
                spin_lock(&dentry->d_lock);
                if (!atomic_read(&dentry->d_count)) {
-                       __dget_locked(dentry);
+                       __dget_locked_dlock(dentry);
                        __d_drop(dentry);
                        spin_unlock(&dentry->d_lock);
                        spin_unlock(&dcache_lock);
@@ -489,7 +520,6 @@ EXPORT_SYMBOL(d_prune_aliases);
 static void prune_one_dentry(struct dentry * dentry)
        __releases(dentry->d_lock)
        __releases(dcache_lock)
-       __acquires(dcache_lock)
 {
        __d_drop(dentry);
        dentry = d_kill(dentry);
@@ -498,15 +528,16 @@ static void prune_one_dentry(struct dentry * dentry)
         * Prune ancestors.  Locking is simpler than in dput(),
         * because dcache_lock needs to be taken anyway.
         */
-       spin_lock(&dcache_lock);
        while (dentry) {
-               if (!atomic_dec_and_lock(&dentry->d_count, &dentry->d_lock))
+               spin_lock(&dcache_lock);
+               if (!atomic_dec_and_lock(&dentry->d_count, &dentry->d_lock)) {
+                       spin_unlock(&dcache_lock);
                        return;
+               }
 
                dentry_lru_del(dentry);
                __d_drop(dentry);
                dentry = d_kill(dentry);
-               spin_lock(&dcache_lock);
        }
 }
 
@@ -516,21 +547,31 @@ static void shrink_dentry_list(struct list_head *list)
 
        while (!list_empty(list)) {
                dentry = list_entry(list->prev, struct dentry, d_lru);
-               dentry_lru_del(dentry);
+
+               if (!spin_trylock(&dentry->d_lock)) {
+                       spin_unlock(&dcache_lru_lock);
+                       cpu_relax();
+                       spin_lock(&dcache_lru_lock);
+                       continue;
+               }
+
+               __dentry_lru_del(dentry);
 
                /*
                 * We found an inuse dentry which was not removed from
                 * the LRU because of laziness during lookup.  Do not free
                 * it - just keep it off the LRU list.
                 */
-               spin_lock(&dentry->d_lock);
                if (atomic_read(&dentry->d_count)) {
                        spin_unlock(&dentry->d_lock);
                        continue;
                }
+               spin_unlock(&dcache_lru_lock);
+
                prune_one_dentry(dentry);
-               /* dentry->d_lock was dropped in prune_one_dentry() */
-               cond_resched_lock(&dcache_lock);
+               /* dcache_lock and dentry->d_lock dropped */
+               spin_lock(&dcache_lock);
+               spin_lock(&dcache_lru_lock);
        }
 }
 
@@ -551,32 +592,36 @@ static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags)
        int cnt = *count;
 
        spin_lock(&dcache_lock);
+relock:
+       spin_lock(&dcache_lru_lock);
        while (!list_empty(&sb->s_dentry_lru)) {
                dentry = list_entry(sb->s_dentry_lru.prev,
                                struct dentry, d_lru);
                BUG_ON(dentry->d_sb != sb);
 
+               if (!spin_trylock(&dentry->d_lock)) {
+                       spin_unlock(&dcache_lru_lock);
+                       cpu_relax();
+                       goto relock;
+               }
+
                /*
                 * If we are honouring the DCACHE_REFERENCED flag and the
                 * dentry has this flag set, don't free it.  Clear the flag
                 * and put it back on the LRU.
                 */
-               if (flags & DCACHE_REFERENCED) {
-                       spin_lock(&dentry->d_lock);
-                       if (dentry->d_flags & DCACHE_REFERENCED) {
-                               dentry->d_flags &= ~DCACHE_REFERENCED;
-                               list_move(&dentry->d_lru, &referenced);
-                               spin_unlock(&dentry->d_lock);
-                               cond_resched_lock(&dcache_lock);
-                               continue;
-                       }
+               if (flags & DCACHE_REFERENCED &&
+                               dentry->d_flags & DCACHE_REFERENCED) {
+                       dentry->d_flags &= ~DCACHE_REFERENCED;
+                       list_move(&dentry->d_lru, &referenced);
+                       spin_unlock(&dentry->d_lock);
+               } else {
+                       list_move_tail(&dentry->d_lru, &tmp);
                        spin_unlock(&dentry->d_lock);
+                       if (!--cnt)
+                               break;
                }
-
-               list_move_tail(&dentry->d_lru, &tmp);
-               if (!--cnt)
-                       break;
-               cond_resched_lock(&dcache_lock);
+               /* XXX: re-add cond_resched_lock when dcache_lock goes away */
        }
 
        *count = cnt;
@@ -584,6 +629,7 @@ static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags)
 
        if (!list_empty(&referenced))
                list_splice(&referenced, &sb->s_dentry_lru);
+       spin_unlock(&dcache_lru_lock);
        spin_unlock(&dcache_lock);
 
 }
@@ -679,10 +725,12 @@ void shrink_dcache_sb(struct super_block *sb)
        LIST_HEAD(tmp);
 
        spin_lock(&dcache_lock);
+       spin_lock(&dcache_lru_lock);
        while (!list_empty(&sb->s_dentry_lru)) {
                list_splice_init(&sb->s_dentry_lru, &tmp);
                shrink_dentry_list(&tmp);
        }
+       spin_unlock(&dcache_lru_lock);
        spin_unlock(&dcache_lock);
 }
 EXPORT_SYMBOL(shrink_dcache_sb);
@@ -701,7 +749,9 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
 
        /* detach this root from the system */
        spin_lock(&dcache_lock);
+       spin_lock(&dentry->d_lock);
        dentry_lru_del(dentry);
+       spin_unlock(&dentry->d_lock);
        __d_drop(dentry);
        spin_unlock(&dcache_lock);
 
@@ -715,7 +765,9 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
                        spin_lock(&dcache_lock);
                        list_for_each_entry(loop, &dentry->d_subdirs,
                                            d_u.d_child) {
+                               spin_lock(&loop->d_lock);
                                dentry_lru_del(loop);
+                               spin_unlock(&loop->d_lock);
                                __d_drop(loop);
                                cond_resched_lock(&dcache_lock);
                        }
@@ -892,6 +944,8 @@ resume:
                struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
                next = tmp->next;
 
+               spin_lock(&dentry->d_lock);
+
                /* 
                 * move only zero ref count dentries to the end 
                 * of the unused list for prune_dcache
@@ -903,6 +957,8 @@ resume:
                        dentry_lru_del(dentry);
                }
 
+               spin_unlock(&dentry->d_lock);
+
                /*
                 * We can return to the caller if we have found some (this
                 * ensures forward progress). We'll be coming back to find