parallel lookups machinery, part 2

We'll need to verify that there's neither a hashed nor in-lookup
dentry with desired parent/name before adding to in-lookup set.

One possible solution would be to hold the parent's ->d_lock through
both checks, but while the in-lookup set is relatively small at any
time, dcache is not.  And holding the parent's ->d_lock through
something like __d_lookup_rcu() would suck too badly.

So we leave the parent's ->d_lock alone, which means that we watch
out for the following scenario:
	* we verify that there's no hashed match
	* existing in-lookup match gets hashed by another process
	* we verify that there's no in-lookup matches and decide
that everything's fine.

Solution: per-directory kinda-sorta seqlock, bumped around the times
we hash something that used to be in-lookup or move (and hash)
something in place of in-lookup.  Then the above would turn into
	* read the counter
	* do dcache lookup
	* if no matches found, check for in-lookup matches
	* if there had been none of those either, check if the
counter has changed; repeat if it has.

The "kinda-sorta" part is due to the fact that we don't have much spare
space in inode.  There is a spare word (shared with i_bdev/i_cdev/i_pipe),
so the counter part is not a problem, but spinlock is a different story.

We could use the parent's ->d_lock, and it would be less painful in
terms of contention, for __d_add() it would be rather inconvenient to
grab; we could do that (using lock_parent()), but...

Fortunately, we can get serialization on the counter itself, and it
might be a good idea in general; we can use cmpxchg() in a loop to
get from even to odd and smp_store_release() from odd to even.

This commit adds the counter and updating logics; the readers will be
added in the next commit.

Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

diff --git a/Documentation/filesystems/porting b/Documentation/filesystems/porting
index 57bb3754a027f53e3df7439fc1e550d54c6d7ae9..8810e2367fe6252cad7c8a4536d0d69a6eb2075b 100644 (file)
--- a/Documentation/filesystems/porting
+++ b/Documentation/filesystems/porting
@@ -531,3 +531,11 @@ in your dentry operations instead.
        dentry might be yet to be attached to inode, so do _not_ use its ->d_inode
        in the instances.  Rationale: !@#!@# security_d_instantiate() needs to be
        called before we attach dentry to inode.
+--
+[mandatory]
+       symlinks are no longer the only inodes that do *not* have i_bdev/i_cdev/
+       i_pipe/i_link union zeroed out at inode eviction.  As the result, you can't
+       assume that non-NULL value in ->i_nlink at ->destroy_inode() implies that
+       it's a symlink.  Checking ->i_mode is really needed now.  In-tree we had
+       to fix shmem_destroy_callback() that used to take that kind of shortcut;
+       watch out, since that shortcut is no longer valid.

diff --git a/fs/dcache.c b/fs/dcache.c
index 0f1d93866e692a84d555c3ebfa60ffbc880385f6..10988f7e5a23ce223a609a0cd30a3ef4ad2b1aca 100644 (file)
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -2364,6 +2364,22 @@ void d_rehash(struct dentry * entry)
 }
 EXPORT_SYMBOL(d_rehash);
 
+static inline unsigned start_dir_add(struct inode *dir)
+{
+
+       for (;;) {
+               unsigned n = dir->i_dir_seq;
+               if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
+                       return n;
+               cpu_relax();
+       }
+}
+
+static inline void end_dir_add(struct inode *dir, unsigned n)
+{
+       smp_store_release(&dir->i_dir_seq, n + 2);
+}
+
 void __d_lookup_done(struct dentry *dentry)
 {
        dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
@@ -2375,9 +2391,14 @@ EXPORT_SYMBOL(__d_lookup_done);
 
 static inline void __d_add(struct dentry *dentry, struct inode *inode)
 {
+       struct inode *dir = NULL;
+       unsigned n;
        spin_lock(&dentry->d_lock);
-       if (unlikely(d_in_lookup(dentry)))
+       if (unlikely(d_in_lookup(dentry))) {
+               dir = dentry->d_parent->d_inode;
+               n = start_dir_add(dir);
                __d_lookup_done(dentry);
+       }
        if (inode) {
                unsigned add_flags = d_flags_for_inode(inode);
                hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
@@ -2387,6 +2408,8 @@ static inline void __d_add(struct dentry *dentry, struct inode *inode)
                __fsnotify_d_instantiate(dentry);
        }
        _d_rehash(dentry);
+       if (dir)
+               end_dir_add(dir, n);
        spin_unlock(&dentry->d_lock);
        if (inode)
                spin_unlock(&inode->i_lock);
@@ -2616,6 +2639,8 @@ static void dentry_unlock_for_move(struct dentry *dentry, struct dentry *target)
 static void __d_move(struct dentry *dentry, struct dentry *target,
                     bool exchange)
 {
+       struct inode *dir = NULL;
+       unsigned n;
        if (!dentry->d_inode)
                printk(KERN_WARNING "VFS: moving negative dcache entry\n");
 
@@ -2623,8 +2648,11 @@ static void __d_move(struct dentry *dentry, struct dentry *target,
        BUG_ON(d_ancestor(target, dentry));
 
        dentry_lock_for_move(dentry, target);
-       if (unlikely(d_in_lookup(target)))
+       if (unlikely(d_in_lookup(target))) {
+               dir = target->d_parent->d_inode;
+               n = start_dir_add(dir);
                __d_lookup_done(target);
+       }
 
        write_seqcount_begin(&dentry->d_seq);
        write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
@@ -2674,6 +2702,8 @@ static void __d_move(struct dentry *dentry, struct dentry *target,
        write_seqcount_end(&target->d_seq);
        write_seqcount_end(&dentry->d_seq);
 
+       if (dir)
+               end_dir_add(dir, n);
        dentry_unlock_for_move(dentry, target);
 }
 

diff --git a/fs/inode.c b/fs/inode.c
index 4202aac99464079e89609a492fb0c514a0af8a7c..4b884f73214e457b33320a06ddce823791abfad9 100644 (file)
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -151,6 +151,7 @@ int inode_init_always(struct super_block *sb, struct inode *inode)
        inode->i_bdev = NULL;
        inode->i_cdev = NULL;
        inode->i_link = NULL;
+       inode->i_dir_seq = 0;
        inode->i_rdev = 0;
        inode->dirtied_when = 0;
 

diff --git a/include/linux/fs.h b/include/linux/fs.h
index 6d0fa9174a24b0609fd20032dc3973e3bccfc154..00cecc5a2f75ded40d75420285c6027db4677f21 100644 (file)
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -684,6 +684,7 @@ struct inode {
                struct block_device     *i_bdev;
                struct cdev             *i_cdev;
                char                    *i_link;
+               unsigned                i_dir_seq;
        };
 
        __u32                   i_generation;

diff --git a/mm/shmem.c b/mm/shmem.c
index 4640699b209bf7239809bc43063777c021348f1b..e684a914022805cce8dddf3fb84ea66359f4b315 100644 (file)
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -3123,7 +3123,8 @@ static struct inode *shmem_alloc_inode(struct super_block *sb)
 static void shmem_destroy_callback(struct rcu_head *head)
 {
        struct inode *inode = container_of(head, struct inode, i_rcu);
-       kfree(inode->i_link);
+       if (S_ISLNK(inode->i_mode))
+               kfree(inode->i_link);
        kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
 }