&& key1->both.offset == key2->both.offset);
}
- - if (!key->both.ptr)
+ /*
+ * Take a reference to the resource addressed by a key.
+ * Can be called while holding spinlocks.
+ *
+ */
+ static void get_futex_key_refs(union futex_key *key)
+ {
+ if (!key->both.ptr)
+ return;
+
+ switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
+ case FUT_OFF_INODE:
+ atomic_inc(&key->shared.inode->i_count);
+ break;
+ case FUT_OFF_MMSHARED:
+ atomic_inc(&key->private.mm->mm_count);
+ break;
+ }
+ }
+
+ /*
+ * Drop a reference to the resource addressed by a key.
+ * The hash bucket spinlock must not be held.
+ */
+ static void drop_futex_key_refs(union futex_key *key)
+ {
+ ++ if (!key->both.ptr) {
+ ++ /* If we're here then we tried to put a key we failed to get */
+ ++ WARN_ON_ONCE(1);
+ return;
+ ++ }
+
+ switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
+ case FUT_OFF_INODE:
+ iput(key->shared.inode);
+ break;
+ case FUT_OFF_MMSHARED:
+ mmdrop(key->private.mm);
+ break;
+ }
+ }
+
/**
* get_futex_key - Get parameters which are the keys for a futex.
* @uaddr: virtual address of the futex
}
spin_unlock(&hb->lock);
- -out:
+ put_futex_key(fshared, &key);
+ +out:
- futex_unlock_mm(fshared);
return ret;
}
*/
if (attempt++) {
ret = futex_handle_fault((unsigned long)uaddr2,
- fshared, attempt);
+ attempt);
if (ret)
- -- goto out;
+ ++ goto out_put_keys;
goto retry;
}
spin_unlock(&hb1->lock);
if (hb1 != hb2)
spin_unlock(&hb2->lock);
- -out:
+ ++out_put_keys:
+ put_futex_key(fshared, &key2);
+ ++out_put_key1:
+ put_futex_key(fshared, &key1);
- -
+ +out:
- futex_unlock_mm(fshared);
-
return ret;
}
struct futex_q *this, *next;
int ret, drop_count = 0;
- -- retry:
- futex_lock_mm(fshared);
-
+ ++retry:
ret = get_futex_key(uaddr1, fshared, &key1);
if (unlikely(ret != 0))
goto out;
while (--drop_count >= 0)
drop_futex_key_refs(&key1);
- -out:
+ ++out_put_keys:
+ put_futex_key(fshared, &key2);
+ ++out_put_key1:
+ put_futex_key(fshared, &key1);
+ +out:
- futex_unlock_mm(fshared);
return ret;
}
q.pi_state = NULL;
q.bitset = bitset;
- -- retry:
- futex_lock_mm(fshared);
-
+ ++retry:
+ q.key = FUTEX_KEY_INIT;
ret = get_futex_key(uaddr, fshared, &q.key);
if (unlikely(ret != 0))
- -- goto out_release_sem;
+ ++ goto out;
hb = queue_lock(&q);
if (unlikely(ret)) {
queue_unlock(&q, hb);
-
- /*
- * If we would have faulted, release mmap_sem, fault it in and
- * start all over again.
- */
- futex_unlock_mm(fshared);
+ ++ put_futex_key(fshared, &q.key);
ret = get_user(uval, uaddr);
return -ERESTART_RESTARTBLOCK;
}
- -- out_unlock_release_sem:
+ ++out_unlock_put_key:
queue_unlock(&q, hb);
- -
- - out_release_sem:
+ put_futex_key(fshared, &q.key);
+ +
- out_release_sem:
- futex_unlock_mm(fshared);
+ ++out:
return ret;
}
}
q.pi_state = NULL;
- -- retry:
- futex_lock_mm(fshared);
-
+ ++retry:
+ q.key = FUTEX_KEY_INIT;
ret = get_futex_key(uaddr, fshared, &q.key);
if (unlikely(ret != 0))
- -- goto out_release_sem;
+ ++ goto out;
- -- retry_unlocked:
+ ++retry_unlocked:
hb = queue_lock(&q);
- -- retry_locked:
+ ++retry_locked:
ret = lock_taken = 0;
/*
destroy_hrtimer_on_stack(&to->timer);
return ret != -EINTR ? ret : -ERESTARTNOINTR;
- -- out_unlock_release_sem:
+ ++out_unlock_put_key:
queue_unlock(&q, hb);
- -- out_release_sem:
- futex_unlock_mm(fshared);
+ ++out_put_key:
+ put_futex_key(fshared, &q.key);
+ ++out:
if (to)
destroy_hrtimer_on_stack(&to->timer);
return ret;
- -- uaddr_faulted:
+ ++uaddr_faulted:
/*
- * We have to r/w *(int __user *)uaddr, but we can't modify it
- * non-atomically. Therefore, if get_user below is not
- * enough, we need to handle the fault ourselves, while
- * still holding the mmap_sem.
- *
- * ... and hb->lock. :-) --ANK
+ * We have to r/w *(int __user *)uaddr, and we have to modify it
+ * atomically. Therefore, if we continue to fault after get_user()
+ * below, we need to handle the fault ourselves, while still holding
+ * the mmap_sem. This can occur if the uaddr is under contention as
+ * we have to drop the mmap_sem in order to call get_user().
*/
queue_unlock(&q, hb);
if (attempt++) {
- ret = futex_handle_fault((unsigned long)uaddr, fshared,
- attempt);
+ ret = futex_handle_fault((unsigned long)uaddr, attempt);
if (ret)
- -- goto out_release_sem;
+ ++ goto out_put_key;
goto retry_unlocked;
}
out_unlock:
spin_unlock(&hb->lock);
- --out:
- futex_unlock_mm(fshared);
+ put_futex_key(fshared, &key);
+ ++out:
return ret;
pi_faulted: