[GitHub/mt8127/android_kernel_alcatel_ttab.git] / mm / mmu_notifier.c

/*
 *  linux/mm/mmu_notifier.c
 *
 *  Copyright (C) 2008  Qumranet, Inc.
 *  Copyright (C) 2008  SGI
 *             Christoph Lameter <clameter@sgi.com>
 *
 *  This work is licensed under the terms of the GNU GPL, version 2. See
 *  the COPYING file in the top-level directory.
 */

#include <linux/rculist.h>
#include <linux/mmu_notifier.h>
#include <linux/export.h>
#include <linux/mm.h>
#include <linux/err.h>
#include <linux/srcu.h>
#include <linux/rcupdate.h>
#include <linux/sched.h>
#include <linux/slab.h>

/* global SRCU for all MMs */
static struct srcu_struct srcu;

/*
 * This function can't run concurrently against mmu_notifier_register
 * because mm->mm_users > 0 during mmu_notifier_register and exit_mmap
 * runs with mm_users == 0. Other tasks may still invoke mmu notifiers
 * in parallel despite there being no task using this mm any more,
 * through the vmas outside of the exit_mmap context, such as with
 * vmtruncate. This serializes against mmu_notifier_unregister with
 * the mmu_notifier_mm->lock in addition to SRCU and it serializes
 * against the other mmu notifiers with SRCU. struct mmu_notifier_mm
 * can't go away from under us as exit_mmap holds an mm_count pin
 * itself.
 */
void __mmu_notifier_release(struct mm_struct *mm)
{
	struct mmu_notifier *mn;
	int id;

	/*
	 * srcu_read_lock() here will block synchronize_srcu() in
	 * mmu_notifier_unregister() until all registered
	 * ->release() callouts this function makes have
	 * returned.
	 */
	id = srcu_read_lock(&srcu);
	spin_lock(&mm->mmu_notifier_mm->lock);
	while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) {
		mn = hlist_entry(mm->mmu_notifier_mm->list.first,
				 struct mmu_notifier,
				 hlist);

		/*
		 * Unlink.  This will prevent mmu_notifier_unregister()
		 * from also making the ->release() callout.
		 */
		hlist_del_init_rcu(&mn->hlist);
		spin_unlock(&mm->mmu_notifier_mm->lock);

		/*
		 * Clear sptes. (see 'release' description in mmu_notifier.h)
		 */
		if (mn->ops->release)
			mn->ops->release(mn, mm);

		spin_lock(&mm->mmu_notifier_mm->lock);
	}
	spin_unlock(&mm->mmu_notifier_mm->lock);

	/*
	 * All callouts to ->release() which we have done are complete.
	 * Allow synchronize_srcu() in mmu_notifier_unregister() to complete
	 */
	srcu_read_unlock(&srcu, id);

	/*
	 * mmu_notifier_unregister() may have unlinked a notifier and may
	 * still be calling out to it.	Additionally, other notifiers
	 * may have been active via vmtruncate() et. al. Block here
	 * to ensure that all notifier callouts for this mm have been
	 * completed and the sptes are really cleaned up before returning
	 * to exit_mmap().
	 */
	synchronize_srcu(&srcu);
}

/*
 * If no young bitflag is supported by the hardware, ->clear_flush_young can
 * unmap the address and return 1 or 0 depending if the mapping previously
 * existed or not.
 */
int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
					unsigned long address)
{
	struct mmu_notifier *mn;
	struct hlist_node *n;
	int young = 0, id;

	id = srcu_read_lock(&srcu);
	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
		if (mn->ops->clear_flush_young)
			young |= mn->ops->clear_flush_young(mn, mm, address);
	}
	srcu_read_unlock(&srcu, id);

	return young;
}

int __mmu_notifier_test_young(struct mm_struct *mm,
			      unsigned long address)
{
	struct mmu_notifier *mn;
	struct hlist_node *n;
	int young = 0, id;

	id = srcu_read_lock(&srcu);
	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
		if (mn->ops->test_young) {
			young = mn->ops->test_young(mn, mm, address);
			if (young)
				break;
		}
	}
	srcu_read_unlock(&srcu, id);

	return young;
}

void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,
			       pte_t pte)
{
	struct mmu_notifier *mn;
	struct hlist_node *n;
	int id;

	id = srcu_read_lock(&srcu);
	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
		if (mn->ops->change_pte)
			mn->ops->change_pte(mn, mm, address, pte);
	}
	srcu_read_unlock(&srcu, id);
}

void __mmu_notifier_invalidate_page(struct mm_struct *mm,
					  unsigned long address)
{
	struct mmu_notifier *mn;
	struct hlist_node *n;
	int id;

	id = srcu_read_lock(&srcu);
	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
		if (mn->ops->invalidate_page)
			mn->ops->invalidate_page(mn, mm, address);
	}
	srcu_read_unlock(&srcu, id);
}

void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
				  unsigned long start, unsigned long end)
{
	struct mmu_notifier *mn;
	struct hlist_node *n;
	int id;

	id = srcu_read_lock(&srcu);
	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
		if (mn->ops->invalidate_range_start)
			mn->ops->invalidate_range_start(mn, mm, start, end);
	}
	srcu_read_unlock(&srcu, id);
}
EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_start);

void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
				  unsigned long start, unsigned long end)
{
	struct mmu_notifier *mn;
	struct hlist_node *n;
	int id;

	id = srcu_read_lock(&srcu);
	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
		if (mn->ops->invalidate_range_end)
			mn->ops->invalidate_range_end(mn, mm, start, end);
	}
	srcu_read_unlock(&srcu, id);
}
EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end);

static int do_mmu_notifier_register(struct mmu_notifier *mn,
				    struct mm_struct *mm,
				    int take_mmap_sem)
{
	struct mmu_notifier_mm *mmu_notifier_mm;
	int ret;

	BUG_ON(atomic_read(&mm->mm_users) <= 0);

	/*
	 * Verify that mmu_notifier_init() already run and the global srcu is
	 * initialized.
	 */
	BUG_ON(!srcu.per_cpu_ref);

	ret = -ENOMEM;
	mmu_notifier_mm = kmalloc(sizeof(struct mmu_notifier_mm), GFP_KERNEL);
	if (unlikely(!mmu_notifier_mm))
		goto out;

	if (take_mmap_sem)
		down_write(&mm->mmap_sem);
	ret = mm_take_all_locks(mm);
	if (unlikely(ret))
		goto out_clean;

	if (!mm_has_notifiers(mm)) {
		INIT_HLIST_HEAD(&mmu_notifier_mm->list);
		spin_lock_init(&mmu_notifier_mm->lock);

		mm->mmu_notifier_mm = mmu_notifier_mm;
		mmu_notifier_mm = NULL;
	}
	atomic_inc(&mm->mm_count);

	/*
	 * Serialize the update against mmu_notifier_unregister. A
	 * side note: mmu_notifier_release can't run concurrently with
	 * us because we hold the mm_users pin (either implicitly as
	 * current->mm or explicitly with get_task_mm() or similar).
	 * We can't race against any other mmu notifier method either
	 * thanks to mm_take_all_locks().
	 */
	spin_lock(&mm->mmu_notifier_mm->lock);
	hlist_add_head(&mn->hlist, &mm->mmu_notifier_mm->list);
	spin_unlock(&mm->mmu_notifier_mm->lock);

	mm_drop_all_locks(mm);
out_clean:
	if (take_mmap_sem)
		up_write(&mm->mmap_sem);
	kfree(mmu_notifier_mm);
out:
	BUG_ON(atomic_read(&mm->mm_users) <= 0);
	return ret;
}

/*
 * Must not hold mmap_sem nor any other VM related lock when calling
 * this registration function. Must also ensure mm_users can't go down
 * to zero while this runs to avoid races with mmu_notifier_release,
 * so mm has to be current->mm or the mm should be pinned safely such
 * as with get_task_mm(). If the mm is not current->mm, the mm_users
 * pin should be released by calling mmput after mmu_notifier_register
 * returns. mmu_notifier_unregister must be always called to
 * unregister the notifier. mm_count is automatically pinned to allow
 * mmu_notifier_unregister to safely run at any time later, before or
 * after exit_mmap. ->release will always be called before exit_mmap
 * frees the pages.
 */
int mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)
{
	return do_mmu_notifier_register(mn, mm, 1);
}
EXPORT_SYMBOL_GPL(mmu_notifier_register);

/*
 * Same as mmu_notifier_register but here the caller must hold the
 * mmap_sem in write mode.
 */
int __mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)
{
	return do_mmu_notifier_register(mn, mm, 0);
}
EXPORT_SYMBOL_GPL(__mmu_notifier_register);

/* this is called after the last mmu_notifier_unregister() returned */
void __mmu_notifier_mm_destroy(struct mm_struct *mm)
{
	BUG_ON(!hlist_empty(&mm->mmu_notifier_mm->list));
	kfree(mm->mmu_notifier_mm);
	mm->mmu_notifier_mm = LIST_POISON1; /* debug */
}

/*
 * This releases the mm_count pin automatically and frees the mm
 * structure if it was the last user of it. It serializes against
 * running mmu notifiers with SRCU and against mmu_notifier_unregister
 * with the unregister lock + SRCU. All sptes must be dropped before
 * calling mmu_notifier_unregister. ->release or any other notifier
 * method may be invoked concurrently with mmu_notifier_unregister,
 * and only after mmu_notifier_unregister returned we're guaranteed
 * that ->release or any other method can't run anymore.
 */
void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm)
{
	BUG_ON(atomic_read(&mm->mm_count) <= 0);

	spin_lock(&mm->mmu_notifier_mm->lock);
	if (!hlist_unhashed(&mn->hlist)) {
		int id;

		/*
		 * Ensure we synchronize up with __mmu_notifier_release().
		 */
		id = srcu_read_lock(&srcu);

		hlist_del_rcu(&mn->hlist);
		spin_unlock(&mm->mmu_notifier_mm->lock);

		if (mn->ops->release)
			mn->ops->release(mn, mm);

		/*
		 * Allow __mmu_notifier_release() to complete.
		 */
		srcu_read_unlock(&srcu, id);
	} else
		spin_unlock(&mm->mmu_notifier_mm->lock);

	/*
	 * Wait for any running method to finish, including ->release() if it
	 * was run by __mmu_notifier_release() instead of us.
	 */
	synchronize_srcu(&srcu);

	BUG_ON(atomic_read(&mm->mm_count) <= 0);

	mmdrop(mm);
}
EXPORT_SYMBOL_GPL(mmu_notifier_unregister);

static int __init mmu_notifier_init(void)
{
	return init_srcu_struct(&srcu);
}

module_init(mmu_notifier_init);
Commit	Line	Data
cddb8a5c AA	1	/*
	2	* linux/mm/mmu_notifier.c
	3	*
	4	* Copyright (C) 2008 Qumranet, Inc.
	5	* Copyright (C) 2008 SGI
	6	* Christoph Lameter <clameter@sgi.com>
	7	*
	8	* This work is licensed under the terms of the GNU GPL, version 2. See
	9	* the COPYING file in the top-level directory.
	10	*/
	11
	12	#include <linux/rculist.h>
	13	#include <linux/mmu_notifier.h>
b95f1b31	14	#include <linux/export.h>
cddb8a5c AA	15	#include <linux/mm.h>
cddb8a5c AA	16	#include <linux/err.h>
21a92735	17	#include <linux/srcu.h>
cddb8a5c AA	18	#include <linux/rcupdate.h>
cddb8a5c AA	19	#include <linux/sched.h>
5a0e3ad6	20	#include <linux/slab.h>
cddb8a5c	21
21a92735	22	/* global SRCU for all MMs */
70400303	23	static struct srcu_struct srcu;
21a92735	24
cddb8a5c AA	25	/*
	26	* This function can't run concurrently against mmu_notifier_register
	27	* because mm->mm_users > 0 during mmu_notifier_register and exit_mmap
	28	* runs with mm_users == 0. Other tasks may still invoke mmu notifiers
	29	* in parallel despite there being no task using this mm any more,
	30	* through the vmas outside of the exit_mmap context, such as with
	31	* vmtruncate. This serializes against mmu_notifier_unregister with
21a92735 SG	32	* the mmu_notifier_mm->lock in addition to SRCU and it serializes
21a92735 SG	33	* against the other mmu notifiers with SRCU. struct mmu_notifier_mm
cddb8a5c AA	34	* can't go away from under us as exit_mmap holds an mm_count pin
	35	* itself.
	36	*/
	37	void __mmu_notifier_release(struct mm_struct *mm)
	38	{
	39	struct mmu_notifier *mn;
21a92735	40	int id;
3ad3d901 XG	41
3ad3d901 XG	42	/*
751efd86 RH	43	* srcu_read_lock() here will block synchronize_srcu() in
	44	* mmu_notifier_unregister() until all registered
	45	* ->release() callouts this function makes have
	46	* returned.
3ad3d901	47	*/
21a92735	48	id = srcu_read_lock(&srcu);
cddb8a5c AA	49	spin_lock(&mm->mmu_notifier_mm->lock);
	50	while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) {
	51	mn = hlist_entry(mm->mmu_notifier_mm->list.first,
	52	struct mmu_notifier,
	53	hlist);
751efd86	54
cddb8a5c	55	/*
751efd86 RH	56	* Unlink. This will prevent mmu_notifier_unregister()
751efd86 RH	57	* from also making the ->release() callout.
cddb8a5c AA	58	*/
cddb8a5c AA	59	hlist_del_init_rcu(&mn->hlist);
751efd86 RH	60	spin_unlock(&mm->mmu_notifier_mm->lock);
	61
	62	/*
	63	* Clear sptes. (see 'release' description in mmu_notifier.h)
	64	*/
	65	if (mn->ops->release)
	66	mn->ops->release(mn, mm);
	67
	68	spin_lock(&mm->mmu_notifier_mm->lock);
cddb8a5c AA	69	}
	70	spin_unlock(&mm->mmu_notifier_mm->lock);
	71
	72	/*
751efd86 RH	73	* All callouts to ->release() which we have done are complete.
	74	* Allow synchronize_srcu() in mmu_notifier_unregister() to complete
	75	*/
	76	srcu_read_unlock(&srcu, id);
	77
	78	/*
	79	* mmu_notifier_unregister() may have unlinked a notifier and may
	80	* still be calling out to it. Additionally, other notifiers
	81	* may have been active via vmtruncate() et. al. Block here
	82	* to ensure that all notifier callouts for this mm have been
	83	* completed and the sptes are really cleaned up before returning
	84	* to exit_mmap().
cddb8a5c	85	*/
21a92735	86	synchronize_srcu(&srcu);
cddb8a5c AA	87	}
	88
	89	/*
	90	* If no young bitflag is supported by the hardware, ->clear_flush_young can
	91	* unmap the address and return 1 or 0 depending if the mapping previously
	92	* existed or not.
	93	*/
	94	int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
	95	unsigned long address)
	96	{
	97	struct mmu_notifier *mn;
	98	struct hlist_node *n;
21a92735	99	int young = 0, id;
cddb8a5c	100
21a92735	101	id = srcu_read_lock(&srcu);
cddb8a5c AA	102	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
	103	if (mn->ops->clear_flush_young)
	104	young \|= mn->ops->clear_flush_young(mn, mm, address);
	105	}
21a92735	106	srcu_read_unlock(&srcu, id);
cddb8a5c AA	107
	108	return young;
	109	}
	110
8ee53820 AA	111	int __mmu_notifier_test_young(struct mm_struct *mm,
	112	unsigned long address)
	113	{
	114	struct mmu_notifier *mn;
	115	struct hlist_node *n;
21a92735	116	int young = 0, id;
8ee53820	117
21a92735	118	id = srcu_read_lock(&srcu);
8ee53820 AA	119	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
	120	if (mn->ops->test_young) {
	121	young = mn->ops->test_young(mn, mm, address);
	122	if (young)
	123	break;
	124	}
	125	}
21a92735	126	srcu_read_unlock(&srcu, id);
8ee53820 AA	127
	128	return young;
	129	}
	130
828502d3 IE	131	void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,
	132	pte_t pte)
	133	{
	134	struct mmu_notifier *mn;
	135	struct hlist_node *n;
21a92735	136	int id;
828502d3	137
21a92735	138	id = srcu_read_lock(&srcu);
828502d3 IE	139	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
	140	if (mn->ops->change_pte)
	141	mn->ops->change_pte(mn, mm, address, pte);
828502d3	142	}
21a92735	143	srcu_read_unlock(&srcu, id);
828502d3 IE	144	}
828502d3 IE	145
cddb8a5c AA	146	void __mmu_notifier_invalidate_page(struct mm_struct *mm,
	147	unsigned long address)
	148	{
	149	struct mmu_notifier *mn;
	150	struct hlist_node *n;
21a92735	151	int id;
cddb8a5c	152
21a92735	153	id = srcu_read_lock(&srcu);
cddb8a5c AA	154	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
	155	if (mn->ops->invalidate_page)
	156	mn->ops->invalidate_page(mn, mm, address);
	157	}
21a92735	158	srcu_read_unlock(&srcu, id);
cddb8a5c AA	159	}
	160
	161	void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
	162	unsigned long start, unsigned long end)
	163	{
	164	struct mmu_notifier *mn;
	165	struct hlist_node *n;
21a92735	166	int id;
cddb8a5c	167
21a92735	168	id = srcu_read_lock(&srcu);
cddb8a5c AA	169	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
	170	if (mn->ops->invalidate_range_start)
	171	mn->ops->invalidate_range_start(mn, mm, start, end);
	172	}
21a92735	173	srcu_read_unlock(&srcu, id);
cddb8a5c	174	}
fa794199	175	EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_start);
cddb8a5c AA	176
	177	void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
	178	unsigned long start, unsigned long end)
	179	{
	180	struct mmu_notifier *mn;
	181	struct hlist_node *n;
21a92735	182	int id;
cddb8a5c	183
21a92735	184	id = srcu_read_lock(&srcu);
cddb8a5c AA	185	hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
	186	if (mn->ops->invalidate_range_end)
	187	mn->ops->invalidate_range_end(mn, mm, start, end);
	188	}
21a92735	189	srcu_read_unlock(&srcu, id);
cddb8a5c	190	}
fa794199	191	EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end);
cddb8a5c AA	192
	193	static int do_mmu_notifier_register(struct mmu_notifier *mn,
	194	struct mm_struct *mm,
	195	int take_mmap_sem)
	196	{
	197	struct mmu_notifier_mm *mmu_notifier_mm;
	198	int ret;
	199
	200	BUG_ON(atomic_read(&mm->mm_users) <= 0);
	201
21a92735	202	/*
35cfa2b0 GS	203	* Verify that mmu_notifier_init() already run and the global srcu is
	204	* initialized.
	205	*/
21a92735 SG	206	BUG_ON(!srcu.per_cpu_ref);
21a92735 SG	207
35cfa2b0 GS	208	ret = -ENOMEM;
	209	mmu_notifier_mm = kmalloc(sizeof(struct mmu_notifier_mm), GFP_KERNEL);
	210	if (unlikely(!mmu_notifier_mm))
	211	goto out;
	212
cddb8a5c AA	213	if (take_mmap_sem)
	214	down_write(&mm->mmap_sem);
	215	ret = mm_take_all_locks(mm);
	216	if (unlikely(ret))
35cfa2b0	217	goto out_clean;
cddb8a5c AA	218
	219	if (!mm_has_notifiers(mm)) {
	220	INIT_HLIST_HEAD(&mmu_notifier_mm->list);
	221	spin_lock_init(&mmu_notifier_mm->lock);
e0f3c3f7	222
cddb8a5c	223	mm->mmu_notifier_mm = mmu_notifier_mm;
35cfa2b0	224	mmu_notifier_mm = NULL;
cddb8a5c AA	225	}
	226	atomic_inc(&mm->mm_count);
	227
	228	/*
	229	* Serialize the update against mmu_notifier_unregister. A
	230	* side note: mmu_notifier_release can't run concurrently with
	231	* us because we hold the mm_users pin (either implicitly as
	232	* current->mm or explicitly with get_task_mm() or similar).
	233	* We can't race against any other mmu notifier method either
	234	* thanks to mm_take_all_locks().
	235	*/
	236	spin_lock(&mm->mmu_notifier_mm->lock);
	237	hlist_add_head(&mn->hlist, &mm->mmu_notifier_mm->list);
	238	spin_unlock(&mm->mmu_notifier_mm->lock);
	239
	240	mm_drop_all_locks(mm);
35cfa2b0	241	out_clean:
cddb8a5c AA	242	if (take_mmap_sem)
cddb8a5c AA	243	up_write(&mm->mmap_sem);
35cfa2b0 GS	244	kfree(mmu_notifier_mm);
35cfa2b0 GS	245	out:
cddb8a5c AA	246	BUG_ON(atomic_read(&mm->mm_users) <= 0);
	247	return ret;
	248	}
	249
	250	/*
	251	* Must not hold mmap_sem nor any other VM related lock when calling
	252	* this registration function. Must also ensure mm_users can't go down
	253	* to zero while this runs to avoid races with mmu_notifier_release,
	254	* so mm has to be current->mm or the mm should be pinned safely such
	255	* as with get_task_mm(). If the mm is not current->mm, the mm_users
	256	* pin should be released by calling mmput after mmu_notifier_register
	257	* returns. mmu_notifier_unregister must be always called to
	258	* unregister the notifier. mm_count is automatically pinned to allow
	259	* mmu_notifier_unregister to safely run at any time later, before or
	260	* after exit_mmap. ->release will always be called before exit_mmap
	261	* frees the pages.
	262	*/
	263	int mmu_notifier_register(struct mmu_notifier mn, struct mm_struct mm)
	264	{
	265	return do_mmu_notifier_register(mn, mm, 1);
	266	}
	267	EXPORT_SYMBOL_GPL(mmu_notifier_register);
	268
	269	/*
	270	* Same as mmu_notifier_register but here the caller must hold the
	271	* mmap_sem in write mode.
	272	*/
	273	int __mmu_notifier_register(struct mmu_notifier mn, struct mm_struct mm)
	274	{
	275	return do_mmu_notifier_register(mn, mm, 0);
	276	}
	277	EXPORT_SYMBOL_GPL(__mmu_notifier_register);
	278
	279	/* this is called after the last mmu_notifier_unregister() returned */
	280	void __mmu_notifier_mm_destroy(struct mm_struct *mm)
	281	{
	282	BUG_ON(!hlist_empty(&mm->mmu_notifier_mm->list));
	283	kfree(mm->mmu_notifier_mm);
	284	mm->mmu_notifier_mm = LIST_POISON1; /* debug */
	285	}
	286
	287	/*
	288	* This releases the mm_count pin automatically and frees the mm
	289	* structure if it was the last user of it. It serializes against
21a92735 SG	290	* running mmu notifiers with SRCU and against mmu_notifier_unregister
21a92735 SG	291	* with the unregister lock + SRCU. All sptes must be dropped before
cddb8a5c AA	292	* calling mmu_notifier_unregister. ->release or any other notifier
	293	* method may be invoked concurrently with mmu_notifier_unregister,
	294	* and only after mmu_notifier_unregister returned we're guaranteed
	295	* that ->release or any other method can't run anymore.
	296	*/
	297	void mmu_notifier_unregister(struct mmu_notifier mn, struct mm_struct mm)
	298	{
	299	BUG_ON(atomic_read(&mm->mm_count) <= 0);
	300
751efd86	301	spin_lock(&mm->mmu_notifier_mm->lock);
cddb8a5c	302	if (!hlist_unhashed(&mn->hlist)) {
21a92735	303	int id;
3ad3d901	304
cddb8a5c	305	/*
751efd86	306	* Ensure we synchronize up with __mmu_notifier_release().
cddb8a5c	307	*/
751efd86 RH	308	id = srcu_read_lock(&srcu);
	309
	310	hlist_del_rcu(&mn->hlist);
	311	spin_unlock(&mm->mmu_notifier_mm->lock);
	312
cddb8a5c AA	313	if (mn->ops->release)
cddb8a5c AA	314	mn->ops->release(mn, mm);
3ad3d901	315
751efd86 RH	316	/*
	317	* Allow __mmu_notifier_release() to complete.
	318	*/
	319	srcu_read_unlock(&srcu, id);
	320	} else
cddb8a5c AA	321	spin_unlock(&mm->mmu_notifier_mm->lock);
	322
	323	/*
751efd86 RH	324	* Wait for any running method to finish, including ->release() if it
751efd86 RH	325	* was run by __mmu_notifier_release() instead of us.
cddb8a5c	326	*/
21a92735	327	synchronize_srcu(&srcu);
cddb8a5c AA	328
	329	BUG_ON(atomic_read(&mm->mm_count) <= 0);
	330
	331	mmdrop(mm);
	332	}
	333	EXPORT_SYMBOL_GPL(mmu_notifier_unregister);
21a92735 SG	334
	335	static int __init mmu_notifier_init(void)
	336	{
	337	return init_srcu_struct(&srcu);
	338	}
	339
	340	module_init(mmu_notifier_init);