[GitHub/mt8127/android_kernel_alcatel_ttab.git] / mm / page_cgroup.c

#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/bootmem.h>
#include <linux/bit_spinlock.h>
#include <linux/page_cgroup.h>
#include <linux/hash.h>
#include <linux/slab.h>
#include <linux/memory.h>
#include <linux/vmalloc.h>
#include <linux/cgroup.h>
#include <linux/swapops.h>
#include <linux/kmemleak.h>

static void __meminit init_page_cgroup(struct page_cgroup *pc, unsigned long id)
{
	pc->flags = 0;
	set_page_cgroup_array_id(pc, id);
	pc->mem_cgroup = NULL;
	INIT_LIST_HEAD(&pc->lru);
}
static unsigned long total_usage;

#if !defined(CONFIG_SPARSEMEM)


void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
{
	pgdat->node_page_cgroup = NULL;
}

struct page_cgroup *lookup_page_cgroup(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);
	unsigned long offset;
	struct page_cgroup *base;

	base = NODE_DATA(page_to_nid(page))->node_page_cgroup;
	if (unlikely(!base))
		return NULL;

	offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn;
	return base + offset;
}

struct page *lookup_cgroup_page(struct page_cgroup *pc)
{
	unsigned long pfn;
	struct page *page;
	pg_data_t *pgdat;

	pgdat = NODE_DATA(page_cgroup_array_id(pc));
	pfn = pc - pgdat->node_page_cgroup + pgdat->node_start_pfn;
	page = pfn_to_page(pfn);
	VM_BUG_ON(pc != lookup_page_cgroup(page));
	return page;
}

static int __init alloc_node_page_cgroup(int nid)
{
	struct page_cgroup *base, *pc;
	unsigned long table_size;
	unsigned long start_pfn, nr_pages, index;

	start_pfn = NODE_DATA(nid)->node_start_pfn;
	nr_pages = NODE_DATA(nid)->node_spanned_pages;

	if (!nr_pages)
		return 0;

	table_size = sizeof(struct page_cgroup) * nr_pages;

	base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
			table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
	if (!base)
		return -ENOMEM;
	for (index = 0; index < nr_pages; index++) {
		pc = base + index;
		init_page_cgroup(pc, nid);
	}
	NODE_DATA(nid)->node_page_cgroup = base;
	total_usage += table_size;
	return 0;
}

void __init page_cgroup_init_flatmem(void)
{

	int nid, fail;

	if (mem_cgroup_disabled())
		return;

	for_each_online_node(nid)  {
		fail = alloc_node_page_cgroup(nid);
		if (fail)
			goto fail;
	}
	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you"
	" don't want memory cgroups\n");
	return;
fail:
	printk(KERN_CRIT "allocation of page_cgroup failed.\n");
	printk(KERN_CRIT "please try 'cgroup_disable=memory' boot option\n");
	panic("Out of memory");
}

#else /* CONFIG_FLAT_NODE_MEM_MAP */

struct page_cgroup *lookup_page_cgroup(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);
	struct mem_section *section = __pfn_to_section(pfn);

	if (!section->page_cgroup)
		return NULL;
	return section->page_cgroup + pfn;
}

struct page *lookup_cgroup_page(struct page_cgroup *pc)
{
	struct mem_section *section;
	struct page *page;
	unsigned long nr;

	nr = page_cgroup_array_id(pc);
	section = __nr_to_section(nr);
	page = pfn_to_page(pc - section->page_cgroup);
	VM_BUG_ON(pc != lookup_page_cgroup(page));
	return page;
}

static void *__meminit alloc_page_cgroup(size_t size, int nid)
{
	void *addr = NULL;

	addr = alloc_pages_exact_nid(nid, size, GFP_KERNEL | __GFP_NOWARN);
	if (addr)
		return addr;

	if (node_state(nid, N_HIGH_MEMORY))
		addr = vmalloc_node(size, nid);
	else
		addr = vmalloc(size);

	return addr;
}

#ifdef CONFIG_MEMORY_HOTPLUG
static void free_page_cgroup(void *addr)
{
	if (is_vmalloc_addr(addr)) {
		vfree(addr);
	} else {
		struct page *page = virt_to_page(addr);
		size_t table_size =
			sizeof(struct page_cgroup) * PAGES_PER_SECTION;

		BUG_ON(PageReserved(page));
		free_pages_exact(addr, table_size);
	}
}
#endif

static int __meminit init_section_page_cgroup(unsigned long pfn, int nid)
{
	struct page_cgroup *base, *pc;
	struct mem_section *section;
	unsigned long table_size;
	unsigned long nr;
	int index;

	nr = pfn_to_section_nr(pfn);
	section = __nr_to_section(nr);

	if (section->page_cgroup)
		return 0;

	table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
	base = alloc_page_cgroup(table_size, nid);

	/*
	 * The value stored in section->page_cgroup is (base - pfn)
	 * and it does not point to the memory block allocated above,
	 * causing kmemleak false positives.
	 */
	kmemleak_not_leak(base);

	if (!base) {
		printk(KERN_ERR "page cgroup allocation failure\n");
		return -ENOMEM;
	}

	for (index = 0; index < PAGES_PER_SECTION; index++) {
		pc = base + index;
		init_page_cgroup(pc, nr);
	}
	/*
	 * The passed "pfn" may not be aligned to SECTION.  For the calculation
	 * we need to apply a mask.
	 */
	pfn &= PAGE_SECTION_MASK;
	section->page_cgroup = base - pfn;
	total_usage += table_size;
	return 0;
}
#ifdef CONFIG_MEMORY_HOTPLUG
void __free_page_cgroup(unsigned long pfn)
{
	struct mem_section *ms;
	struct page_cgroup *base;

	ms = __pfn_to_section(pfn);
	if (!ms || !ms->page_cgroup)
		return;
	base = ms->page_cgroup + pfn;
	free_page_cgroup(base);
	ms->page_cgroup = NULL;
}

int __meminit online_page_cgroup(unsigned long start_pfn,
			unsigned long nr_pages,
			int nid)
{
	unsigned long start, end, pfn;
	int fail = 0;

	start = start_pfn & ~(PAGES_PER_SECTION - 1);
	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);

	if (nid == -1) {
		/*
		 * In this case, "nid" already exists and contains valid memory.
		 * "start_pfn" passed to us is a pfn which is an arg for
		 * online__pages(), and start_pfn should exist.
		 */
		nid = pfn_to_nid(start_pfn);
		VM_BUG_ON(!node_state(nid, N_ONLINE));
	}

	for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) {
		if (!pfn_present(pfn))
			continue;
		fail = init_section_page_cgroup(pfn, nid);
	}
	if (!fail)
		return 0;

	/* rollback */
	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
		__free_page_cgroup(pfn);

	return -ENOMEM;
}

int __meminit offline_page_cgroup(unsigned long start_pfn,
		unsigned long nr_pages, int nid)
{
	unsigned long start, end, pfn;

	start = start_pfn & ~(PAGES_PER_SECTION - 1);
	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);

	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
		__free_page_cgroup(pfn);
	return 0;

}

static int __meminit page_cgroup_callback(struct notifier_block *self,
			       unsigned long action, void *arg)
{
	struct memory_notify *mn = arg;
	int ret = 0;
	switch (action) {
	case MEM_GOING_ONLINE:
		ret = online_page_cgroup(mn->start_pfn,
				   mn->nr_pages, mn->status_change_nid);
		break;
	case MEM_OFFLINE:
		offline_page_cgroup(mn->start_pfn,
				mn->nr_pages, mn->status_change_nid);
		break;
	case MEM_CANCEL_ONLINE:
	case MEM_GOING_OFFLINE:
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}

	return notifier_from_errno(ret);
}

#endif

void __init page_cgroup_init(void)
{
	unsigned long pfn;
	int nid;

	if (mem_cgroup_disabled())
		return;

	for_each_node_state(nid, N_HIGH_MEMORY) {
		unsigned long start_pfn, end_pfn;

		start_pfn = node_start_pfn(nid);
		end_pfn = node_end_pfn(nid);
		/*
		 * start_pfn and end_pfn may not be aligned to SECTION and the
		 * page->flags of out of node pages are not initialized.  So we
		 * scan [start_pfn, the biggest section's pfn < end_pfn) here.
		 */
		for (pfn = start_pfn;
		     pfn < end_pfn;
                     pfn = ALIGN(pfn + 1, PAGES_PER_SECTION)) {

			if (!pfn_valid(pfn))
				continue;
			/*
			 * Nodes's pfns can be overlapping.
			 * We know some arch can have a nodes layout such as
			 * -------------pfn-------------->
			 * N0 | N1 | N2 | N0 | N1 | N2|....
			 */
			if (pfn_to_nid(pfn) != nid)
				continue;
			if (init_section_page_cgroup(pfn, nid))
				goto oom;
		}
	}
	hotplug_memory_notifier(page_cgroup_callback, 0);
	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you "
			 "don't want memory cgroups\n");
	return;
oom:
	printk(KERN_CRIT "try 'cgroup_disable=memory' boot option\n");
	panic("Out of memory");
}

void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
{
	return;
}

#endif


#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP

static DEFINE_MUTEX(swap_cgroup_mutex);
struct swap_cgroup_ctrl {
	struct page **map;
	unsigned long length;
	spinlock_t	lock;
};

struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];

struct swap_cgroup {
	unsigned short		id;
};
#define SC_PER_PAGE	(PAGE_SIZE/sizeof(struct swap_cgroup))
#define SC_POS_MASK	(SC_PER_PAGE - 1)

/*
 * SwapCgroup implements "lookup" and "exchange" operations.
 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
 * against SwapCache. At swap_free(), this is accessed directly from swap.
 *
 * This means,
 *  - we have no race in "exchange" when we're accessed via SwapCache because
 *    SwapCache(and its swp_entry) is under lock.
 *  - When called via swap_free(), there is no user of this entry and no race.
 * Then, we don't need lock around "exchange".
 *
 * TODO: we can push these buffers out to HIGHMEM.
 */

/*
 * allocate buffer for swap_cgroup.
 */
static int swap_cgroup_prepare(int type)
{
	struct page *page;
	struct swap_cgroup_ctrl *ctrl;
	unsigned long idx, max;

	ctrl = &swap_cgroup_ctrl[type];

	for (idx = 0; idx < ctrl->length; idx++) {
		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
		if (!page)
			goto not_enough_page;
		ctrl->map[idx] = page;
	}
	return 0;
not_enough_page:
	max = idx;
	for (idx = 0; idx < max; idx++)
		__free_page(ctrl->map[idx]);

	return -ENOMEM;
}

/**
 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
 * @end: swap entry to be cmpxchged
 * @old: old id
 * @new: new id
 *
 * Returns old id at success, 0 at failure.
 * (There is no mem_cgroup using 0 as its id)
 */
unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
					unsigned short old, unsigned short new)
{
	int type = swp_type(ent);
	unsigned long offset = swp_offset(ent);
	unsigned long idx = offset / SC_PER_PAGE;
	unsigned long pos = offset & SC_POS_MASK;
	struct swap_cgroup_ctrl *ctrl;
	struct page *mappage;
	struct swap_cgroup *sc;
	unsigned long flags;
	unsigned short retval;

	ctrl = &swap_cgroup_ctrl[type];

	mappage = ctrl->map[idx];
	sc = page_address(mappage);
	sc += pos;
	spin_lock_irqsave(&ctrl->lock, flags);
	retval = sc->id;
	if (retval == old)
		sc->id = new;
	else
		retval = 0;
	spin_unlock_irqrestore(&ctrl->lock, flags);
	return retval;
}

/**
 * swap_cgroup_record - record mem_cgroup for this swp_entry.
 * @ent: swap entry to be recorded into
 * @mem: mem_cgroup to be recorded
 *
 * Returns old value at success, 0 at failure.
 * (Of course, old value can be 0.)
 */
unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
{
	int type = swp_type(ent);
	unsigned long offset = swp_offset(ent);
	unsigned long idx = offset / SC_PER_PAGE;
	unsigned long pos = offset & SC_POS_MASK;
	struct swap_cgroup_ctrl *ctrl;
	struct page *mappage;
	struct swap_cgroup *sc;
	unsigned short old;
	unsigned long flags;

	ctrl = &swap_cgroup_ctrl[type];

	mappage = ctrl->map[idx];
	sc = page_address(mappage);
	sc += pos;
	spin_lock_irqsave(&ctrl->lock, flags);
	old = sc->id;
	sc->id = id;
	spin_unlock_irqrestore(&ctrl->lock, flags);

	return old;
}

/**
 * lookup_swap_cgroup - lookup mem_cgroup tied to swap entry
 * @ent: swap entry to be looked up.
 *
 * Returns CSS ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
 */
unsigned short lookup_swap_cgroup(swp_entry_t ent)
{
	int type = swp_type(ent);
	unsigned long offset = swp_offset(ent);
	unsigned long idx = offset / SC_PER_PAGE;
	unsigned long pos = offset & SC_POS_MASK;
	struct swap_cgroup_ctrl *ctrl;
	struct page *mappage;
	struct swap_cgroup *sc;
	unsigned short ret;

	ctrl = &swap_cgroup_ctrl[type];
	mappage = ctrl->map[idx];
	sc = page_address(mappage);
	sc += pos;
	ret = sc->id;
	return ret;
}

int swap_cgroup_swapon(int type, unsigned long max_pages)
{
	void *array;
	unsigned long array_size;
	unsigned long length;
	struct swap_cgroup_ctrl *ctrl;

	if (!do_swap_account)
		return 0;

	length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
	array_size = length * sizeof(void *);

	array = vmalloc(array_size);
	if (!array)
		goto nomem;

	memset(array, 0, array_size);
	ctrl = &swap_cgroup_ctrl[type];
	mutex_lock(&swap_cgroup_mutex);
	ctrl->length = length;
	ctrl->map = array;
	spin_lock_init(&ctrl->lock);
	if (swap_cgroup_prepare(type)) {
		/* memory shortage */
		ctrl->map = NULL;
		ctrl->length = 0;
		mutex_unlock(&swap_cgroup_mutex);
		vfree(array);
		goto nomem;
	}
	mutex_unlock(&swap_cgroup_mutex);

	return 0;
nomem:
	printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
	printk(KERN_INFO
		"swap_cgroup can be disabled by noswapaccount boot option\n");
	return -ENOMEM;
}

void swap_cgroup_swapoff(int type)
{
	struct page **map;
	unsigned long i, length;
	struct swap_cgroup_ctrl *ctrl;

	if (!do_swap_account)
		return;

	mutex_lock(&swap_cgroup_mutex);
	ctrl = &swap_cgroup_ctrl[type];
	map = ctrl->map;
	length = ctrl->length;
	ctrl->map = NULL;
	ctrl->length = 0;
	mutex_unlock(&swap_cgroup_mutex);

	if (map) {
		for (i = 0; i < length; i++) {
			struct page *page = map[i];
			if (page)
				__free_page(page);
		}
		vfree(map);
	}
}

#endif
Commit	Line	Data
52d4b9ac KH	1	#include <linux/mm.h>
	2	#include <linux/mmzone.h>
	3	#include <linux/bootmem.h>
	4	#include <linux/bit_spinlock.h>
	5	#include <linux/page_cgroup.h>
	6	#include <linux/hash.h>
94b6da5a	7	#include <linux/slab.h>
52d4b9ac	8	#include <linux/memory.h>
4c821042	9	#include <linux/vmalloc.h>
94b6da5a	10	#include <linux/cgroup.h>
27a7faa0	11	#include <linux/swapops.h>
7952f988	12	#include <linux/kmemleak.h>
52d4b9ac	13
6b3ae58e	14	static void __meminit init_page_cgroup(struct page_cgroup *pc, unsigned long id)
52d4b9ac KH	15	{
52d4b9ac KH	16	pc->flags = 0;
6b3ae58e	17	set_page_cgroup_array_id(pc, id);
52d4b9ac	18	pc->mem_cgroup = NULL;
08e552c6	19	INIT_LIST_HEAD(&pc->lru);
52d4b9ac KH	20	}
	21	static unsigned long total_usage;
	22
	23	#if !defined(CONFIG_SPARSEMEM)
	24
	25
31168481	26	void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
52d4b9ac KH	27	{
	28	pgdat->node_page_cgroup = NULL;
	29	}
	30
	31	struct page_cgroup lookup_page_cgroup(struct page page)
	32	{
	33	unsigned long pfn = page_to_pfn(page);
	34	unsigned long offset;
	35	struct page_cgroup *base;
	36
	37	base = NODE_DATA(page_to_nid(page))->node_page_cgroup;
	38	if (unlikely(!base))
	39	return NULL;
	40
	41	offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn;
	42	return base + offset;
	43	}
	44
6b3ae58e JW	45	struct page lookup_cgroup_page(struct page_cgroup pc)
	46	{
	47	unsigned long pfn;
	48	struct page *page;
	49	pg_data_t *pgdat;
	50
	51	pgdat = NODE_DATA(page_cgroup_array_id(pc));
	52	pfn = pc - pgdat->node_page_cgroup + pgdat->node_start_pfn;
	53	page = pfn_to_page(pfn);
	54	VM_BUG_ON(pc != lookup_page_cgroup(page));
	55	return page;
	56	}
	57
52d4b9ac KH	58	static int __init alloc_node_page_cgroup(int nid)
	59	{
	60	struct page_cgroup base, pc;
	61	unsigned long table_size;
	62	unsigned long start_pfn, nr_pages, index;
	63
	64	start_pfn = NODE_DATA(nid)->node_start_pfn;
	65	nr_pages = NODE_DATA(nid)->node_spanned_pages;
	66
653d22c0 KH	67	if (!nr_pages)
	68	return 0;
	69
52d4b9ac	70	table_size = sizeof(struct page_cgroup) * nr_pages;
ca371c0d KH	71
	72	base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
	73	table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
	74	if (!base)
52d4b9ac KH	75	return -ENOMEM;
	76	for (index = 0; index < nr_pages; index++) {
	77	pc = base + index;
6b3ae58e	78	init_page_cgroup(pc, nid);
52d4b9ac KH	79	}
	80	NODE_DATA(nid)->node_page_cgroup = base;
	81	total_usage += table_size;
	82	return 0;
	83	}
	84
ca371c0d	85	void __init page_cgroup_init_flatmem(void)
52d4b9ac KH	86	{
	87
	88	int nid, fail;
	89
f8d66542	90	if (mem_cgroup_disabled())
94b6da5a KH	91	return;
94b6da5a KH	92
52d4b9ac KH	93	for_each_online_node(nid) {
	94	fail = alloc_node_page_cgroup(nid);
	95	if (fail)
	96	goto fail;
	97	}
	98	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
8ca739e3 RD	99	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you"
8ca739e3 RD	100	" don't want memory cgroups\n");
52d4b9ac KH	101	return;
52d4b9ac KH	102	fail:
8ca739e3 RD	103	printk(KERN_CRIT "allocation of page_cgroup failed.\n");
8ca739e3 RD	104	printk(KERN_CRIT "please try 'cgroup_disable=memory' boot option\n");
52d4b9ac KH	105	panic("Out of memory");
	106	}
	107
	108	#else /* CONFIG_FLAT_NODE_MEM_MAP */
	109
	110	struct page_cgroup lookup_page_cgroup(struct page page)
	111	{
	112	unsigned long pfn = page_to_pfn(page);
	113	struct mem_section *section = __pfn_to_section(pfn);
	114
d69b042f BS	115	if (!section->page_cgroup)
d69b042f BS	116	return NULL;
52d4b9ac KH	117	return section->page_cgroup + pfn;
	118	}
	119
6b3ae58e JW	120	struct page lookup_cgroup_page(struct page_cgroup pc)
	121	{
	122	struct mem_section *section;
	123	struct page *page;
	124	unsigned long nr;
	125
	126	nr = page_cgroup_array_id(pc);
	127	section = __nr_to_section(nr);
	128	page = pfn_to_page(pc - section->page_cgroup);
	129	VM_BUG_ON(pc != lookup_page_cgroup(page));
	130	return page;
	131	}
	132
268433b8	133	static void *__meminit alloc_page_cgroup(size_t size, int nid)
dde79e00 MH	134	{
	135	void *addr = NULL;
	136
21a3c964	137	addr = alloc_pages_exact_nid(nid, size, GFP_KERNEL \| __GFP_NOWARN);
dde79e00 MH	138	if (addr)
	139	return addr;
	140
	141	if (node_state(nid, N_HIGH_MEMORY))
	142	addr = vmalloc_node(size, nid);
	143	else
	144	addr = vmalloc(size);
	145
	146	return addr;
	147	}
	148
	149	#ifdef CONFIG_MEMORY_HOTPLUG
	150	static void free_page_cgroup(void *addr)
	151	{
	152	if (is_vmalloc_addr(addr)) {
	153	vfree(addr);
	154	} else {
	155	struct page *page = virt_to_page(addr);
6cfddb26 MH	156	size_t table_size =
	157	sizeof(struct page_cgroup) * PAGES_PER_SECTION;
	158
	159	BUG_ON(PageReserved(page));
	160	free_pages_exact(addr, table_size);
dde79e00 MH	161	}
	162	}
	163	#endif
	164
37573e8c	165	static int __meminit init_section_page_cgroup(unsigned long pfn, int nid)
52d4b9ac	166	{
52d4b9ac	167	struct page_cgroup base, pc;
6b3ae58e	168	struct mem_section *section;
52d4b9ac	169	unsigned long table_size;
6b3ae58e	170	unsigned long nr;
37573e8c	171	int index;
52d4b9ac	172
6b3ae58e JW	173	nr = pfn_to_section_nr(pfn);
	174	section = __nr_to_section(nr);
	175
	176	if (section->page_cgroup)
	177	return 0;
	178
6b3ae58e	179	table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
dde79e00 MH	180	base = alloc_page_cgroup(table_size, nid);
dde79e00 MH	181
6b3ae58e JW	182	/*
	183	* The value stored in section->page_cgroup is (base - pfn)
	184	* and it does not point to the memory block allocated above,
	185	* causing kmemleak false positives.
	186	*/
	187	kmemleak_not_leak(base);
52d4b9ac KH	188
	189	if (!base) {
	190	printk(KERN_ERR "page cgroup allocation failure\n");
	191	return -ENOMEM;
	192	}
	193
	194	for (index = 0; index < PAGES_PER_SECTION; index++) {
	195	pc = base + index;
6b3ae58e	196	init_page_cgroup(pc, nr);
52d4b9ac	197	}
37573e8c KH	198	/*
	199	* The passed "pfn" may not be aligned to SECTION. For the calculation
	200	* we need to apply a mask.
	201	*/
	202	pfn &= PAGE_SECTION_MASK;
52d4b9ac KH	203	section->page_cgroup = base - pfn;
	204	total_usage += table_size;
	205	return 0;
	206	}
	207	#ifdef CONFIG_MEMORY_HOTPLUG
	208	void __free_page_cgroup(unsigned long pfn)
	209	{
	210	struct mem_section *ms;
	211	struct page_cgroup *base;
	212
	213	ms = __pfn_to_section(pfn);
	214	if (!ms \|\| !ms->page_cgroup)
	215	return;
	216	base = ms->page_cgroup + pfn;
dde79e00 MH	217	free_page_cgroup(base);
dde79e00 MH	218	ms->page_cgroup = NULL;
52d4b9ac KH	219	}
52d4b9ac KH	220
31168481	221	int __meminit online_page_cgroup(unsigned long start_pfn,
52d4b9ac KH	222	unsigned long nr_pages,
	223	int nid)
	224	{
	225	unsigned long start, end, pfn;
	226	int fail = 0;
	227
33c5d3d6	228	start = start_pfn & ~(PAGES_PER_SECTION - 1);
52d4b9ac KH	229	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
52d4b9ac KH	230
37573e8c KH	231	if (nid == -1) {
	232	/*
	233	* In this case, "nid" already exists and contains valid memory.
	234	* "start_pfn" passed to us is a pfn which is an arg for
	235	* online__pages(), and start_pfn should exist.
	236	*/
	237	nid = pfn_to_nid(start_pfn);
	238	VM_BUG_ON(!node_state(nid, N_ONLINE));
	239	}
	240
52d4b9ac KH	241	for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) {
	242	if (!pfn_present(pfn))
	243	continue;
37573e8c	244	fail = init_section_page_cgroup(pfn, nid);
52d4b9ac KH	245	}
	246	if (!fail)
	247	return 0;
	248
	249	/* rollback */
	250	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
	251	__free_page_cgroup(pfn);
	252
	253	return -ENOMEM;
	254	}
	255
31168481	256	int __meminit offline_page_cgroup(unsigned long start_pfn,
52d4b9ac KH	257	unsigned long nr_pages, int nid)
	258	{
	259	unsigned long start, end, pfn;
	260
33c5d3d6	261	start = start_pfn & ~(PAGES_PER_SECTION - 1);
52d4b9ac KH	262	end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
	263
	264	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
	265	__free_page_cgroup(pfn);
	266	return 0;
	267
	268	}
	269
31168481	270	static int __meminit page_cgroup_callback(struct notifier_block *self,
52d4b9ac KH	271	unsigned long action, void *arg)
	272	{
	273	struct memory_notify *mn = arg;
	274	int ret = 0;
	275	switch (action) {
	276	case MEM_GOING_ONLINE:
	277	ret = online_page_cgroup(mn->start_pfn,
	278	mn->nr_pages, mn->status_change_nid);
	279	break;
52d4b9ac KH	280	case MEM_OFFLINE:
	281	offline_page_cgroup(mn->start_pfn,
	282	mn->nr_pages, mn->status_change_nid);
	283	break;
dc19f9db	284	case MEM_CANCEL_ONLINE:
52d4b9ac KH	285	case MEM_GOING_OFFLINE:
	286	break;
	287	case MEM_ONLINE:
	288	case MEM_CANCEL_OFFLINE:
	289	break;
	290	}
dc19f9db	291
5fda1bd5	292	return notifier_from_errno(ret);
52d4b9ac KH	293	}
	294
	295	#endif
	296
	297	void __init page_cgroup_init(void)
	298	{
	299	unsigned long pfn;
37573e8c	300	int nid;
52d4b9ac	301
f8d66542	302	if (mem_cgroup_disabled())
94b6da5a KH	303	return;
94b6da5a KH	304
37573e8c KH	305	for_each_node_state(nid, N_HIGH_MEMORY) {
	306	unsigned long start_pfn, end_pfn;
	307
	308	start_pfn = node_start_pfn(nid);
	309	end_pfn = node_end_pfn(nid);
	310	/*
	311	* start_pfn and end_pfn may not be aligned to SECTION and the
	312	* page->flags of out of node pages are not initialized. So we
	313	* scan [start_pfn, the biggest section's pfn < end_pfn) here.
	314	*/
	315	for (pfn = start_pfn;
	316	pfn < end_pfn;
	317	pfn = ALIGN(pfn + 1, PAGES_PER_SECTION)) {
	318
	319	if (!pfn_valid(pfn))
	320	continue;
	321	/*
	322	* Nodes's pfns can be overlapping.
	323	* We know some arch can have a nodes layout such as
	324	* -------------pfn-------------->
	325	* N0 \| N1 \| N2 \| N0 \| N1 \| N2\|....
	326	*/
	327	if (pfn_to_nid(pfn) != nid)
	328	continue;
	329	if (init_section_page_cgroup(pfn, nid))
	330	goto oom;
	331	}
52d4b9ac	332	}
37573e8c	333	hotplug_memory_notifier(page_cgroup_callback, 0);
52d4b9ac	334	printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
37573e8c KH	335	printk(KERN_INFO "please try 'cgroup_disable=memory' option if you "
	336	"don't want memory cgroups\n");
	337	return;
	338	oom:
	339	printk(KERN_CRIT "try 'cgroup_disable=memory' boot option\n");
	340	panic("Out of memory");
52d4b9ac KH	341	}
52d4b9ac KH	342
31168481	343	void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
52d4b9ac KH	344	{
	345	return;
	346	}
	347
	348	#endif
27a7faa0 KH	349
	350
	351	#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
	352
	353	static DEFINE_MUTEX(swap_cgroup_mutex);
	354	struct swap_cgroup_ctrl {
	355	struct page **map;
	356	unsigned long length;
e9e58a4e	357	spinlock_t lock;
27a7faa0 KH	358	};
	359
	360	struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
	361
27a7faa0	362	struct swap_cgroup {
a3b2d692	363	unsigned short id;
27a7faa0 KH	364	};
	365	#define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup))
	366	#define SC_POS_MASK (SC_PER_PAGE - 1)
	367
	368	/*
	369	* SwapCgroup implements "lookup" and "exchange" operations.
	370	* In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
	371	* against SwapCache. At swap_free(), this is accessed directly from swap.
	372	*
	373	* This means,
	374	* - we have no race in "exchange" when we're accessed via SwapCache because
	375	* SwapCache(and its swp_entry) is under lock.
	376	* - When called via swap_free(), there is no user of this entry and no race.
	377	* Then, we don't need lock around "exchange".
	378	*
	379	* TODO: we can push these buffers out to HIGHMEM.
	380	*/
	381
	382	/*
	383	* allocate buffer for swap_cgroup.
	384	*/
	385	static int swap_cgroup_prepare(int type)
	386	{
	387	struct page *page;
	388	struct swap_cgroup_ctrl *ctrl;
	389	unsigned long idx, max;
	390
27a7faa0 KH	391	ctrl = &swap_cgroup_ctrl[type];
	392
	393	for (idx = 0; idx < ctrl->length; idx++) {
	394	page = alloc_page(GFP_KERNEL \| __GFP_ZERO);
	395	if (!page)
	396	goto not_enough_page;
	397	ctrl->map[idx] = page;
	398	}
	399	return 0;
	400	not_enough_page:
	401	max = idx;
	402	for (idx = 0; idx < max; idx++)
	403	__free_page(ctrl->map[idx]);
	404
	405	return -ENOMEM;
	406	}
	407
02491447 DN	408	/**
	409	* swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
	410	* @end: swap entry to be cmpxchged
	411	* @old: old id
	412	* @new: new id
	413	*
	414	* Returns old id at success, 0 at failure.
25985edc	415	* (There is no mem_cgroup using 0 as its id)
02491447 DN	416	*/
	417	unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
	418	unsigned short old, unsigned short new)
	419	{
	420	int type = swp_type(ent);
	421	unsigned long offset = swp_offset(ent);
	422	unsigned long idx = offset / SC_PER_PAGE;
	423	unsigned long pos = offset & SC_POS_MASK;
	424	struct swap_cgroup_ctrl *ctrl;
	425	struct page *mappage;
	426	struct swap_cgroup *sc;
e9e58a4e KH	427	unsigned long flags;
e9e58a4e KH	428	unsigned short retval;
02491447 DN	429
	430	ctrl = &swap_cgroup_ctrl[type];
	431
	432	mappage = ctrl->map[idx];
	433	sc = page_address(mappage);
	434	sc += pos;
e9e58a4e KH	435	spin_lock_irqsave(&ctrl->lock, flags);
	436	retval = sc->id;
	437	if (retval == old)
	438	sc->id = new;
02491447	439	else
e9e58a4e KH	440	retval = 0;
	441	spin_unlock_irqrestore(&ctrl->lock, flags);
	442	return retval;
02491447 DN	443	}
02491447 DN	444
27a7faa0 KH	445	/**
	446	* swap_cgroup_record - record mem_cgroup for this swp_entry.
	447	* @ent: swap entry to be recorded into
	448	* @mem: mem_cgroup to be recorded
	449	*
a3b2d692 KH	450	* Returns old value at success, 0 at failure.
a3b2d692 KH	451	* (Of course, old value can be 0.)
27a7faa0	452	*/
a3b2d692	453	unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
27a7faa0 KH	454	{
	455	int type = swp_type(ent);
	456	unsigned long offset = swp_offset(ent);
	457	unsigned long idx = offset / SC_PER_PAGE;
	458	unsigned long pos = offset & SC_POS_MASK;
	459	struct swap_cgroup_ctrl *ctrl;
	460	struct page *mappage;
	461	struct swap_cgroup *sc;
a3b2d692	462	unsigned short old;
e9e58a4e	463	unsigned long flags;
27a7faa0	464
27a7faa0 KH	465	ctrl = &swap_cgroup_ctrl[type];
	466
	467	mappage = ctrl->map[idx];
	468	sc = page_address(mappage);
	469	sc += pos;
e9e58a4e KH	470	spin_lock_irqsave(&ctrl->lock, flags);
	471	old = sc->id;
	472	sc->id = id;
	473	spin_unlock_irqrestore(&ctrl->lock, flags);
27a7faa0 KH	474
	475	return old;
	476	}
	477
	478	/**
	479	* lookup_swap_cgroup - lookup mem_cgroup tied to swap entry
	480	* @ent: swap entry to be looked up.
	481	*
a3b2d692	482	* Returns CSS ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
27a7faa0	483	*/
a3b2d692	484	unsigned short lookup_swap_cgroup(swp_entry_t ent)
27a7faa0 KH	485	{
	486	int type = swp_type(ent);
	487	unsigned long offset = swp_offset(ent);
	488	unsigned long idx = offset / SC_PER_PAGE;
	489	unsigned long pos = offset & SC_POS_MASK;
	490	struct swap_cgroup_ctrl *ctrl;
	491	struct page *mappage;
	492	struct swap_cgroup *sc;
a3b2d692	493	unsigned short ret;
27a7faa0	494
27a7faa0 KH	495	ctrl = &swap_cgroup_ctrl[type];
	496	mappage = ctrl->map[idx];
	497	sc = page_address(mappage);
	498	sc += pos;
a3b2d692	499	ret = sc->id;
27a7faa0 KH	500	return ret;
	501	}
	502
	503	int swap_cgroup_swapon(int type, unsigned long max_pages)
	504	{
	505	void *array;
	506	unsigned long array_size;
	507	unsigned long length;
	508	struct swap_cgroup_ctrl *ctrl;
	509
	510	if (!do_swap_account)
	511	return 0;
	512
33278f7f	513	length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
27a7faa0 KH	514	array_size = length * sizeof(void *);
	515
	516	array = vmalloc(array_size);
	517	if (!array)
	518	goto nomem;
	519
	520	memset(array, 0, array_size);
	521	ctrl = &swap_cgroup_ctrl[type];
	522	mutex_lock(&swap_cgroup_mutex);
	523	ctrl->length = length;
	524	ctrl->map = array;
e9e58a4e	525	spin_lock_init(&ctrl->lock);
27a7faa0 KH	526	if (swap_cgroup_prepare(type)) {
	527	/* memory shortage */
	528	ctrl->map = NULL;
	529	ctrl->length = 0;
27a7faa0	530	mutex_unlock(&swap_cgroup_mutex);
6a5b18d2	531	vfree(array);
27a7faa0 KH	532	goto nomem;
	533	}
	534	mutex_unlock(&swap_cgroup_mutex);
	535
27a7faa0 KH	536	return 0;
	537	nomem:
	538	printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
	539	printk(KERN_INFO
	540	"swap_cgroup can be disabled by noswapaccount boot option\n");
	541	return -ENOMEM;
	542	}
	543
	544	void swap_cgroup_swapoff(int type)
	545	{
6a5b18d2 NK	546	struct page **map;
6a5b18d2 NK	547	unsigned long i, length;
27a7faa0 KH	548	struct swap_cgroup_ctrl *ctrl;
	549
	550	if (!do_swap_account)
	551	return;
	552
	553	mutex_lock(&swap_cgroup_mutex);
	554	ctrl = &swap_cgroup_ctrl[type];
6a5b18d2 NK	555	map = ctrl->map;
	556	length = ctrl->length;
	557	ctrl->map = NULL;
	558	ctrl->length = 0;
	559	mutex_unlock(&swap_cgroup_mutex);
	560
	561	if (map) {
	562	for (i = 0; i < length; i++) {
	563	struct page *page = map[i];
27a7faa0 KH	564	if (page)
	565	__free_page(page);
	566	}
6a5b18d2	567	vfree(map);
27a7faa0	568	}
27a7faa0 KH	569	}
	570
	571	#endif