Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * linux/mm/oom_kill.c | |
3 | * | |
4 | * Copyright (C) 1998,2000 Rik van Riel | |
5 | * Thanks go out to Claus Fischer for some serious inspiration and | |
6 | * for goading me into coding this file... | |
a63d83f4 DR |
7 | * Copyright (C) 2010 Google, Inc. |
8 | * Rewritten by David Rientjes | |
1da177e4 LT |
9 | * |
10 | * The routines in this file are used to kill a process when | |
a49335cc PJ |
11 | * we're seriously out of memory. This gets called from __alloc_pages() |
12 | * in mm/page_alloc.c when we really run out of memory. | |
1da177e4 LT |
13 | * |
14 | * Since we won't call these routines often (on a well-configured | |
15 | * machine) this file will double as a 'coding guide' and a signpost | |
16 | * for newbie kernel hackers. It features several pointers to major | |
17 | * kernel subsystems and hints as to where to find out what things do. | |
18 | */ | |
19 | ||
8ac773b4 | 20 | #include <linux/oom.h> |
1da177e4 | 21 | #include <linux/mm.h> |
4e950f6f | 22 | #include <linux/err.h> |
5a0e3ad6 | 23 | #include <linux/gfp.h> |
1da177e4 | 24 | #include <linux/sched.h> |
6e84f315 | 25 | #include <linux/sched/mm.h> |
f7ccbae4 | 26 | #include <linux/sched/coredump.h> |
29930025 | 27 | #include <linux/sched/task.h> |
1da177e4 LT |
28 | #include <linux/swap.h> |
29 | #include <linux/timex.h> | |
30 | #include <linux/jiffies.h> | |
ef08e3b4 | 31 | #include <linux/cpuset.h> |
b95f1b31 | 32 | #include <linux/export.h> |
8bc719d3 | 33 | #include <linux/notifier.h> |
c7ba5c9e | 34 | #include <linux/memcontrol.h> |
6f48d0eb | 35 | #include <linux/mempolicy.h> |
5cd9c58f | 36 | #include <linux/security.h> |
edd45544 | 37 | #include <linux/ptrace.h> |
f660daac | 38 | #include <linux/freezer.h> |
43d2b113 | 39 | #include <linux/ftrace.h> |
dc3f21ea | 40 | #include <linux/ratelimit.h> |
aac45363 MH |
41 | #include <linux/kthread.h> |
42 | #include <linux/init.h> | |
4d4bbd85 | 43 | #include <linux/mmu_notifier.h> |
aac45363 MH |
44 | |
45 | #include <asm/tlb.h> | |
46 | #include "internal.h" | |
43d2b113 KH |
47 | |
48 | #define CREATE_TRACE_POINTS | |
49 | #include <trace/events/oom.h> | |
1da177e4 | 50 | |
fadd8fbd | 51 | int sysctl_panic_on_oom; |
fe071d7e | 52 | int sysctl_oom_kill_allocating_task; |
ad915c43 | 53 | int sysctl_oom_dump_tasks = 1; |
2b3cc5e2 | 54 | int sysctl_reap_mem_on_sigkill; |
dc56401f JW |
55 | |
56 | DEFINE_MUTEX(oom_lock); | |
1da177e4 | 57 | |
6f48d0eb DR |
58 | #ifdef CONFIG_NUMA |
59 | /** | |
60 | * has_intersects_mems_allowed() - check task eligiblity for kill | |
ad962441 | 61 | * @start: task struct of which task to consider |
6f48d0eb DR |
62 | * @mask: nodemask passed to page allocator for mempolicy ooms |
63 | * | |
64 | * Task eligibility is determined by whether or not a candidate task, @tsk, | |
65 | * shares the same mempolicy nodes as current if it is bound by such a policy | |
66 | * and whether or not it has the same set of allowed cpuset nodes. | |
495789a5 | 67 | */ |
ad962441 | 68 | static bool has_intersects_mems_allowed(struct task_struct *start, |
6f48d0eb | 69 | const nodemask_t *mask) |
495789a5 | 70 | { |
ad962441 ON |
71 | struct task_struct *tsk; |
72 | bool ret = false; | |
495789a5 | 73 | |
ad962441 | 74 | rcu_read_lock(); |
1da4db0c | 75 | for_each_thread(start, tsk) { |
6f48d0eb DR |
76 | if (mask) { |
77 | /* | |
78 | * If this is a mempolicy constrained oom, tsk's | |
79 | * cpuset is irrelevant. Only return true if its | |
80 | * mempolicy intersects current, otherwise it may be | |
81 | * needlessly killed. | |
82 | */ | |
ad962441 | 83 | ret = mempolicy_nodemask_intersects(tsk, mask); |
6f48d0eb DR |
84 | } else { |
85 | /* | |
86 | * This is not a mempolicy constrained oom, so only | |
87 | * check the mems of tsk's cpuset. | |
88 | */ | |
ad962441 | 89 | ret = cpuset_mems_allowed_intersects(current, tsk); |
6f48d0eb | 90 | } |
ad962441 ON |
91 | if (ret) |
92 | break; | |
1da4db0c | 93 | } |
ad962441 | 94 | rcu_read_unlock(); |
df1090a8 | 95 | |
ad962441 | 96 | return ret; |
6f48d0eb DR |
97 | } |
98 | #else | |
99 | static bool has_intersects_mems_allowed(struct task_struct *tsk, | |
100 | const nodemask_t *mask) | |
101 | { | |
102 | return true; | |
495789a5 | 103 | } |
6f48d0eb | 104 | #endif /* CONFIG_NUMA */ |
495789a5 | 105 | |
6f48d0eb DR |
106 | /* |
107 | * The process p may have detached its own ->mm while exiting or through | |
108 | * use_mm(), but one or more of its subthreads may still have a valid | |
109 | * pointer. Return p, or any of its subthreads with a valid ->mm, with | |
110 | * task_lock() held. | |
111 | */ | |
158e0a2d | 112 | struct task_struct *find_lock_task_mm(struct task_struct *p) |
dd8e8f40 | 113 | { |
1da4db0c | 114 | struct task_struct *t; |
dd8e8f40 | 115 | |
4d4048be ON |
116 | rcu_read_lock(); |
117 | ||
1da4db0c | 118 | for_each_thread(p, t) { |
dd8e8f40 ON |
119 | task_lock(t); |
120 | if (likely(t->mm)) | |
4d4048be | 121 | goto found; |
dd8e8f40 | 122 | task_unlock(t); |
1da4db0c | 123 | } |
4d4048be ON |
124 | t = NULL; |
125 | found: | |
126 | rcu_read_unlock(); | |
dd8e8f40 | 127 | |
4d4048be | 128 | return t; |
dd8e8f40 ON |
129 | } |
130 | ||
db2a0dd7 YB |
131 | /* |
132 | * order == -1 means the oom kill is required by sysrq, otherwise only | |
133 | * for display purposes. | |
134 | */ | |
135 | static inline bool is_sysrq_oom(struct oom_control *oc) | |
136 | { | |
137 | return oc->order == -1; | |
138 | } | |
139 | ||
7c5f64f8 VD |
140 | static inline bool is_memcg_oom(struct oom_control *oc) |
141 | { | |
142 | return oc->memcg != NULL; | |
143 | } | |
144 | ||
ab290adb | 145 | /* return true if the task is not adequate as candidate victim task. */ |
e85bfd3a | 146 | static bool oom_unkillable_task(struct task_struct *p, |
2314b42d | 147 | struct mem_cgroup *memcg, const nodemask_t *nodemask) |
ab290adb KM |
148 | { |
149 | if (is_global_init(p)) | |
150 | return true; | |
151 | if (p->flags & PF_KTHREAD) | |
152 | return true; | |
153 | ||
154 | /* When mem_cgroup_out_of_memory() and p is not member of the group */ | |
72835c86 | 155 | if (memcg && !task_in_mem_cgroup(p, memcg)) |
ab290adb KM |
156 | return true; |
157 | ||
158 | /* p may not have freeable memory in nodemask */ | |
159 | if (!has_intersects_mems_allowed(p, nodemask)) | |
160 | return true; | |
161 | ||
162 | return false; | |
163 | } | |
164 | ||
1da177e4 | 165 | /** |
a63d83f4 | 166 | * oom_badness - heuristic function to determine which candidate task to kill |
1da177e4 | 167 | * @p: task struct of which task we should calculate |
a63d83f4 | 168 | * @totalpages: total present RAM allowed for page allocation |
1da177e4 | 169 | * |
a63d83f4 DR |
170 | * The heuristic for determining which task to kill is made to be as simple and |
171 | * predictable as possible. The goal is to return the highest value for the | |
172 | * task consuming the most memory to avoid subsequent oom failures. | |
1da177e4 | 173 | */ |
a7f638f9 DR |
174 | unsigned long oom_badness(struct task_struct *p, struct mem_cgroup *memcg, |
175 | const nodemask_t *nodemask, unsigned long totalpages) | |
1da177e4 | 176 | { |
1e11ad8d | 177 | long points; |
61eafb00 | 178 | long adj; |
28b83c51 | 179 | |
72835c86 | 180 | if (oom_unkillable_task(p, memcg, nodemask)) |
26ebc984 | 181 | return 0; |
1da177e4 | 182 | |
dd8e8f40 ON |
183 | p = find_lock_task_mm(p); |
184 | if (!p) | |
1da177e4 LT |
185 | return 0; |
186 | ||
bb8a4b7f MH |
187 | /* |
188 | * Do not even consider tasks which are explicitly marked oom | |
b18dc5f2 MH |
189 | * unkillable or have been already oom reaped or the are in |
190 | * the middle of vfork | |
bb8a4b7f | 191 | */ |
a9c58b90 | 192 | adj = (long)p->signal->oom_score_adj; |
bb8a4b7f | 193 | if (adj == OOM_SCORE_ADJ_MIN || |
862e3073 | 194 | test_bit(MMF_OOM_SKIP, &p->mm->flags) || |
b18dc5f2 | 195 | in_vfork(p)) { |
5aecc85a MH |
196 | task_unlock(p); |
197 | return 0; | |
198 | } | |
199 | ||
1da177e4 | 200 | /* |
a63d83f4 | 201 | * The baseline for the badness score is the proportion of RAM that each |
f755a042 | 202 | * task's rss, pagetable and swap space use. |
1da177e4 | 203 | */ |
dc6c9a35 KS |
204 | points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) + |
205 | atomic_long_read(&p->mm->nr_ptes) + mm_nr_pmds(p->mm); | |
a63d83f4 | 206 | task_unlock(p); |
1da177e4 LT |
207 | |
208 | /* | |
a63d83f4 DR |
209 | * Root processes get 3% bonus, just like the __vm_enough_memory() |
210 | * implementation used by LSMs. | |
1da177e4 | 211 | */ |
a63d83f4 | 212 | if (has_capability_noaudit(p, CAP_SYS_ADMIN)) |
778c14af | 213 | points -= (points * 3) / 100; |
1da177e4 | 214 | |
61eafb00 DR |
215 | /* Normalize to oom_score_adj units */ |
216 | adj *= totalpages / 1000; | |
217 | points += adj; | |
1da177e4 | 218 | |
f19e8aa1 | 219 | /* |
a7f638f9 DR |
220 | * Never return 0 for an eligible task regardless of the root bonus and |
221 | * oom_score_adj (oom_score_adj can't be OOM_SCORE_ADJ_MIN here). | |
f19e8aa1 | 222 | */ |
1e11ad8d | 223 | return points > 0 ? points : 1; |
1da177e4 LT |
224 | } |
225 | ||
7c5f64f8 VD |
226 | enum oom_constraint { |
227 | CONSTRAINT_NONE, | |
228 | CONSTRAINT_CPUSET, | |
229 | CONSTRAINT_MEMORY_POLICY, | |
230 | CONSTRAINT_MEMCG, | |
231 | }; | |
232 | ||
9b0f8b04 CL |
233 | /* |
234 | * Determine the type of allocation constraint. | |
235 | */ | |
7c5f64f8 | 236 | static enum oom_constraint constrained_alloc(struct oom_control *oc) |
4365a567 | 237 | { |
54a6eb5c | 238 | struct zone *zone; |
dd1a239f | 239 | struct zoneref *z; |
6e0fc46d | 240 | enum zone_type high_zoneidx = gfp_zone(oc->gfp_mask); |
a63d83f4 DR |
241 | bool cpuset_limited = false; |
242 | int nid; | |
9b0f8b04 | 243 | |
7c5f64f8 VD |
244 | if (is_memcg_oom(oc)) { |
245 | oc->totalpages = mem_cgroup_get_limit(oc->memcg) ?: 1; | |
246 | return CONSTRAINT_MEMCG; | |
247 | } | |
248 | ||
a63d83f4 | 249 | /* Default to all available memory */ |
7c5f64f8 VD |
250 | oc->totalpages = totalram_pages + total_swap_pages; |
251 | ||
252 | if (!IS_ENABLED(CONFIG_NUMA)) | |
253 | return CONSTRAINT_NONE; | |
a63d83f4 | 254 | |
6e0fc46d | 255 | if (!oc->zonelist) |
a63d83f4 | 256 | return CONSTRAINT_NONE; |
4365a567 KH |
257 | /* |
258 | * Reach here only when __GFP_NOFAIL is used. So, we should avoid | |
259 | * to kill current.We have to random task kill in this case. | |
260 | * Hopefully, CONSTRAINT_THISNODE...but no way to handle it, now. | |
261 | */ | |
6e0fc46d | 262 | if (oc->gfp_mask & __GFP_THISNODE) |
4365a567 | 263 | return CONSTRAINT_NONE; |
9b0f8b04 | 264 | |
4365a567 | 265 | /* |
a63d83f4 DR |
266 | * This is not a __GFP_THISNODE allocation, so a truncated nodemask in |
267 | * the page allocator means a mempolicy is in effect. Cpuset policy | |
268 | * is enforced in get_page_from_freelist(). | |
4365a567 | 269 | */ |
6e0fc46d DR |
270 | if (oc->nodemask && |
271 | !nodes_subset(node_states[N_MEMORY], *oc->nodemask)) { | |
7c5f64f8 | 272 | oc->totalpages = total_swap_pages; |
6e0fc46d | 273 | for_each_node_mask(nid, *oc->nodemask) |
7c5f64f8 | 274 | oc->totalpages += node_spanned_pages(nid); |
9b0f8b04 | 275 | return CONSTRAINT_MEMORY_POLICY; |
a63d83f4 | 276 | } |
4365a567 KH |
277 | |
278 | /* Check this allocation failure is caused by cpuset's wall function */ | |
6e0fc46d DR |
279 | for_each_zone_zonelist_nodemask(zone, z, oc->zonelist, |
280 | high_zoneidx, oc->nodemask) | |
281 | if (!cpuset_zone_allowed(zone, oc->gfp_mask)) | |
a63d83f4 | 282 | cpuset_limited = true; |
9b0f8b04 | 283 | |
a63d83f4 | 284 | if (cpuset_limited) { |
7c5f64f8 | 285 | oc->totalpages = total_swap_pages; |
a63d83f4 | 286 | for_each_node_mask(nid, cpuset_current_mems_allowed) |
7c5f64f8 | 287 | oc->totalpages += node_spanned_pages(nid); |
a63d83f4 DR |
288 | return CONSTRAINT_CPUSET; |
289 | } | |
9b0f8b04 CL |
290 | return CONSTRAINT_NONE; |
291 | } | |
292 | ||
7c5f64f8 | 293 | static int oom_evaluate_task(struct task_struct *task, void *arg) |
462607ec | 294 | { |
7c5f64f8 VD |
295 | struct oom_control *oc = arg; |
296 | unsigned long points; | |
297 | ||
6e0fc46d | 298 | if (oom_unkillable_task(task, NULL, oc->nodemask)) |
7c5f64f8 | 299 | goto next; |
462607ec DR |
300 | |
301 | /* | |
302 | * This task already has access to memory reserves and is being killed. | |
a373966d | 303 | * Don't allow any other task to have access to the reserves unless |
862e3073 | 304 | * the task has MMF_OOM_SKIP because chances that it would release |
a373966d | 305 | * any memory is quite low. |
462607ec | 306 | */ |
862e3073 MH |
307 | if (!is_sysrq_oom(oc) && tsk_is_oom_victim(task)) { |
308 | if (test_bit(MMF_OOM_SKIP, &task->signal->oom_mm->flags)) | |
7c5f64f8 VD |
309 | goto next; |
310 | goto abort; | |
a373966d | 311 | } |
462607ec | 312 | |
e1e12d2f DR |
313 | /* |
314 | * If task is allocating a lot of memory and has been marked to be | |
315 | * killed first if it triggers an oom, then select it. | |
316 | */ | |
7c5f64f8 VD |
317 | if (oom_task_origin(task)) { |
318 | points = ULONG_MAX; | |
319 | goto select; | |
320 | } | |
e1e12d2f | 321 | |
7c5f64f8 VD |
322 | points = oom_badness(task, NULL, oc->nodemask, oc->totalpages); |
323 | if (!points || points < oc->chosen_points) | |
324 | goto next; | |
325 | ||
326 | /* Prefer thread group leaders for display purposes */ | |
327 | if (points == oc->chosen_points && thread_group_leader(oc->chosen)) | |
328 | goto next; | |
329 | select: | |
330 | if (oc->chosen) | |
331 | put_task_struct(oc->chosen); | |
332 | get_task_struct(task); | |
333 | oc->chosen = task; | |
334 | oc->chosen_points = points; | |
335 | next: | |
336 | return 0; | |
337 | abort: | |
338 | if (oc->chosen) | |
339 | put_task_struct(oc->chosen); | |
340 | oc->chosen = (void *)-1UL; | |
341 | return 1; | |
462607ec DR |
342 | } |
343 | ||
1da177e4 | 344 | /* |
7c5f64f8 VD |
345 | * Simple selection loop. We choose the process with the highest number of |
346 | * 'points'. In case scan was aborted, oc->chosen is set to -1. | |
1da177e4 | 347 | */ |
7c5f64f8 | 348 | static void select_bad_process(struct oom_control *oc) |
1da177e4 | 349 | { |
7c5f64f8 VD |
350 | if (is_memcg_oom(oc)) |
351 | mem_cgroup_scan_tasks(oc->memcg, oom_evaluate_task, oc); | |
352 | else { | |
353 | struct task_struct *p; | |
d49ad935 | 354 | |
7c5f64f8 VD |
355 | rcu_read_lock(); |
356 | for_each_process(p) | |
357 | if (oom_evaluate_task(p, oc)) | |
358 | break; | |
359 | rcu_read_unlock(); | |
1da4db0c | 360 | } |
972c4ea5 | 361 | |
7c5f64f8 | 362 | oc->chosen_points = oc->chosen_points * 1000 / oc->totalpages; |
1da177e4 LT |
363 | } |
364 | ||
fef1bdd6 | 365 | /** |
1b578df0 | 366 | * dump_tasks - dump current memory state of all system tasks |
dad7557e | 367 | * @memcg: current's memory controller, if constrained |
e85bfd3a | 368 | * @nodemask: nodemask passed to page allocator for mempolicy ooms |
1b578df0 | 369 | * |
e85bfd3a DR |
370 | * Dumps the current memory state of all eligible tasks. Tasks not in the same |
371 | * memcg, not in the same cpuset, or bound to a disjoint set of mempolicy nodes | |
372 | * are not shown. | |
de34d965 DR |
373 | * State information includes task's pid, uid, tgid, vm size, rss, nr_ptes, |
374 | * swapents, oom_score_adj value, and name. | |
fef1bdd6 | 375 | */ |
2314b42d | 376 | static void dump_tasks(struct mem_cgroup *memcg, const nodemask_t *nodemask) |
fef1bdd6 | 377 | { |
c55db957 KM |
378 | struct task_struct *p; |
379 | struct task_struct *task; | |
fef1bdd6 | 380 | |
dc6c9a35 | 381 | pr_info("[ pid ] uid tgid total_vm rss nr_ptes nr_pmds swapents oom_score_adj name\n"); |
6b0c81b3 | 382 | rcu_read_lock(); |
c55db957 | 383 | for_each_process(p) { |
72835c86 | 384 | if (oom_unkillable_task(p, memcg, nodemask)) |
b4416d2b | 385 | continue; |
fef1bdd6 | 386 | |
c55db957 KM |
387 | task = find_lock_task_mm(p); |
388 | if (!task) { | |
6d2661ed | 389 | /* |
74ab7f1d DR |
390 | * This is a kthread or all of p's threads have already |
391 | * detached their mm's. There's no need to report | |
c55db957 | 392 | * them; they can't be oom killed anyway. |
6d2661ed | 393 | */ |
6d2661ed DR |
394 | continue; |
395 | } | |
c55db957 | 396 | |
dc6c9a35 | 397 | pr_info("[%5d] %5d %5d %8lu %8lu %7ld %7ld %8lu %5hd %s\n", |
078de5f7 EB |
398 | task->pid, from_kuid(&init_user_ns, task_uid(task)), |
399 | task->tgid, task->mm->total_vm, get_mm_rss(task->mm), | |
e1f56c89 | 400 | atomic_long_read(&task->mm->nr_ptes), |
dc6c9a35 | 401 | mm_nr_pmds(task->mm), |
de34d965 | 402 | get_mm_counter(task->mm, MM_SWAPENTS), |
a63d83f4 | 403 | task->signal->oom_score_adj, task->comm); |
c55db957 KM |
404 | task_unlock(task); |
405 | } | |
6b0c81b3 | 406 | rcu_read_unlock(); |
fef1bdd6 DR |
407 | } |
408 | ||
2a966b77 | 409 | static void dump_header(struct oom_control *oc, struct task_struct *p) |
1b604d75 | 410 | { |
299c517a DR |
411 | pr_warn("%s invoked oom-killer: gfp_mask=%#x(%pGg), nodemask=", |
412 | current->comm, oc->gfp_mask, &oc->gfp_mask); | |
413 | if (oc->nodemask) | |
414 | pr_cont("%*pbl", nodemask_pr_args(oc->nodemask)); | |
415 | else | |
416 | pr_cont("(null)"); | |
417 | pr_cont(", order=%d, oom_score_adj=%hd\n", | |
418 | oc->order, current->signal->oom_score_adj); | |
9254990f MH |
419 | if (!IS_ENABLED(CONFIG_COMPACTION) && oc->order) |
420 | pr_warn("COMPACTION is disabled!!!\n"); | |
a0795cd4 | 421 | |
da39da3a | 422 | cpuset_print_current_mems_allowed(); |
1b604d75 | 423 | dump_stack(); |
2a966b77 VD |
424 | if (oc->memcg) |
425 | mem_cgroup_print_oom_info(oc->memcg, p); | |
58cf188e | 426 | else |
299c517a | 427 | show_mem(SHOW_MEM_FILTER_NODES, oc->nodemask); |
1b604d75 | 428 | if (sysctl_oom_dump_tasks) |
2a966b77 | 429 | dump_tasks(oc->memcg, oc->nodemask); |
1b604d75 DR |
430 | } |
431 | ||
5695be14 | 432 | /* |
c32b3cbe | 433 | * Number of OOM victims in flight |
5695be14 | 434 | */ |
c32b3cbe MH |
435 | static atomic_t oom_victims = ATOMIC_INIT(0); |
436 | static DECLARE_WAIT_QUEUE_HEAD(oom_victims_wait); | |
5695be14 | 437 | |
7c5f64f8 | 438 | static bool oom_killer_disabled __read_mostly; |
5695be14 | 439 | |
bc448e89 MH |
440 | #define K(x) ((x) << (PAGE_SHIFT-10)) |
441 | ||
3ef22dff MH |
442 | /* |
443 | * task->mm can be NULL if the task is the exited group leader. So to | |
444 | * determine whether the task is using a particular mm, we examine all the | |
445 | * task's threads: if one of those is using this mm then this task was also | |
446 | * using it. | |
447 | */ | |
44a70ade | 448 | bool process_shares_mm(struct task_struct *p, struct mm_struct *mm) |
3ef22dff MH |
449 | { |
450 | struct task_struct *t; | |
451 | ||
452 | for_each_thread(p, t) { | |
453 | struct mm_struct *t_mm = READ_ONCE(t->mm); | |
454 | if (t_mm) | |
455 | return t_mm == mm; | |
456 | } | |
457 | return false; | |
458 | } | |
459 | ||
aac45363 MH |
460 | #ifdef CONFIG_MMU |
461 | /* | |
462 | * OOM Reaper kernel thread which tries to reap the memory used by the OOM | |
463 | * victim (if that is possible) to help the OOM killer to move on. | |
464 | */ | |
465 | static struct task_struct *oom_reaper_th; | |
aac45363 | 466 | static DECLARE_WAIT_QUEUE_HEAD(oom_reaper_wait); |
29c696e1 | 467 | static struct task_struct *oom_reaper_list; |
03049269 MH |
468 | static DEFINE_SPINLOCK(oom_reaper_lock); |
469 | ||
2270dfcc | 470 | void __oom_reap_task_mm(struct mm_struct *mm) |
aac45363 | 471 | { |
aac45363 | 472 | struct vm_area_struct *vma; |
2270dfcc DR |
473 | |
474 | /* | |
475 | * Tell all users of get_user/copy_from_user etc... that the content | |
476 | * is no longer stable. No barriers really needed because unmapping | |
477 | * should imply barriers already and the reader would hit a page fault | |
478 | * if it stumbled over a reaped memory. | |
479 | */ | |
480 | set_bit(MMF_UNSTABLE, &mm->flags); | |
481 | ||
482 | for (vma = mm->mmap ; vma; vma = vma->vm_next) { | |
483 | if (!can_madv_dontneed_vma(vma)) | |
484 | continue; | |
485 | ||
486 | /* | |
487 | * Only anonymous pages have a good chance to be dropped | |
488 | * without additional steps which we cannot afford as we | |
489 | * are OOM already. | |
490 | * | |
491 | * We do not even care about fs backed pages because all | |
492 | * which are reclaimable have already been reclaimed and | |
493 | * we do not want to block exit_mmap by keeping mm ref | |
494 | * count elevated without a good reason. | |
495 | */ | |
496 | if (vma_is_anonymous(vma) || !(vma->vm_flags & VM_SHARED)) { | |
497 | struct mmu_gather tlb; | |
498 | ||
499 | tlb_gather_mmu(&tlb, mm, vma->vm_start, vma->vm_end); | |
500 | unmap_page_range(&tlb, vma, vma->vm_start, vma->vm_end, | |
501 | NULL); | |
502 | tlb_finish_mmu(&tlb, vma->vm_start, vma->vm_end); | |
503 | } | |
504 | } | |
505 | } | |
506 | ||
507 | static bool oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm) | |
508 | { | |
aac45363 MH |
509 | bool ret = true; |
510 | ||
e2fe1456 MH |
511 | /* |
512 | * We have to make sure to not race with the victim exit path | |
513 | * and cause premature new oom victim selection: | |
2270dfcc | 514 | * oom_reap_task_mm exit_mm |
e5e3f4c4 | 515 | * mmget_not_zero |
e2fe1456 MH |
516 | * mmput |
517 | * atomic_dec_and_test | |
518 | * exit_oom_victim | |
519 | * [...] | |
520 | * out_of_memory | |
521 | * select_bad_process | |
522 | * # no TIF_MEMDIE task selects new victim | |
523 | * unmap_page_range # frees some memory | |
524 | */ | |
525 | mutex_lock(&oom_lock); | |
526 | ||
aac45363 MH |
527 | if (!down_read_trylock(&mm->mmap_sem)) { |
528 | ret = false; | |
422580c3 | 529 | trace_skip_task_reaping(tsk->pid); |
7ebffa45 | 530 | goto unlock_oom; |
e5e3f4c4 MH |
531 | } |
532 | ||
4d4bbd85 MH |
533 | /* |
534 | * If the mm has notifiers then we would need to invalidate them around | |
535 | * unmap_page_range and that is risky because notifiers can sleep and | |
536 | * what they do is basically undeterministic. So let's have a short | |
537 | * sleep to give the oom victim some more time. | |
538 | * TODO: we really want to get rid of this ugly hack and make sure that | |
539 | * notifiers cannot block for unbounded amount of time and add | |
540 | * mmu_notifier_invalidate_range_{start,end} around unmap_page_range | |
541 | */ | |
542 | if (mm_has_notifiers(mm)) { | |
543 | up_read(&mm->mmap_sem); | |
544 | schedule_timeout_idle(HZ); | |
545 | goto unlock_oom; | |
546 | } | |
547 | ||
e5e3f4c4 | 548 | /* |
21292580 AA |
549 | * MMF_OOM_SKIP is set by exit_mmap when the OOM reaper can't |
550 | * work on the mm anymore. The check for MMF_OOM_SKIP must run | |
551 | * under mmap_sem for reading because it serializes against the | |
552 | * down_write();up_write() cycle in exit_mmap(). | |
e5e3f4c4 | 553 | */ |
21292580 | 554 | if (test_bit(MMF_OOM_SKIP, &mm->flags)) { |
e5e3f4c4 | 555 | up_read(&mm->mmap_sem); |
422580c3 | 556 | trace_skip_task_reaping(tsk->pid); |
7ebffa45 | 557 | goto unlock_oom; |
aac45363 MH |
558 | } |
559 | ||
422580c3 RG |
560 | trace_start_task_reaping(tsk->pid); |
561 | ||
2270dfcc | 562 | __oom_reap_task_mm(mm); |
aac45363 | 563 | |
bc448e89 MH |
564 | pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n", |
565 | task_pid_nr(tsk), tsk->comm, | |
566 | K(get_mm_counter(mm, MM_ANONPAGES)), | |
567 | K(get_mm_counter(mm, MM_FILEPAGES)), | |
568 | K(get_mm_counter(mm, MM_SHMEMPAGES))); | |
aac45363 | 569 | up_read(&mm->mmap_sem); |
36324a99 | 570 | |
422580c3 | 571 | trace_finish_task_reaping(tsk->pid); |
e5e3f4c4 MH |
572 | unlock_oom: |
573 | mutex_unlock(&oom_lock); | |
aac45363 MH |
574 | return ret; |
575 | } | |
576 | ||
bc448e89 | 577 | #define MAX_OOM_REAP_RETRIES 10 |
36324a99 | 578 | static void oom_reap_task(struct task_struct *tsk) |
aac45363 MH |
579 | { |
580 | int attempts = 0; | |
26db62f1 | 581 | struct mm_struct *mm = tsk->signal->oom_mm; |
aac45363 MH |
582 | |
583 | /* Retry the down_read_trylock(mmap_sem) a few times */ | |
2270dfcc | 584 | while (attempts++ < MAX_OOM_REAP_RETRIES && !oom_reap_task_mm(tsk, mm)) |
aac45363 MH |
585 | schedule_timeout_idle(HZ/10); |
586 | ||
7ebffa45 TH |
587 | if (attempts <= MAX_OOM_REAP_RETRIES) |
588 | goto done; | |
11a410d5 | 589 | |
7ebffa45 TH |
590 | pr_info("oom_reaper: unable to reap pid:%d (%s)\n", |
591 | task_pid_nr(tsk), tsk->comm); | |
7ebffa45 | 592 | debug_show_all_locks(); |
bc448e89 | 593 | |
7ebffa45 | 594 | done: |
449d777d | 595 | tsk->oom_reaper_list = NULL; |
449d777d | 596 | |
26db62f1 MH |
597 | /* |
598 | * Hide this mm from OOM killer because it has been either reaped or | |
599 | * somebody can't call up_write(mmap_sem). | |
600 | */ | |
862e3073 | 601 | set_bit(MMF_OOM_SKIP, &mm->flags); |
26db62f1 | 602 | |
aac45363 | 603 | /* Drop a reference taken by wake_oom_reaper */ |
36324a99 | 604 | put_task_struct(tsk); |
aac45363 MH |
605 | } |
606 | ||
607 | static int oom_reaper(void *unused) | |
608 | { | |
609 | while (true) { | |
03049269 | 610 | struct task_struct *tsk = NULL; |
aac45363 | 611 | |
29c696e1 | 612 | wait_event_freezable(oom_reaper_wait, oom_reaper_list != NULL); |
03049269 | 613 | spin_lock(&oom_reaper_lock); |
29c696e1 VD |
614 | if (oom_reaper_list != NULL) { |
615 | tsk = oom_reaper_list; | |
616 | oom_reaper_list = tsk->oom_reaper_list; | |
03049269 MH |
617 | } |
618 | spin_unlock(&oom_reaper_lock); | |
619 | ||
620 | if (tsk) | |
621 | oom_reap_task(tsk); | |
aac45363 MH |
622 | } |
623 | ||
624 | return 0; | |
625 | } | |
626 | ||
7c5f64f8 | 627 | static void wake_oom_reaper(struct task_struct *tsk) |
aac45363 | 628 | { |
af8e15cc MH |
629 | if (!oom_reaper_th) |
630 | return; | |
2b3cc5e2 | 631 | /* |
632 | * Move the lock here to avoid scenario of queuing | |
633 | * the same task by both OOM killer and any other SIGKILL | |
634 | * path. | |
635 | */ | |
636 | ||
637 | spin_lock(&oom_reaper_lock); | |
af8e15cc | 638 | |
73178548 | 639 | /* mm is already queued? */ |
2b3cc5e2 | 640 | if (test_and_set_bit(MMF_OOM_REAP_QUEUED, &tsk->signal->oom_mm->flags)) { |
641 | spin_unlock(&oom_reaper_lock); | |
aac45363 | 642 | return; |
2b3cc5e2 | 643 | } |
aac45363 | 644 | |
36324a99 | 645 | get_task_struct(tsk); |
aac45363 | 646 | |
29c696e1 VD |
647 | tsk->oom_reaper_list = oom_reaper_list; |
648 | oom_reaper_list = tsk; | |
03049269 | 649 | spin_unlock(&oom_reaper_lock); |
422580c3 | 650 | trace_wake_reaper(tsk->pid); |
03049269 | 651 | wake_up(&oom_reaper_wait); |
aac45363 MH |
652 | } |
653 | ||
654 | static int __init oom_init(void) | |
655 | { | |
656 | oom_reaper_th = kthread_run(oom_reaper, NULL, "oom_reaper"); | |
657 | if (IS_ERR(oom_reaper_th)) { | |
658 | pr_err("Unable to start OOM reaper %ld. Continuing regardless\n", | |
659 | PTR_ERR(oom_reaper_th)); | |
660 | oom_reaper_th = NULL; | |
661 | } | |
662 | return 0; | |
663 | } | |
664 | subsys_initcall(oom_init) | |
7c5f64f8 VD |
665 | #else |
666 | static inline void wake_oom_reaper(struct task_struct *tsk) | |
667 | { | |
668 | } | |
669 | #endif /* CONFIG_MMU */ | |
aac45363 | 670 | |
2b3cc5e2 | 671 | static void __mark_oom_victim(struct task_struct *tsk) |
672 | { | |
673 | struct mm_struct *mm = tsk->mm; | |
674 | ||
675 | if (!cmpxchg(&tsk->signal->oom_mm, NULL, mm)) { | |
676 | mmgrab(tsk->signal->oom_mm); | |
677 | set_bit(MMF_OOM_VICTIM, &mm->flags); | |
678 | } | |
679 | } | |
680 | ||
49550b60 | 681 | /** |
16e95196 | 682 | * mark_oom_victim - mark the given task as OOM victim |
49550b60 | 683 | * @tsk: task to mark |
c32b3cbe | 684 | * |
dc56401f | 685 | * Has to be called with oom_lock held and never after |
c32b3cbe | 686 | * oom has been disabled already. |
26db62f1 MH |
687 | * |
688 | * tsk->mm has to be non NULL and caller has to guarantee it is stable (either | |
689 | * under task_lock or operate on the current). | |
49550b60 | 690 | */ |
7c5f64f8 | 691 | static void mark_oom_victim(struct task_struct *tsk) |
49550b60 | 692 | { |
c32b3cbe MH |
693 | WARN_ON(oom_killer_disabled); |
694 | /* OOM killer might race with memcg OOM */ | |
695 | if (test_and_set_tsk_thread_flag(tsk, TIF_MEMDIE)) | |
696 | return; | |
26db62f1 | 697 | |
26db62f1 | 698 | /* oom_mm is bound to the signal struct life time. */ |
2b3cc5e2 | 699 | __mark_oom_victim(tsk); |
26db62f1 | 700 | |
63a8ca9b MH |
701 | /* |
702 | * Make sure that the task is woken up from uninterruptible sleep | |
703 | * if it is frozen because OOM killer wouldn't be able to free | |
704 | * any memory and livelock. freezing_slow_path will tell the freezer | |
705 | * that TIF_MEMDIE tasks should be ignored. | |
706 | */ | |
707 | __thaw_task(tsk); | |
c32b3cbe | 708 | atomic_inc(&oom_victims); |
422580c3 | 709 | trace_mark_victim(tsk->pid); |
49550b60 MH |
710 | } |
711 | ||
712 | /** | |
16e95196 | 713 | * exit_oom_victim - note the exit of an OOM victim |
49550b60 | 714 | */ |
38531201 | 715 | void exit_oom_victim(void) |
49550b60 | 716 | { |
38531201 | 717 | clear_thread_flag(TIF_MEMDIE); |
c32b3cbe | 718 | |
c38f1025 | 719 | if (!atomic_dec_return(&oom_victims)) |
c32b3cbe | 720 | wake_up_all(&oom_victims_wait); |
c32b3cbe MH |
721 | } |
722 | ||
7d2e7a22 MH |
723 | /** |
724 | * oom_killer_enable - enable OOM killer | |
725 | */ | |
726 | void oom_killer_enable(void) | |
727 | { | |
728 | oom_killer_disabled = false; | |
d75da004 | 729 | pr_info("OOM killer enabled.\n"); |
7d2e7a22 MH |
730 | } |
731 | ||
c32b3cbe MH |
732 | /** |
733 | * oom_killer_disable - disable OOM killer | |
7d2e7a22 | 734 | * @timeout: maximum timeout to wait for oom victims in jiffies |
c32b3cbe MH |
735 | * |
736 | * Forces all page allocations to fail rather than trigger OOM killer. | |
7d2e7a22 MH |
737 | * Will block and wait until all OOM victims are killed or the given |
738 | * timeout expires. | |
c32b3cbe MH |
739 | * |
740 | * The function cannot be called when there are runnable user tasks because | |
741 | * the userspace would see unexpected allocation failures as a result. Any | |
742 | * new usage of this function should be consulted with MM people. | |
743 | * | |
744 | * Returns true if successful and false if the OOM killer cannot be | |
745 | * disabled. | |
746 | */ | |
7d2e7a22 | 747 | bool oom_killer_disable(signed long timeout) |
c32b3cbe | 748 | { |
7d2e7a22 MH |
749 | signed long ret; |
750 | ||
c32b3cbe | 751 | /* |
6afcf289 TH |
752 | * Make sure to not race with an ongoing OOM killer. Check that the |
753 | * current is not killed (possibly due to sharing the victim's memory). | |
c32b3cbe | 754 | */ |
6afcf289 | 755 | if (mutex_lock_killable(&oom_lock)) |
c32b3cbe | 756 | return false; |
c32b3cbe | 757 | oom_killer_disabled = true; |
dc56401f | 758 | mutex_unlock(&oom_lock); |
c32b3cbe | 759 | |
7d2e7a22 MH |
760 | ret = wait_event_interruptible_timeout(oom_victims_wait, |
761 | !atomic_read(&oom_victims), timeout); | |
762 | if (ret <= 0) { | |
763 | oom_killer_enable(); | |
764 | return false; | |
765 | } | |
d75da004 | 766 | pr_info("OOM killer disabled.\n"); |
c32b3cbe MH |
767 | |
768 | return true; | |
769 | } | |
770 | ||
1af8bb43 MH |
771 | static inline bool __task_will_free_mem(struct task_struct *task) |
772 | { | |
773 | struct signal_struct *sig = task->signal; | |
774 | ||
775 | /* | |
776 | * A coredumping process may sleep for an extended period in exit_mm(), | |
777 | * so the oom killer cannot assume that the process will promptly exit | |
778 | * and release memory. | |
779 | */ | |
780 | if (sig->flags & SIGNAL_GROUP_COREDUMP) | |
781 | return false; | |
782 | ||
783 | if (sig->flags & SIGNAL_GROUP_EXIT) | |
784 | return true; | |
785 | ||
786 | if (thread_group_empty(task) && (task->flags & PF_EXITING)) | |
787 | return true; | |
788 | ||
789 | return false; | |
790 | } | |
791 | ||
792 | /* | |
793 | * Checks whether the given task is dying or exiting and likely to | |
794 | * release its address space. This means that all threads and processes | |
795 | * sharing the same mm have to be killed or exiting. | |
091f362c MH |
796 | * Caller has to make sure that task->mm is stable (hold task_lock or |
797 | * it operates on the current). | |
1af8bb43 | 798 | */ |
7c5f64f8 | 799 | static bool task_will_free_mem(struct task_struct *task) |
1af8bb43 | 800 | { |
091f362c | 801 | struct mm_struct *mm = task->mm; |
1af8bb43 | 802 | struct task_struct *p; |
f33e6f06 | 803 | bool ret = true; |
1af8bb43 | 804 | |
1af8bb43 | 805 | /* |
091f362c MH |
806 | * Skip tasks without mm because it might have passed its exit_mm and |
807 | * exit_oom_victim. oom_reaper could have rescued that but do not rely | |
808 | * on that for now. We can consider find_lock_task_mm in future. | |
1af8bb43 | 809 | */ |
091f362c | 810 | if (!mm) |
1af8bb43 MH |
811 | return false; |
812 | ||
091f362c MH |
813 | if (!__task_will_free_mem(task)) |
814 | return false; | |
696453e6 MH |
815 | |
816 | /* | |
817 | * This task has already been drained by the oom reaper so there are | |
818 | * only small chances it will free some more | |
819 | */ | |
862e3073 | 820 | if (test_bit(MMF_OOM_SKIP, &mm->flags)) |
696453e6 | 821 | return false; |
696453e6 | 822 | |
091f362c | 823 | if (atomic_read(&mm->mm_users) <= 1) |
1af8bb43 | 824 | return true; |
1af8bb43 MH |
825 | |
826 | /* | |
5870c2e1 MH |
827 | * Make sure that all tasks which share the mm with the given tasks |
828 | * are dying as well to make sure that a) nobody pins its mm and | |
829 | * b) the task is also reapable by the oom reaper. | |
1af8bb43 MH |
830 | */ |
831 | rcu_read_lock(); | |
832 | for_each_process(p) { | |
833 | if (!process_shares_mm(p, mm)) | |
834 | continue; | |
835 | if (same_thread_group(task, p)) | |
836 | continue; | |
837 | ret = __task_will_free_mem(p); | |
838 | if (!ret) | |
839 | break; | |
840 | } | |
841 | rcu_read_unlock(); | |
1af8bb43 MH |
842 | |
843 | return ret; | |
844 | } | |
845 | ||
7c5f64f8 | 846 | static void oom_kill_process(struct oom_control *oc, const char *message) |
1da177e4 | 847 | { |
7c5f64f8 VD |
848 | struct task_struct *p = oc->chosen; |
849 | unsigned int points = oc->chosen_points; | |
52d3c036 | 850 | struct task_struct *victim = p; |
5e9d834a | 851 | struct task_struct *child; |
1da4db0c | 852 | struct task_struct *t; |
647f2bdf | 853 | struct mm_struct *mm; |
52d3c036 | 854 | unsigned int victim_points = 0; |
dc3f21ea DR |
855 | static DEFINE_RATELIMIT_STATE(oom_rs, DEFAULT_RATELIMIT_INTERVAL, |
856 | DEFAULT_RATELIMIT_BURST); | |
bb29902a | 857 | bool can_oom_reap = true; |
1da177e4 | 858 | |
50ec3bbf NP |
859 | /* |
860 | * If the task is already exiting, don't alarm the sysadmin or kill | |
cd04ae1e MH |
861 | * its children or threads, just give it access to memory reserves |
862 | * so it can die quickly | |
50ec3bbf | 863 | */ |
091f362c | 864 | task_lock(p); |
1af8bb43 | 865 | if (task_will_free_mem(p)) { |
16e95196 | 866 | mark_oom_victim(p); |
1af8bb43 | 867 | wake_oom_reaper(p); |
091f362c | 868 | task_unlock(p); |
6b0c81b3 | 869 | put_task_struct(p); |
2a1c9b1f | 870 | return; |
50ec3bbf | 871 | } |
091f362c | 872 | task_unlock(p); |
50ec3bbf | 873 | |
dc3f21ea | 874 | if (__ratelimit(&oom_rs)) |
2a966b77 | 875 | dump_header(oc, p); |
8447d950 | 876 | |
f0d6647e | 877 | pr_err("%s: Kill process %d (%s) score %u or sacrifice child\n", |
5e9d834a | 878 | message, task_pid_nr(p), p->comm, points); |
f3af38d3 | 879 | |
5e9d834a DR |
880 | /* |
881 | * If any of p's children has a different mm and is eligible for kill, | |
11239836 | 882 | * the one with the highest oom_badness() score is sacrificed for its |
5e9d834a DR |
883 | * parent. This attempts to lose the minimal amount of work done while |
884 | * still freeing memory. | |
885 | */ | |
6b0c81b3 | 886 | read_lock(&tasklist_lock); |
43f7e8be SB |
887 | |
888 | /* | |
889 | * The task 'p' might have already exited before reaching here. The | |
890 | * put_task_struct() will free task_struct 'p' while the loop still try | |
891 | * to access the field of 'p', so, get an extra reference. | |
892 | */ | |
893 | get_task_struct(p); | |
1da4db0c | 894 | for_each_thread(p, t) { |
5e9d834a | 895 | list_for_each_entry(child, &t->children, sibling) { |
a63d83f4 | 896 | unsigned int child_points; |
5e9d834a | 897 | |
4d7b3394 | 898 | if (process_shares_mm(child, p->mm)) |
edd45544 | 899 | continue; |
a63d83f4 DR |
900 | /* |
901 | * oom_badness() returns 0 if the thread is unkillable | |
902 | */ | |
2a966b77 | 903 | child_points = oom_badness(child, |
7c5f64f8 | 904 | oc->memcg, oc->nodemask, oc->totalpages); |
5e9d834a | 905 | if (child_points > victim_points) { |
6b0c81b3 | 906 | put_task_struct(victim); |
5e9d834a DR |
907 | victim = child; |
908 | victim_points = child_points; | |
6b0c81b3 | 909 | get_task_struct(victim); |
5e9d834a | 910 | } |
dd8e8f40 | 911 | } |
1da4db0c | 912 | } |
43f7e8be | 913 | put_task_struct(p); |
6b0c81b3 | 914 | read_unlock(&tasklist_lock); |
dd8e8f40 | 915 | |
6b0c81b3 DR |
916 | p = find_lock_task_mm(victim); |
917 | if (!p) { | |
6b0c81b3 | 918 | put_task_struct(victim); |
647f2bdf | 919 | return; |
6b0c81b3 DR |
920 | } else if (victim != p) { |
921 | get_task_struct(p); | |
922 | put_task_struct(victim); | |
923 | victim = p; | |
924 | } | |
647f2bdf | 925 | |
880b7689 | 926 | /* Get a reference to safely compare mm after task_unlock(victim) */ |
647f2bdf | 927 | mm = victim->mm; |
f1f10076 | 928 | mmgrab(mm); |
8e675f7a KK |
929 | |
930 | /* Raise event before sending signal: task reaper must see this */ | |
931 | count_vm_event(OOM_KILL); | |
932 | count_memcg_event_mm(mm, OOM_KILL); | |
933 | ||
426fb5e7 | 934 | /* |
cd04ae1e MH |
935 | * We should send SIGKILL before granting access to memory reserves |
936 | * in order to prevent the OOM victim from depleting the memory | |
937 | * reserves from the user space under its control. | |
426fb5e7 TH |
938 | */ |
939 | do_send_sig_info(SIGKILL, SEND_SIG_FORCED, victim, true); | |
16e95196 | 940 | mark_oom_victim(victim); |
eca56ff9 | 941 | pr_err("Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n", |
647f2bdf DR |
942 | task_pid_nr(victim), victim->comm, K(victim->mm->total_vm), |
943 | K(get_mm_counter(victim->mm, MM_ANONPAGES)), | |
eca56ff9 JM |
944 | K(get_mm_counter(victim->mm, MM_FILEPAGES)), |
945 | K(get_mm_counter(victim->mm, MM_SHMEMPAGES))); | |
647f2bdf DR |
946 | task_unlock(victim); |
947 | ||
948 | /* | |
949 | * Kill all user processes sharing victim->mm in other thread groups, if | |
950 | * any. They don't get access to memory reserves, though, to avoid | |
951 | * depletion of all memory. This prevents mm->mmap_sem livelock when an | |
952 | * oom killed thread cannot exit because it requires the semaphore and | |
953 | * its contended by another thread trying to allocate memory itself. | |
954 | * That thread will now get access to memory reserves since it has a | |
955 | * pending fatal signal. | |
956 | */ | |
4d4048be | 957 | rcu_read_lock(); |
c319025a | 958 | for_each_process(p) { |
4d7b3394 | 959 | if (!process_shares_mm(p, mm)) |
c319025a ON |
960 | continue; |
961 | if (same_thread_group(p, victim)) | |
962 | continue; | |
1b51e65e | 963 | if (is_global_init(p)) { |
aac45363 | 964 | can_oom_reap = false; |
862e3073 | 965 | set_bit(MMF_OOM_SKIP, &mm->flags); |
a373966d MH |
966 | pr_info("oom killer %d (%s) has mm pinned by %d (%s)\n", |
967 | task_pid_nr(victim), victim->comm, | |
968 | task_pid_nr(p), p->comm); | |
c319025a | 969 | continue; |
aac45363 | 970 | } |
1b51e65e MH |
971 | /* |
972 | * No use_mm() user needs to read from the userspace so we are | |
973 | * ok to reap it. | |
974 | */ | |
975 | if (unlikely(p->flags & PF_KTHREAD)) | |
976 | continue; | |
c319025a ON |
977 | do_send_sig_info(SIGKILL, SEND_SIG_FORCED, p, true); |
978 | } | |
6b0c81b3 | 979 | rcu_read_unlock(); |
647f2bdf | 980 | |
aac45363 | 981 | if (can_oom_reap) |
36324a99 | 982 | wake_oom_reaper(victim); |
aac45363 | 983 | |
880b7689 | 984 | mmdrop(mm); |
6b0c81b3 | 985 | put_task_struct(victim); |
1da177e4 | 986 | } |
647f2bdf | 987 | #undef K |
1da177e4 | 988 | |
309ed882 DR |
989 | /* |
990 | * Determines whether the kernel must panic because of the panic_on_oom sysctl. | |
991 | */ | |
7c5f64f8 VD |
992 | static void check_panic_on_oom(struct oom_control *oc, |
993 | enum oom_constraint constraint) | |
309ed882 DR |
994 | { |
995 | if (likely(!sysctl_panic_on_oom)) | |
996 | return; | |
997 | if (sysctl_panic_on_oom != 2) { | |
998 | /* | |
999 | * panic_on_oom == 1 only affects CONSTRAINT_NONE, the kernel | |
1000 | * does not panic for cpuset, mempolicy, or memcg allocation | |
1001 | * failures. | |
1002 | */ | |
1003 | if (constraint != CONSTRAINT_NONE) | |
1004 | return; | |
1005 | } | |
071a4bef | 1006 | /* Do not panic for oom kills triggered by sysrq */ |
db2a0dd7 | 1007 | if (is_sysrq_oom(oc)) |
071a4bef | 1008 | return; |
2a966b77 | 1009 | dump_header(oc, NULL); |
309ed882 DR |
1010 | panic("Out of memory: %s panic_on_oom is enabled\n", |
1011 | sysctl_panic_on_oom == 2 ? "compulsory" : "system-wide"); | |
1012 | } | |
1013 | ||
8bc719d3 MS |
1014 | static BLOCKING_NOTIFIER_HEAD(oom_notify_list); |
1015 | ||
1016 | int register_oom_notifier(struct notifier_block *nb) | |
1017 | { | |
1018 | return blocking_notifier_chain_register(&oom_notify_list, nb); | |
1019 | } | |
1020 | EXPORT_SYMBOL_GPL(register_oom_notifier); | |
1021 | ||
1022 | int unregister_oom_notifier(struct notifier_block *nb) | |
1023 | { | |
1024 | return blocking_notifier_chain_unregister(&oom_notify_list, nb); | |
1025 | } | |
1026 | EXPORT_SYMBOL_GPL(unregister_oom_notifier); | |
1027 | ||
1da177e4 | 1028 | /** |
6e0fc46d DR |
1029 | * out_of_memory - kill the "best" process when we run out of memory |
1030 | * @oc: pointer to struct oom_control | |
1da177e4 LT |
1031 | * |
1032 | * If we run out of memory, we have the choice between either | |
1033 | * killing a random task (bad), letting the system crash (worse) | |
1034 | * OR try to be smart about which process to kill. Note that we | |
1035 | * don't have to be perfect here, we just have to be good. | |
1036 | */ | |
6e0fc46d | 1037 | bool out_of_memory(struct oom_control *oc) |
1da177e4 | 1038 | { |
8bc719d3 | 1039 | unsigned long freed = 0; |
e3658932 | 1040 | enum oom_constraint constraint = CONSTRAINT_NONE; |
8bc719d3 | 1041 | |
dc56401f JW |
1042 | if (oom_killer_disabled) |
1043 | return false; | |
1044 | ||
7c5f64f8 VD |
1045 | if (!is_memcg_oom(oc)) { |
1046 | blocking_notifier_call_chain(&oom_notify_list, 0, &freed); | |
1047 | if (freed > 0) | |
1048 | /* Got some memory back in the last second. */ | |
1049 | return true; | |
1050 | } | |
1da177e4 | 1051 | |
7b98c2e4 | 1052 | /* |
9ff4868e DR |
1053 | * If current has a pending SIGKILL or is exiting, then automatically |
1054 | * select it. The goal is to allow it to allocate so that it may | |
1055 | * quickly exit and free its memory. | |
7b98c2e4 | 1056 | */ |
091f362c | 1057 | if (task_will_free_mem(current)) { |
16e95196 | 1058 | mark_oom_victim(current); |
1af8bb43 | 1059 | wake_oom_reaper(current); |
75e8f8b2 | 1060 | return true; |
7b98c2e4 DR |
1061 | } |
1062 | ||
3da88fb3 MH |
1063 | /* |
1064 | * The OOM killer does not compensate for IO-less reclaim. | |
1065 | * pagefault_out_of_memory lost its gfp context so we have to | |
1066 | * make sure exclude 0 mask - all other users should have at least | |
1067 | * ___GFP_DIRECT_RECLAIM to get here. | |
1068 | */ | |
06ad276a | 1069 | if (oc->gfp_mask && !(oc->gfp_mask & __GFP_FS)) |
3da88fb3 MH |
1070 | return true; |
1071 | ||
9b0f8b04 CL |
1072 | /* |
1073 | * Check if there were limitations on the allocation (only relevant for | |
7c5f64f8 | 1074 | * NUMA and memcg) that may require different handling. |
9b0f8b04 | 1075 | */ |
7c5f64f8 | 1076 | constraint = constrained_alloc(oc); |
6e0fc46d DR |
1077 | if (constraint != CONSTRAINT_MEMORY_POLICY) |
1078 | oc->nodemask = NULL; | |
2a966b77 | 1079 | check_panic_on_oom(oc, constraint); |
0aad4b31 | 1080 | |
7c5f64f8 VD |
1081 | if (!is_memcg_oom(oc) && sysctl_oom_kill_allocating_task && |
1082 | current->mm && !oom_unkillable_task(current, NULL, oc->nodemask) && | |
121d1ba0 | 1083 | current->signal->oom_score_adj != OOM_SCORE_ADJ_MIN) { |
6b0c81b3 | 1084 | get_task_struct(current); |
7c5f64f8 VD |
1085 | oc->chosen = current; |
1086 | oom_kill_process(oc, "Out of memory (oom_kill_allocating_task)"); | |
75e8f8b2 | 1087 | return true; |
0aad4b31 DR |
1088 | } |
1089 | ||
7c5f64f8 | 1090 | select_bad_process(oc); |
0aad4b31 | 1091 | /* Found nothing?!?! Either we hang forever, or we panic. */ |
7c5f64f8 | 1092 | if (!oc->chosen && !is_sysrq_oom(oc) && !is_memcg_oom(oc)) { |
2a966b77 | 1093 | dump_header(oc, NULL); |
0aad4b31 DR |
1094 | panic("Out of memory and no killable processes...\n"); |
1095 | } | |
7c5f64f8 VD |
1096 | if (oc->chosen && oc->chosen != (void *)-1UL) { |
1097 | oom_kill_process(oc, !is_memcg_oom(oc) ? "Out of memory" : | |
1098 | "Memory cgroup out of memory"); | |
75e8f8b2 DR |
1099 | /* |
1100 | * Give the killed process a good chance to exit before trying | |
1101 | * to allocate memory again. | |
1102 | */ | |
4f774b91 | 1103 | schedule_timeout_killable(1); |
75e8f8b2 | 1104 | } |
7c5f64f8 | 1105 | return !!oc->chosen; |
c32b3cbe MH |
1106 | } |
1107 | ||
e3658932 DR |
1108 | /* |
1109 | * The pagefault handler calls here because it is out of memory, so kill a | |
798fd756 VD |
1110 | * memory-hogging task. If oom_lock is held by somebody else, a parallel oom |
1111 | * killing is already in progress so do nothing. | |
e3658932 DR |
1112 | */ |
1113 | void pagefault_out_of_memory(void) | |
1114 | { | |
6e0fc46d DR |
1115 | struct oom_control oc = { |
1116 | .zonelist = NULL, | |
1117 | .nodemask = NULL, | |
2a966b77 | 1118 | .memcg = NULL, |
6e0fc46d DR |
1119 | .gfp_mask = 0, |
1120 | .order = 0, | |
6e0fc46d DR |
1121 | }; |
1122 | ||
49426420 | 1123 | if (mem_cgroup_oom_synchronize(true)) |
dc56401f | 1124 | return; |
3812c8c8 | 1125 | |
dc56401f JW |
1126 | if (!mutex_trylock(&oom_lock)) |
1127 | return; | |
a104808e | 1128 | out_of_memory(&oc); |
dc56401f | 1129 | mutex_unlock(&oom_lock); |
e3658932 | 1130 | } |
2b3cc5e2 | 1131 | |
1132 | void add_to_oom_reaper(struct task_struct *p) | |
1133 | { | |
1134 | if (!sysctl_reap_mem_on_sigkill) | |
1135 | return; | |
1136 | ||
1137 | p = find_lock_task_mm(p); | |
1138 | if (!p) | |
1139 | return; | |
1140 | ||
1141 | get_task_struct(p); | |
1142 | if (task_will_free_mem(p)) { | |
1143 | __mark_oom_victim(p); | |
1144 | wake_oom_reaper(p); | |
1145 | } | |
1146 | task_unlock(p); | |
1147 | put_task_struct(p); | |
1148 | } |