1 #include <linux/kdebug.h>
2 #include <linux/kprobes.h>
3 #include <linux/export.h>
4 #include <linux/notifier.h>
5 #include <linux/rcupdate.h>
6 #include <linux/vmalloc.h>
7 #include <linux/reboot.h>
10 * Notifier list for kernel code which wants to be called
11 * at shutdown. This is used to stop any idling DMA operations
14 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list
);
17 * Notifier chain core routines. The exported routines below
18 * are layered on top of these, with appropriate locking added.
21 static int notifier_chain_register(struct notifier_block
**nl
,
22 struct notifier_block
*n
)
24 while ((*nl
) != NULL
) {
25 if (n
->priority
> (*nl
)->priority
)
30 rcu_assign_pointer(*nl
, n
);
34 static int notifier_chain_cond_register(struct notifier_block
**nl
,
35 struct notifier_block
*n
)
37 while ((*nl
) != NULL
) {
40 if (n
->priority
> (*nl
)->priority
)
45 rcu_assign_pointer(*nl
, n
);
49 static int notifier_chain_unregister(struct notifier_block
**nl
,
50 struct notifier_block
*n
)
52 while ((*nl
) != NULL
) {
54 rcu_assign_pointer(*nl
, n
->next
);
63 * notifier_call_chain - Informs the registered notifiers about an event.
64 * @nl: Pointer to head of the blocking notifier chain
65 * @val: Value passed unmodified to notifier function
66 * @v: Pointer passed unmodified to notifier function
67 * @nr_to_call: Number of notifier functions to be called. Don't care
68 * value of this parameter is -1.
69 * @nr_calls: Records the number of notifications sent. Don't care
70 * value of this field is NULL.
71 * @returns: notifier_call_chain returns the value returned by the
72 * last notifier function called.
74 static int notifier_call_chain(struct notifier_block
**nl
,
75 unsigned long val
, void *v
,
76 int nr_to_call
, int *nr_calls
)
78 int ret
= NOTIFY_DONE
;
79 struct notifier_block
*nb
, *next_nb
;
81 nb
= rcu_dereference_raw(*nl
);
83 while (nb
&& nr_to_call
) {
84 next_nb
= rcu_dereference_raw(nb
->next
);
86 #ifdef CONFIG_DEBUG_NOTIFIERS
87 if (unlikely(!func_ptr_is_kernel_text(nb
->notifier_call
))) {
88 WARN(1, "Invalid notifier called!");
93 dbg_snapshot_print_notifier_call((void **)nl
,
94 (unsigned long)nb
->notifier_call
, DSS_FLAG_IN
);
95 ret
= nb
->notifier_call(nb
, val
, v
);
96 dbg_snapshot_print_notifier_call((void **)nl
,
97 (unsigned long)nb
->notifier_call
, DSS_FLAG_OUT
);
102 if (ret
& NOTIFY_STOP_MASK
)
109 NOKPROBE_SYMBOL(notifier_call_chain
);
112 * Atomic notifier chain routines. Registration and unregistration
113 * use a spinlock, and call_chain is synchronized by RCU (no locks).
117 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
118 * @nh: Pointer to head of the atomic notifier chain
119 * @n: New entry in notifier chain
121 * Adds a notifier to an atomic notifier chain.
123 * Currently always returns zero.
125 int atomic_notifier_chain_register(struct atomic_notifier_head
*nh
,
126 struct notifier_block
*n
)
131 spin_lock_irqsave(&nh
->lock
, flags
);
132 ret
= notifier_chain_register(&nh
->head
, n
);
133 spin_unlock_irqrestore(&nh
->lock
, flags
);
136 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register
);
139 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
140 * @nh: Pointer to head of the atomic notifier chain
141 * @n: Entry to remove from notifier chain
143 * Removes a notifier from an atomic notifier chain.
145 * Returns zero on success or %-ENOENT on failure.
147 int atomic_notifier_chain_unregister(struct atomic_notifier_head
*nh
,
148 struct notifier_block
*n
)
153 spin_lock_irqsave(&nh
->lock
, flags
);
154 ret
= notifier_chain_unregister(&nh
->head
, n
);
155 spin_unlock_irqrestore(&nh
->lock
, flags
);
159 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister
);
162 * __atomic_notifier_call_chain - Call functions in an atomic notifier chain
163 * @nh: Pointer to head of the atomic notifier chain
164 * @val: Value passed unmodified to notifier function
165 * @v: Pointer passed unmodified to notifier function
166 * @nr_to_call: See the comment for notifier_call_chain.
167 * @nr_calls: See the comment for notifier_call_chain.
169 * Calls each function in a notifier chain in turn. The functions
170 * run in an atomic context, so they must not block.
171 * This routine uses RCU to synchronize with changes to the chain.
173 * If the return value of the notifier can be and'ed
174 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
175 * will return immediately, with the return value of
176 * the notifier function which halted execution.
177 * Otherwise the return value is the return value
178 * of the last notifier function called.
180 int __atomic_notifier_call_chain(struct atomic_notifier_head
*nh
,
181 unsigned long val
, void *v
,
182 int nr_to_call
, int *nr_calls
)
187 ret
= notifier_call_chain(&nh
->head
, val
, v
, nr_to_call
, nr_calls
);
191 EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain
);
192 NOKPROBE_SYMBOL(__atomic_notifier_call_chain
);
194 int atomic_notifier_call_chain(struct atomic_notifier_head
*nh
,
195 unsigned long val
, void *v
)
197 return __atomic_notifier_call_chain(nh
, val
, v
, -1, NULL
);
199 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain
);
200 NOKPROBE_SYMBOL(atomic_notifier_call_chain
);
203 * Blocking notifier chain routines. All access to the chain is
204 * synchronized by an rwsem.
208 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
209 * @nh: Pointer to head of the blocking notifier chain
210 * @n: New entry in notifier chain
212 * Adds a notifier to a blocking notifier chain.
213 * Must be called in process context.
215 * Currently always returns zero.
217 int blocking_notifier_chain_register(struct blocking_notifier_head
*nh
,
218 struct notifier_block
*n
)
223 * This code gets used during boot-up, when task switching is
224 * not yet working and interrupts must remain disabled. At
225 * such times we must not call down_write().
227 if (unlikely(system_state
== SYSTEM_BOOTING
))
228 return notifier_chain_register(&nh
->head
, n
);
230 down_write(&nh
->rwsem
);
231 ret
= notifier_chain_register(&nh
->head
, n
);
232 up_write(&nh
->rwsem
);
235 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register
);
238 * blocking_notifier_chain_cond_register - Cond add notifier to a blocking notifier chain
239 * @nh: Pointer to head of the blocking notifier chain
240 * @n: New entry in notifier chain
242 * Adds a notifier to a blocking notifier chain, only if not already
243 * present in the chain.
244 * Must be called in process context.
246 * Currently always returns zero.
248 int blocking_notifier_chain_cond_register(struct blocking_notifier_head
*nh
,
249 struct notifier_block
*n
)
253 down_write(&nh
->rwsem
);
254 ret
= notifier_chain_cond_register(&nh
->head
, n
);
255 up_write(&nh
->rwsem
);
258 EXPORT_SYMBOL_GPL(blocking_notifier_chain_cond_register
);
261 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
262 * @nh: Pointer to head of the blocking notifier chain
263 * @n: Entry to remove from notifier chain
265 * Removes a notifier from a blocking notifier chain.
266 * Must be called from process context.
268 * Returns zero on success or %-ENOENT on failure.
270 int blocking_notifier_chain_unregister(struct blocking_notifier_head
*nh
,
271 struct notifier_block
*n
)
276 * This code gets used during boot-up, when task switching is
277 * not yet working and interrupts must remain disabled. At
278 * such times we must not call down_write().
280 if (unlikely(system_state
== SYSTEM_BOOTING
))
281 return notifier_chain_unregister(&nh
->head
, n
);
283 down_write(&nh
->rwsem
);
284 ret
= notifier_chain_unregister(&nh
->head
, n
);
285 up_write(&nh
->rwsem
);
288 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister
);
291 * __blocking_notifier_call_chain - Call functions in a blocking notifier chain
292 * @nh: Pointer to head of the blocking notifier chain
293 * @val: Value passed unmodified to notifier function
294 * @v: Pointer passed unmodified to notifier function
295 * @nr_to_call: See comment for notifier_call_chain.
296 * @nr_calls: See comment for notifier_call_chain.
298 * Calls each function in a notifier chain in turn. The functions
299 * run in a process context, so they are allowed to block.
301 * If the return value of the notifier can be and'ed
302 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
303 * will return immediately, with the return value of
304 * the notifier function which halted execution.
305 * Otherwise the return value is the return value
306 * of the last notifier function called.
308 int __blocking_notifier_call_chain(struct blocking_notifier_head
*nh
,
309 unsigned long val
, void *v
,
310 int nr_to_call
, int *nr_calls
)
312 int ret
= NOTIFY_DONE
;
315 * We check the head outside the lock, but if this access is
316 * racy then it does not matter what the result of the test
317 * is, we re-check the list after having taken the lock anyway:
319 if (rcu_access_pointer(nh
->head
)) {
320 down_read(&nh
->rwsem
);
321 ret
= notifier_call_chain(&nh
->head
, val
, v
, nr_to_call
,
327 EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain
);
329 int blocking_notifier_call_chain(struct blocking_notifier_head
*nh
,
330 unsigned long val
, void *v
)
332 return __blocking_notifier_call_chain(nh
, val
, v
, -1, NULL
);
334 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain
);
337 * Raw notifier chain routines. There is no protection;
338 * the caller must provide it. Use at your own risk!
342 * raw_notifier_chain_register - Add notifier to a raw notifier chain
343 * @nh: Pointer to head of the raw notifier chain
344 * @n: New entry in notifier chain
346 * Adds a notifier to a raw notifier chain.
347 * All locking must be provided by the caller.
349 * Currently always returns zero.
351 int raw_notifier_chain_register(struct raw_notifier_head
*nh
,
352 struct notifier_block
*n
)
354 return notifier_chain_register(&nh
->head
, n
);
356 EXPORT_SYMBOL_GPL(raw_notifier_chain_register
);
359 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
360 * @nh: Pointer to head of the raw notifier chain
361 * @n: Entry to remove from notifier chain
363 * Removes a notifier from a raw notifier chain.
364 * All locking must be provided by the caller.
366 * Returns zero on success or %-ENOENT on failure.
368 int raw_notifier_chain_unregister(struct raw_notifier_head
*nh
,
369 struct notifier_block
*n
)
371 return notifier_chain_unregister(&nh
->head
, n
);
373 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister
);
376 * __raw_notifier_call_chain - Call functions in a raw notifier chain
377 * @nh: Pointer to head of the raw notifier chain
378 * @val: Value passed unmodified to notifier function
379 * @v: Pointer passed unmodified to notifier function
380 * @nr_to_call: See comment for notifier_call_chain.
381 * @nr_calls: See comment for notifier_call_chain
383 * Calls each function in a notifier chain in turn. The functions
384 * run in an undefined context.
385 * All locking must be provided by the caller.
387 * If the return value of the notifier can be and'ed
388 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
389 * will return immediately, with the return value of
390 * the notifier function which halted execution.
391 * Otherwise the return value is the return value
392 * of the last notifier function called.
394 int __raw_notifier_call_chain(struct raw_notifier_head
*nh
,
395 unsigned long val
, void *v
,
396 int nr_to_call
, int *nr_calls
)
398 return notifier_call_chain(&nh
->head
, val
, v
, nr_to_call
, nr_calls
);
400 EXPORT_SYMBOL_GPL(__raw_notifier_call_chain
);
402 int raw_notifier_call_chain(struct raw_notifier_head
*nh
,
403 unsigned long val
, void *v
)
405 return __raw_notifier_call_chain(nh
, val
, v
, -1, NULL
);
407 EXPORT_SYMBOL_GPL(raw_notifier_call_chain
);
411 * SRCU notifier chain routines. Registration and unregistration
412 * use a mutex, and call_chain is synchronized by SRCU (no locks).
416 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
417 * @nh: Pointer to head of the SRCU notifier chain
418 * @n: New entry in notifier chain
420 * Adds a notifier to an SRCU notifier chain.
421 * Must be called in process context.
423 * Currently always returns zero.
425 int srcu_notifier_chain_register(struct srcu_notifier_head
*nh
,
426 struct notifier_block
*n
)
431 * This code gets used during boot-up, when task switching is
432 * not yet working and interrupts must remain disabled. At
433 * such times we must not call mutex_lock().
435 if (unlikely(system_state
== SYSTEM_BOOTING
))
436 return notifier_chain_register(&nh
->head
, n
);
438 mutex_lock(&nh
->mutex
);
439 ret
= notifier_chain_register(&nh
->head
, n
);
440 mutex_unlock(&nh
->mutex
);
443 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register
);
446 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
447 * @nh: Pointer to head of the SRCU notifier chain
448 * @n: Entry to remove from notifier chain
450 * Removes a notifier from an SRCU notifier chain.
451 * Must be called from process context.
453 * Returns zero on success or %-ENOENT on failure.
455 int srcu_notifier_chain_unregister(struct srcu_notifier_head
*nh
,
456 struct notifier_block
*n
)
461 * This code gets used during boot-up, when task switching is
462 * not yet working and interrupts must remain disabled. At
463 * such times we must not call mutex_lock().
465 if (unlikely(system_state
== SYSTEM_BOOTING
))
466 return notifier_chain_unregister(&nh
->head
, n
);
468 mutex_lock(&nh
->mutex
);
469 ret
= notifier_chain_unregister(&nh
->head
, n
);
470 mutex_unlock(&nh
->mutex
);
471 synchronize_srcu(&nh
->srcu
);
474 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister
);
477 * __srcu_notifier_call_chain - Call functions in an SRCU notifier chain
478 * @nh: Pointer to head of the SRCU notifier chain
479 * @val: Value passed unmodified to notifier function
480 * @v: Pointer passed unmodified to notifier function
481 * @nr_to_call: See comment for notifier_call_chain.
482 * @nr_calls: See comment for notifier_call_chain
484 * Calls each function in a notifier chain in turn. The functions
485 * run in a process context, so they are allowed to block.
487 * If the return value of the notifier can be and'ed
488 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
489 * will return immediately, with the return value of
490 * the notifier function which halted execution.
491 * Otherwise the return value is the return value
492 * of the last notifier function called.
494 int __srcu_notifier_call_chain(struct srcu_notifier_head
*nh
,
495 unsigned long val
, void *v
,
496 int nr_to_call
, int *nr_calls
)
501 idx
= srcu_read_lock(&nh
->srcu
);
502 ret
= notifier_call_chain(&nh
->head
, val
, v
, nr_to_call
, nr_calls
);
503 srcu_read_unlock(&nh
->srcu
, idx
);
506 EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain
);
508 int srcu_notifier_call_chain(struct srcu_notifier_head
*nh
,
509 unsigned long val
, void *v
)
511 return __srcu_notifier_call_chain(nh
, val
, v
, -1, NULL
);
513 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain
);
516 * srcu_init_notifier_head - Initialize an SRCU notifier head
517 * @nh: Pointer to head of the srcu notifier chain
519 * Unlike other sorts of notifier heads, SRCU notifier heads require
520 * dynamic initialization. Be sure to call this routine before
521 * calling any of the other SRCU notifier routines for this head.
523 * If an SRCU notifier head is deallocated, it must first be cleaned
524 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
525 * per-cpu data (used by the SRCU mechanism) will leak.
527 void srcu_init_notifier_head(struct srcu_notifier_head
*nh
)
529 mutex_init(&nh
->mutex
);
530 if (init_srcu_struct(&nh
->srcu
) < 0)
534 EXPORT_SYMBOL_GPL(srcu_init_notifier_head
);
536 #endif /* CONFIG_SRCU */
538 static ATOMIC_NOTIFIER_HEAD(die_chain
);
540 int notrace
notify_die(enum die_val val
, const char *str
,
541 struct pt_regs
*regs
, long err
, int trap
, int sig
)
543 struct die_args args
= {
551 RCU_LOCKDEP_WARN(!rcu_is_watching(),
552 "notify_die called but RCU thinks we're quiescent");
553 return atomic_notifier_call_chain(&die_chain
, val
, &args
);
555 NOKPROBE_SYMBOL(notify_die
);
557 int register_die_notifier(struct notifier_block
*nb
)
560 return atomic_notifier_chain_register(&die_chain
, nb
);
562 EXPORT_SYMBOL_GPL(register_die_notifier
);
564 int unregister_die_notifier(struct notifier_block
*nb
)
566 return atomic_notifier_chain_unregister(&die_chain
, nb
);
568 EXPORT_SYMBOL_GPL(unregister_die_notifier
);