Commit | Line | Data |
---|---|---|
e73f8959 ON |
1 | #include <linux/spinlock.h> |
2 | #include <linux/task_work.h> | |
3 | #include <linux/tracehook.h> | |
4 | ||
5 | int | |
ac3d0da8 | 6 | task_work_add(struct task_struct *task, struct callback_head *work, bool notify) |
e73f8959 | 7 | { |
ac3d0da8 | 8 | struct callback_head *head; |
e73f8959 | 9 | /* |
ed3e694d AV |
10 | * Not inserting the new work if the task has already passed |
11 | * exit_task_work() is the responisbility of callers. | |
e73f8959 | 12 | */ |
ac3d0da8 ON |
13 | do { |
14 | head = ACCESS_ONCE(task->task_works); | |
15 | work->next = head; | |
16 | } while (cmpxchg(&task->task_works, head, work) != head); | |
e73f8959 | 17 | |
ed3e694d | 18 | if (notify) |
e73f8959 | 19 | set_notify_resume(task); |
ed3e694d | 20 | return 0; |
e73f8959 ON |
21 | } |
22 | ||
67d12145 | 23 | struct callback_head * |
e73f8959 ON |
24 | task_work_cancel(struct task_struct *task, task_work_func_t func) |
25 | { | |
ac3d0da8 ON |
26 | struct callback_head **pprev = &task->task_works; |
27 | struct callback_head *work = NULL; | |
e73f8959 | 28 | unsigned long flags; |
ac3d0da8 ON |
29 | /* |
30 | * If cmpxchg() fails we continue without updating pprev. | |
31 | * Either we raced with task_work_add() which added the | |
32 | * new entry before this work, we will find it again. Or | |
33 | * we raced with task_work_run(), *pprev == NULL. | |
34 | */ | |
e73f8959 | 35 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
ac3d0da8 ON |
36 | while ((work = ACCESS_ONCE(*pprev))) { |
37 | read_barrier_depends(); | |
38 | if (work->func != func) | |
39 | pprev = &work->next; | |
40 | else if (cmpxchg(pprev, work, work->next) == work) | |
41 | break; | |
e73f8959 | 42 | } |
e73f8959 | 43 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
ac3d0da8 ON |
44 | |
45 | return work; | |
e73f8959 ON |
46 | } |
47 | ||
48 | void task_work_run(void) | |
49 | { | |
50 | struct task_struct *task = current; | |
ac3d0da8 | 51 | struct callback_head *work, *head, *next; |
e73f8959 | 52 | |
ac3d0da8 ON |
53 | for (;;) { |
54 | work = xchg(&task->task_works, NULL); | |
55 | if (!work) | |
56 | break; | |
57 | /* | |
58 | * Synchronize with task_work_cancel(). It can't remove | |
59 | * the first entry == work, cmpxchg(task_works) should | |
60 | * fail, but it can play with *work and other entries. | |
61 | */ | |
62 | raw_spin_unlock_wait(&task->pi_lock); | |
63 | smp_mb(); | |
e73f8959 | 64 | |
ac3d0da8 ON |
65 | /* Reverse the list to run the works in fifo order */ |
66 | head = NULL; | |
67 | do { | |
68 | next = work->next; | |
69 | work->next = head; | |
70 | head = work; | |
71 | work = next; | |
72 | } while (work); | |
e73f8959 | 73 | |
ac3d0da8 ON |
74 | work = head; |
75 | do { | |
76 | next = work->next; | |
77 | work->func(work); | |
78 | work = next; | |
f341861f | 79 | cond_resched(); |
ac3d0da8 | 80 | } while (work); |
e73f8959 ON |
81 | } |
82 | } |