Commit | Line | Data |
---|---|---|
d8be8173 PM |
1 | /* |
2 | * Sleepable Read-Copy Update mechanism for mutual exclusion, | |
3 | * tiny version for non-preemptible single-CPU use. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License as published by | |
7 | * the Free Software Foundation; either version 2 of the License, or | |
8 | * (at your option) any later version. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License | |
16 | * along with this program; if not, you can access it online at | |
17 | * http://www.gnu.org/licenses/gpl-2.0.html. | |
18 | * | |
19 | * Copyright (C) IBM Corporation, 2017 | |
20 | * | |
21 | * Author: Paul McKenney <paulmck@us.ibm.com> | |
22 | */ | |
23 | ||
24 | #include <linux/export.h> | |
25 | #include <linux/mutex.h> | |
26 | #include <linux/preempt.h> | |
27 | #include <linux/rcupdate_wait.h> | |
28 | #include <linux/sched.h> | |
29 | #include <linux/delay.h> | |
30 | #include <linux/srcu.h> | |
31 | ||
32 | #include <linux/rcu_node_tree.h> | |
45753c5f | 33 | #include "rcu_segcblist.h" |
d8be8173 PM |
34 | #include "rcu.h" |
35 | ||
36 | static int init_srcu_struct_fields(struct srcu_struct *sp) | |
37 | { | |
38 | sp->srcu_lock_nesting[0] = 0; | |
39 | sp->srcu_lock_nesting[1] = 0; | |
40 | init_swait_queue_head(&sp->srcu_wq); | |
41 | sp->srcu_gp_seq = 0; | |
42 | rcu_segcblist_init(&sp->srcu_cblist); | |
43 | sp->srcu_gp_running = false; | |
44 | sp->srcu_gp_waiting = false; | |
45 | sp->srcu_idx = 0; | |
46 | INIT_WORK(&sp->srcu_work, srcu_drive_gp); | |
47 | return 0; | |
48 | } | |
49 | ||
50 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
51 | ||
52 | int __init_srcu_struct(struct srcu_struct *sp, const char *name, | |
53 | struct lock_class_key *key) | |
54 | { | |
55 | /* Don't re-initialize a lock while it is held. */ | |
56 | debug_check_no_locks_freed((void *)sp, sizeof(*sp)); | |
57 | lockdep_init_map(&sp->dep_map, name, key, 0); | |
58 | return init_srcu_struct_fields(sp); | |
59 | } | |
60 | EXPORT_SYMBOL_GPL(__init_srcu_struct); | |
61 | ||
62 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | |
63 | ||
64 | /* | |
65 | * init_srcu_struct - initialize a sleep-RCU structure | |
66 | * @sp: structure to initialize. | |
67 | * | |
68 | * Must invoke this on a given srcu_struct before passing that srcu_struct | |
69 | * to any other function. Each srcu_struct represents a separate domain | |
70 | * of SRCU protection. | |
71 | */ | |
72 | int init_srcu_struct(struct srcu_struct *sp) | |
73 | { | |
74 | return init_srcu_struct_fields(sp); | |
75 | } | |
76 | EXPORT_SYMBOL_GPL(init_srcu_struct); | |
77 | ||
78 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | |
79 | ||
80 | /* | |
81 | * cleanup_srcu_struct - deconstruct a sleep-RCU structure | |
82 | * @sp: structure to clean up. | |
83 | * | |
84 | * Must invoke this after you are finished using a given srcu_struct that | |
85 | * was initialized via init_srcu_struct(), else you leak memory. | |
86 | */ | |
87 | void cleanup_srcu_struct(struct srcu_struct *sp) | |
88 | { | |
89 | WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]); | |
90 | flush_work(&sp->srcu_work); | |
91 | WARN_ON(rcu_seq_state(sp->srcu_gp_seq)); | |
92 | WARN_ON(sp->srcu_gp_running); | |
93 | WARN_ON(sp->srcu_gp_waiting); | |
94 | WARN_ON(!rcu_segcblist_empty(&sp->srcu_cblist)); | |
95 | } | |
96 | EXPORT_SYMBOL_GPL(cleanup_srcu_struct); | |
97 | ||
98 | /* | |
99 | * Counts the new reader in the appropriate per-CPU element of the | |
cdf7abc4 PB |
100 | * srcu_struct. Can be invoked from irq/bh handlers, but the matching |
101 | * __srcu_read_unlock() must be in the same handler instance. Returns an | |
102 | * index that must be passed to the matching srcu_read_unlock(). | |
d8be8173 PM |
103 | */ |
104 | int __srcu_read_lock(struct srcu_struct *sp) | |
105 | { | |
106 | int idx; | |
107 | ||
108 | idx = READ_ONCE(sp->srcu_idx); | |
109 | WRITE_ONCE(sp->srcu_lock_nesting[idx], sp->srcu_lock_nesting[idx] + 1); | |
110 | return idx; | |
111 | } | |
112 | EXPORT_SYMBOL_GPL(__srcu_read_lock); | |
113 | ||
114 | /* | |
115 | * Removes the count for the old reader from the appropriate element of | |
cdf7abc4 | 116 | * the srcu_struct. |
d8be8173 PM |
117 | */ |
118 | void __srcu_read_unlock(struct srcu_struct *sp, int idx) | |
119 | { | |
120 | int newval = sp->srcu_lock_nesting[idx] - 1; | |
121 | ||
122 | WRITE_ONCE(sp->srcu_lock_nesting[idx], newval); | |
123 | if (!newval && READ_ONCE(sp->srcu_gp_waiting)) | |
124 | swake_up(&sp->srcu_wq); | |
125 | } | |
126 | EXPORT_SYMBOL_GPL(__srcu_read_unlock); | |
127 | ||
128 | /* | |
129 | * Workqueue handler to drive one grace period and invoke any callbacks | |
130 | * that become ready as a result. Single-CPU and !PREEMPT operation | |
131 | * means that we get away with murder on synchronization. ;-) | |
132 | */ | |
133 | void srcu_drive_gp(struct work_struct *wp) | |
134 | { | |
135 | int idx; | |
136 | struct rcu_cblist ready_cbs; | |
137 | struct srcu_struct *sp; | |
138 | struct rcu_head *rhp; | |
139 | ||
140 | sp = container_of(wp, struct srcu_struct, srcu_work); | |
141 | if (sp->srcu_gp_running || rcu_segcblist_empty(&sp->srcu_cblist)) | |
142 | return; /* Already running or nothing to do. */ | |
143 | ||
144 | /* Tag recently arrived callbacks and wait for readers. */ | |
145 | WRITE_ONCE(sp->srcu_gp_running, true); | |
146 | rcu_segcblist_accelerate(&sp->srcu_cblist, | |
147 | rcu_seq_snap(&sp->srcu_gp_seq)); | |
148 | rcu_seq_start(&sp->srcu_gp_seq); | |
149 | idx = sp->srcu_idx; | |
150 | WRITE_ONCE(sp->srcu_idx, !sp->srcu_idx); | |
151 | WRITE_ONCE(sp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ | |
152 | swait_event(sp->srcu_wq, !READ_ONCE(sp->srcu_lock_nesting[idx])); | |
153 | WRITE_ONCE(sp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */ | |
154 | rcu_seq_end(&sp->srcu_gp_seq); | |
155 | ||
156 | /* Update callback list based on GP, and invoke ready callbacks. */ | |
157 | rcu_segcblist_advance(&sp->srcu_cblist, | |
158 | rcu_seq_current(&sp->srcu_gp_seq)); | |
159 | if (rcu_segcblist_ready_cbs(&sp->srcu_cblist)) { | |
160 | rcu_cblist_init(&ready_cbs); | |
161 | local_irq_disable(); | |
162 | rcu_segcblist_extract_done_cbs(&sp->srcu_cblist, &ready_cbs); | |
163 | local_irq_enable(); | |
164 | rhp = rcu_cblist_dequeue(&ready_cbs); | |
165 | for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) { | |
166 | local_bh_disable(); | |
167 | rhp->func(rhp); | |
168 | local_bh_enable(); | |
169 | } | |
170 | local_irq_disable(); | |
171 | rcu_segcblist_insert_count(&sp->srcu_cblist, &ready_cbs); | |
172 | local_irq_enable(); | |
173 | } | |
174 | WRITE_ONCE(sp->srcu_gp_running, false); | |
175 | ||
176 | /* | |
177 | * If more callbacks, reschedule ourselves. This can race with | |
178 | * a call_srcu() at interrupt level, but the ->srcu_gp_running | |
179 | * checks will straighten that out. | |
180 | */ | |
181 | if (!rcu_segcblist_empty(&sp->srcu_cblist)) | |
182 | schedule_work(&sp->srcu_work); | |
183 | } | |
184 | EXPORT_SYMBOL_GPL(srcu_drive_gp); | |
185 | ||
186 | /* | |
187 | * Enqueue an SRCU callback on the specified srcu_struct structure, | |
188 | * initiating grace-period processing if it is not already running. | |
189 | */ | |
190 | void call_srcu(struct srcu_struct *sp, struct rcu_head *head, | |
191 | rcu_callback_t func) | |
192 | { | |
193 | unsigned long flags; | |
194 | ||
195 | head->func = func; | |
196 | local_irq_save(flags); | |
197 | rcu_segcblist_enqueue(&sp->srcu_cblist, head, false); | |
198 | local_irq_restore(flags); | |
199 | if (!READ_ONCE(sp->srcu_gp_running)) | |
200 | schedule_work(&sp->srcu_work); | |
201 | } | |
202 | EXPORT_SYMBOL_GPL(call_srcu); | |
203 | ||
204 | /* | |
205 | * synchronize_srcu - wait for prior SRCU read-side critical-section completion | |
206 | */ | |
207 | void synchronize_srcu(struct srcu_struct *sp) | |
208 | { | |
209 | struct rcu_synchronize rs; | |
210 | ||
211 | init_rcu_head_on_stack(&rs.head); | |
212 | init_completion(&rs.completion); | |
213 | call_srcu(sp, &rs.head, wakeme_after_rcu); | |
214 | wait_for_completion(&rs.completion); | |
215 | destroy_rcu_head_on_stack(&rs.head); | |
216 | } | |
217 | EXPORT_SYMBOL_GPL(synchronize_srcu); |