Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * net/core/dst.c Protocol independent destination cache. | |
3 | * | |
4 | * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> | |
5 | * | |
6 | */ | |
7 | ||
8 | #include <linux/bitops.h> | |
9 | #include <linux/errno.h> | |
10 | #include <linux/init.h> | |
11 | #include <linux/kernel.h> | |
86bba269 | 12 | #include <linux/workqueue.h> |
1da177e4 LT |
13 | #include <linux/mm.h> |
14 | #include <linux/module.h> | |
5a0e3ad6 | 15 | #include <linux/slab.h> |
1da177e4 | 16 | #include <linux/netdevice.h> |
1da177e4 LT |
17 | #include <linux/skbuff.h> |
18 | #include <linux/string.h> | |
19 | #include <linux/types.h> | |
e9dc8653 | 20 | #include <net/net_namespace.h> |
2fc1b5dd | 21 | #include <linux/sched.h> |
268bb0ce | 22 | #include <linux/prefetch.h> |
1da177e4 LT |
23 | |
24 | #include <net/dst.h> | |
25 | ||
86bba269 ED |
26 | /* |
27 | * Theory of operations: | |
28 | * 1) We use a list, protected by a spinlock, to add | |
29 | * new entries from both BH and non-BH context. | |
30 | * 2) In order to keep spinlock held for a small delay, | |
31 | * we use a second list where are stored long lived | |
32 | * entries, that are handled by the garbage collect thread | |
33 | * fired by a workqueue. | |
34 | * 3) This list is guarded by a mutex, | |
35 | * so that the gc_task and dst_dev_event() can be synchronized. | |
1da177e4 | 36 | */ |
1da177e4 | 37 | |
86bba269 ED |
38 | /* |
39 | * We want to keep lock & list close together | |
40 | * to dirty as few cache lines as possible in __dst_free(). | |
41 | * As this is not a very strong hint, we dont force an alignment on SMP. | |
42 | */ | |
43 | static struct { | |
44 | spinlock_t lock; | |
598ed936 | 45 | struct dst_entry *list; |
86bba269 ED |
46 | unsigned long timer_inc; |
47 | unsigned long timer_expires; | |
48 | } dst_garbage = { | |
49 | .lock = __SPIN_LOCK_UNLOCKED(dst_garbage.lock), | |
50 | .timer_inc = DST_GC_MAX, | |
51 | }; | |
52 | static void dst_gc_task(struct work_struct *work); | |
598ed936 | 53 | static void ___dst_free(struct dst_entry *dst); |
1da177e4 | 54 | |
86bba269 | 55 | static DECLARE_DELAYED_WORK(dst_gc_work, dst_gc_task); |
1da177e4 | 56 | |
86bba269 ED |
57 | static DEFINE_MUTEX(dst_gc_mutex); |
58 | /* | |
59 | * long lived entries are maintained in this list, guarded by dst_gc_mutex | |
60 | */ | |
61 | static struct dst_entry *dst_busy_list; | |
62 | ||
63 | static void dst_gc_task(struct work_struct *work) | |
1da177e4 LT |
64 | { |
65 | int delayed = 0; | |
86bba269 ED |
66 | int work_performed = 0; |
67 | unsigned long expires = ~0L; | |
68 | struct dst_entry *dst, *next, head; | |
69 | struct dst_entry *last = &head; | |
1da177e4 | 70 | |
86bba269 ED |
71 | mutex_lock(&dst_gc_mutex); |
72 | next = dst_busy_list; | |
1da177e4 | 73 | |
86bba269 ED |
74 | loop: |
75 | while ((dst = next) != NULL) { | |
76 | next = dst->next; | |
77 | prefetch(&next->next); | |
2fc1b5dd | 78 | cond_resched(); |
86bba269 ED |
79 | if (likely(atomic_read(&dst->__refcnt))) { |
80 | last->next = dst; | |
81 | last = dst; | |
1da177e4 LT |
82 | delayed++; |
83 | continue; | |
84 | } | |
86bba269 | 85 | work_performed++; |
1da177e4 LT |
86 | |
87 | dst = dst_destroy(dst); | |
88 | if (dst) { | |
89 | /* NOHASH and still referenced. Unless it is already | |
90 | * on gc list, invalidate it and add to gc list. | |
91 | * | |
92 | * Note: this is temporary. Actually, NOHASH dst's | |
93 | * must be obsoleted when parent is obsoleted. | |
94 | * But we do not have state "obsoleted, but | |
95 | * referenced by parent", so it is right. | |
96 | */ | |
f5b0a874 | 97 | if (dst->obsolete > 0) |
1da177e4 LT |
98 | continue; |
99 | ||
100 | ___dst_free(dst); | |
86bba269 ED |
101 | dst->next = next; |
102 | next = dst; | |
1da177e4 LT |
103 | } |
104 | } | |
86bba269 ED |
105 | |
106 | spin_lock_bh(&dst_garbage.lock); | |
107 | next = dst_garbage.list; | |
108 | if (next) { | |
109 | dst_garbage.list = NULL; | |
110 | spin_unlock_bh(&dst_garbage.lock); | |
111 | goto loop; | |
1da177e4 | 112 | } |
86bba269 ED |
113 | last->next = NULL; |
114 | dst_busy_list = head.next; | |
115 | if (!dst_busy_list) | |
116 | dst_garbage.timer_inc = DST_GC_MAX; | |
117 | else { | |
118 | /* | |
119 | * if we freed less than 1/10 of delayed entries, | |
120 | * we can sleep longer. | |
121 | */ | |
122 | if (work_performed <= delayed/10) { | |
123 | dst_garbage.timer_expires += dst_garbage.timer_inc; | |
124 | if (dst_garbage.timer_expires > DST_GC_MAX) | |
125 | dst_garbage.timer_expires = DST_GC_MAX; | |
126 | dst_garbage.timer_inc += DST_GC_INC; | |
127 | } else { | |
128 | dst_garbage.timer_inc = DST_GC_INC; | |
129 | dst_garbage.timer_expires = DST_GC_MIN; | |
130 | } | |
131 | expires = dst_garbage.timer_expires; | |
132 | /* | |
598ed936 | 133 | * if the next desired timer is more than 4 seconds in the |
134 | * future then round the timer to whole seconds | |
86bba269 ED |
135 | */ |
136 | if (expires > 4*HZ) | |
137 | expires = round_jiffies_relative(expires); | |
138 | schedule_delayed_work(&dst_gc_work, expires); | |
f0098f78 | 139 | } |
86bba269 ED |
140 | |
141 | spin_unlock_bh(&dst_garbage.lock); | |
142 | mutex_unlock(&dst_gc_mutex); | |
1da177e4 LT |
143 | } |
144 | ||
352e512c | 145 | int dst_discard(struct sk_buff *skb) |
1da177e4 LT |
146 | { |
147 | kfree_skb(skb); | |
148 | return 0; | |
149 | } | |
352e512c | 150 | EXPORT_SYMBOL(dst_discard); |
1da177e4 | 151 | |
a37e6e34 ED |
152 | const u32 dst_default_metrics[RTAX_MAX + 1] = { |
153 | /* This initializer is needed to force linker to place this variable | |
154 | * into const section. Otherwise it might end into bss section. | |
155 | * We really want to avoid false sharing on this variable, and catch | |
156 | * any writes on it. | |
157 | */ | |
158 | [RTAX_MAX] = 0xdeadbeef, | |
159 | }; | |
160 | ||
62fa8a84 | 161 | |
5c1e6aa3 | 162 | void *dst_alloc(struct dst_ops *ops, struct net_device *dev, |
5110effe | 163 | int initial_ref, int initial_obsolete, unsigned short flags) |
1da177e4 | 164 | { |
598ed936 | 165 | struct dst_entry *dst; |
1da177e4 | 166 | |
fc66f95c | 167 | if (ops->gc && dst_entries_get_fast(ops) > ops->gc_thresh) { |
569d3645 | 168 | if (ops->gc(ops)) |
1da177e4 LT |
169 | return NULL; |
170 | } | |
cf911662 | 171 | dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC); |
1da177e4 LT |
172 | if (!dst) |
173 | return NULL; | |
cf911662 | 174 | dst->child = NULL; |
5c1e6aa3 DM |
175 | dst->dev = dev; |
176 | if (dev) | |
177 | dev_hold(dev); | |
1da177e4 | 178 | dst->ops = ops; |
62fa8a84 | 179 | dst_init_metrics(dst, dst_default_metrics, true); |
cf911662 | 180 | dst->expires = 0UL; |
5c1e6aa3 | 181 | dst->path = dst; |
ecd98837 | 182 | dst->from = NULL; |
cf911662 DM |
183 | #ifdef CONFIG_XFRM |
184 | dst->xfrm = NULL; | |
185 | #endif | |
5c1e6aa3 DM |
186 | dst->input = dst_discard; |
187 | dst->output = dst_discard; | |
cf911662 | 188 | dst->error = 0; |
5c1e6aa3 | 189 | dst->obsolete = initial_obsolete; |
cf911662 DM |
190 | dst->header_len = 0; |
191 | dst->trailer_len = 0; | |
192 | #ifdef CONFIG_IP_ROUTE_CLASSID | |
193 | dst->tclassid = 0; | |
1da177e4 | 194 | #endif |
5c1e6aa3 | 195 | atomic_set(&dst->__refcnt, initial_ref); |
cf911662 | 196 | dst->__use = 0; |
5c1e6aa3 DM |
197 | dst->lastuse = jiffies; |
198 | dst->flags = flags; | |
5110effe | 199 | dst->pending_confirm = 0; |
cf911662 | 200 | dst->next = NULL; |
957c665f DM |
201 | if (!(flags & DST_NOCOUNT)) |
202 | dst_entries_add(ops, 1); | |
1da177e4 LT |
203 | return dst; |
204 | } | |
598ed936 | 205 | EXPORT_SYMBOL(dst_alloc); |
1da177e4 | 206 | |
598ed936 | 207 | static void ___dst_free(struct dst_entry *dst) |
1da177e4 LT |
208 | { |
209 | /* The first case (dev==NULL) is required, when | |
210 | protocol module is unloaded. | |
211 | */ | |
598ed936 | 212 | if (dst->dev == NULL || !(dst->dev->flags&IFF_UP)) |
c4b1010f | 213 | dst->input = dst->output = dst_discard; |
f5b0a874 | 214 | dst->obsolete = DST_OBSOLETE_DEAD; |
1da177e4 LT |
215 | } |
216 | ||
598ed936 | 217 | void __dst_free(struct dst_entry *dst) |
1da177e4 | 218 | { |
86bba269 | 219 | spin_lock_bh(&dst_garbage.lock); |
1da177e4 | 220 | ___dst_free(dst); |
86bba269 ED |
221 | dst->next = dst_garbage.list; |
222 | dst_garbage.list = dst; | |
223 | if (dst_garbage.timer_inc > DST_GC_INC) { | |
224 | dst_garbage.timer_inc = DST_GC_INC; | |
225 | dst_garbage.timer_expires = DST_GC_MIN; | |
41f63c53 TH |
226 | mod_delayed_work(system_wq, &dst_gc_work, |
227 | dst_garbage.timer_expires); | |
1da177e4 | 228 | } |
86bba269 | 229 | spin_unlock_bh(&dst_garbage.lock); |
1da177e4 | 230 | } |
d79d9913 | 231 | EXPORT_SYMBOL(__dst_free); |
1da177e4 LT |
232 | |
233 | struct dst_entry *dst_destroy(struct dst_entry * dst) | |
234 | { | |
235 | struct dst_entry *child; | |
1da177e4 LT |
236 | |
237 | smp_rmb(); | |
238 | ||
239 | again: | |
1da177e4 LT |
240 | child = dst->child; |
241 | ||
957c665f DM |
242 | if (!(dst->flags & DST_NOCOUNT)) |
243 | dst_entries_add(dst->ops, -1); | |
1da177e4 LT |
244 | |
245 | if (dst->ops->destroy) | |
246 | dst->ops->destroy(dst); | |
247 | if (dst->dev) | |
248 | dev_put(dst->dev); | |
1da177e4 LT |
249 | kmem_cache_free(dst->ops->kmem_cachep, dst); |
250 | ||
251 | dst = child; | |
252 | if (dst) { | |
6775cab9 HX |
253 | int nohash = dst->flags & DST_NOHASH; |
254 | ||
1da177e4 LT |
255 | if (atomic_dec_and_test(&dst->__refcnt)) { |
256 | /* We were real parent of this dst, so kill child. */ | |
6775cab9 | 257 | if (nohash) |
1da177e4 LT |
258 | goto again; |
259 | } else { | |
260 | /* Child is still referenced, return it for freeing. */ | |
6775cab9 | 261 | if (nohash) |
1da177e4 LT |
262 | return dst; |
263 | /* Child is still in his hash table */ | |
264 | } | |
265 | } | |
266 | return NULL; | |
267 | } | |
598ed936 | 268 | EXPORT_SYMBOL(dst_destroy); |
1da177e4 | 269 | |
86e48c03 ED |
270 | static void dst_destroy_rcu(struct rcu_head *head) |
271 | { | |
272 | struct dst_entry *dst = container_of(head, struct dst_entry, rcu_head); | |
273 | ||
274 | dst = dst_destroy(dst); | |
275 | if (dst) | |
276 | __dst_free(dst); | |
277 | } | |
278 | ||
8d330868 IJ |
279 | void dst_release(struct dst_entry *dst) |
280 | { | |
281 | if (dst) { | |
598ed936 | 282 | int newrefcnt; |
30880ce6 | 283 | unsigned short nocache = dst->flags & DST_NOCACHE; |
ef711cf1 | 284 | |
598ed936 | 285 | newrefcnt = atomic_dec_return(&dst->__refcnt); |
155d792d KK |
286 | if (unlikely(newrefcnt < 0)) |
287 | net_warn_ratelimited("%s: dst:%p refcnt:%d\n", | |
288 | __func__, dst, newrefcnt); | |
30880ce6 | 289 | if (!newrefcnt && unlikely(nocache)) |
86e48c03 | 290 | call_rcu(&dst->rcu_head, dst_destroy_rcu); |
8d330868 IJ |
291 | } |
292 | } | |
293 | EXPORT_SYMBOL(dst_release); | |
294 | ||
62fa8a84 DM |
295 | u32 *dst_cow_metrics_generic(struct dst_entry *dst, unsigned long old) |
296 | { | |
297 | u32 *p = kmalloc(sizeof(u32) * RTAX_MAX, GFP_ATOMIC); | |
298 | ||
299 | if (p) { | |
300 | u32 *old_p = __DST_METRICS_PTR(old); | |
301 | unsigned long prev, new; | |
302 | ||
303 | memcpy(p, old_p, sizeof(u32) * RTAX_MAX); | |
304 | ||
305 | new = (unsigned long) p; | |
306 | prev = cmpxchg(&dst->_metrics, old, new); | |
307 | ||
308 | if (prev != old) { | |
309 | kfree(p); | |
310 | p = __DST_METRICS_PTR(prev); | |
311 | if (prev & DST_METRICS_READ_ONLY) | |
312 | p = NULL; | |
313 | } | |
314 | } | |
315 | return p; | |
316 | } | |
317 | EXPORT_SYMBOL(dst_cow_metrics_generic); | |
318 | ||
319 | /* Caller asserts that dst_metrics_read_only(dst) is false. */ | |
320 | void __dst_destroy_metrics_generic(struct dst_entry *dst, unsigned long old) | |
321 | { | |
322 | unsigned long prev, new; | |
323 | ||
b30c516f | 324 | new = ((unsigned long) dst_default_metrics) | DST_METRICS_READ_ONLY; |
62fa8a84 DM |
325 | prev = cmpxchg(&dst->_metrics, old, new); |
326 | if (prev == old) | |
327 | kfree(__DST_METRICS_PTR(old)); | |
328 | } | |
329 | EXPORT_SYMBOL(__dst_destroy_metrics_generic); | |
330 | ||
27b75c95 | 331 | /** |
932bc4d7 | 332 | * __skb_dst_set_noref - sets skb dst, without a reference |
27b75c95 ED |
333 | * @skb: buffer |
334 | * @dst: dst entry | |
932bc4d7 | 335 | * @force: if force is set, use noref version even for DST_NOCACHE entries |
27b75c95 ED |
336 | * |
337 | * Sets skb dst, assuming a reference was not taken on dst | |
338 | * skb_dst_drop() should not dst_release() this dst | |
339 | */ | |
932bc4d7 | 340 | void __skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst, bool force) |
27b75c95 ED |
341 | { |
342 | WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); | |
343 | /* If dst not in cache, we must take a reference, because | |
344 | * dst_release() will destroy dst as soon as its refcount becomes zero | |
345 | */ | |
932bc4d7 | 346 | if (unlikely((dst->flags & DST_NOCACHE) && !force)) { |
27b75c95 ED |
347 | dst_hold(dst); |
348 | skb_dst_set(skb, dst); | |
349 | } else { | |
350 | skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF; | |
351 | } | |
352 | } | |
932bc4d7 | 353 | EXPORT_SYMBOL(__skb_dst_set_noref); |
27b75c95 | 354 | |
1da177e4 LT |
355 | /* Dirty hack. We did it in 2.2 (in __dst_free), |
356 | * we have _very_ good reasons not to repeat | |
357 | * this mistake in 2.3, but we have no choice | |
358 | * now. _It_ _is_ _explicit_ _deliberate_ | |
359 | * _race_ _condition_. | |
360 | * | |
361 | * Commented and originally written by Alexey. | |
362 | */ | |
56115511 | 363 | static void dst_ifdown(struct dst_entry *dst, struct net_device *dev, |
364 | int unregister) | |
1da177e4 LT |
365 | { |
366 | if (dst->ops->ifdown) | |
367 | dst->ops->ifdown(dst, dev, unregister); | |
368 | ||
369 | if (dev != dst->dev) | |
370 | return; | |
371 | ||
372 | if (!unregister) { | |
c4b1010f | 373 | dst->input = dst->output = dst_discard; |
1da177e4 | 374 | } else { |
c346dca1 | 375 | dst->dev = dev_net(dst->dev)->loopback_dev; |
de3cb747 | 376 | dev_hold(dst->dev); |
1da177e4 | 377 | dev_put(dev); |
1da177e4 LT |
378 | } |
379 | } | |
380 | ||
598ed936 | 381 | static int dst_dev_event(struct notifier_block *this, unsigned long event, |
382 | void *ptr) | |
1da177e4 LT |
383 | { |
384 | struct net_device *dev = ptr; | |
86bba269 | 385 | struct dst_entry *dst, *last = NULL; |
1da177e4 LT |
386 | |
387 | switch (event) { | |
0115e8e3 | 388 | case NETDEV_UNREGISTER_FINAL: |
1da177e4 | 389 | case NETDEV_DOWN: |
86bba269 ED |
390 | mutex_lock(&dst_gc_mutex); |
391 | for (dst = dst_busy_list; dst; dst = dst->next) { | |
392 | last = dst; | |
393 | dst_ifdown(dst, dev, event != NETDEV_DOWN); | |
394 | } | |
395 | ||
396 | spin_lock_bh(&dst_garbage.lock); | |
397 | dst = dst_garbage.list; | |
398 | dst_garbage.list = NULL; | |
399 | spin_unlock_bh(&dst_garbage.lock); | |
400 | ||
401 | if (last) | |
402 | last->next = dst; | |
403 | else | |
404 | dst_busy_list = dst; | |
598ed936 | 405 | for (; dst; dst = dst->next) |
1da177e4 | 406 | dst_ifdown(dst, dev, event != NETDEV_DOWN); |
86bba269 | 407 | mutex_unlock(&dst_gc_mutex); |
1da177e4 LT |
408 | break; |
409 | } | |
410 | return NOTIFY_DONE; | |
411 | } | |
412 | ||
413 | static struct notifier_block dst_dev_notifier = { | |
414 | .notifier_call = dst_dev_event, | |
332dd96f | 415 | .priority = -10, /* must be called after other network notifiers */ |
1da177e4 LT |
416 | }; |
417 | ||
418 | void __init dst_init(void) | |
419 | { | |
420 | register_netdevice_notifier(&dst_dev_notifier); | |
421 | } |