projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
packets: Always register packet sk in the same order
[GitHub/LineageOS/android_kernel_motorola_exynos9610.git] / net / ipv6 / ip6_fib.c
1 /*
2 * Linux INET6 implementation
3 * Forwarding Information Database
4 *
5 * Authors:
6 * Pedro Roque <roque@di.fc.ul.pt>
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version
11 * 2 of the License, or (at your option) any later version.
12 *
13 * Changes:
14 * Yuji SEKIYA @USAGI: Support default route on router node;
15 * remove ip6_null_entry from the top of
16 * routing table.
17 * Ville Nuorvala: Fixed routing subtrees.
18 */
19
20 #define pr_fmt(fmt) "IPv6: " fmt
21
22 #include <linux/errno.h>
23 #include <linux/types.h>
24 #include <linux/net.h>
25 #include <linux/route.h>
26 #include <linux/netdevice.h>
27 #include <linux/in6.h>
28 #include <linux/init.h>
29 #include <linux/list.h>
30 #include <linux/slab.h>
31
32 #include <net/ipv6.h>
33 #include <net/ndisc.h>
34 #include <net/addrconf.h>
35 #include <net/lwtunnel.h>
36 #include <net/fib_notifier.h>
37
38 #include <net/ip6_fib.h>
39 #include <net/ip6_route.h>
40
41 #define RT6_DEBUG 2
42
43 #if RT6_DEBUG >= 3
44 #define RT6_TRACE(x...) pr_debug(x)
45 #else
46 #define RT6_TRACE(x...) do { ; } while (0)
47 #endif
48
49 static struct kmem_cache *fib6_node_kmem __read_mostly;
50
51 struct fib6_cleaner {
52 struct fib6_walker w;
53 struct net *net;
54 int (*func)(struct rt6_info *, void *arg);
55 int sernum;
56 void *arg;
57 };
58
59 #ifdef CONFIG_IPV6_SUBTREES
60 #define FWS_INIT FWS_S
61 #else
62 #define FWS_INIT FWS_L
63 #endif
64
65 static void fib6_prune_clones(struct net *net, struct fib6_node *fn);
66 static struct rt6_info *fib6_find_prefix(struct net *net, struct fib6_node *fn);
67 static struct fib6_node *fib6_repair_tree(struct net *net, struct fib6_node *fn);
68 static int fib6_walk(struct net *net, struct fib6_walker *w);
69 static int fib6_walk_continue(struct fib6_walker *w);
70
71 /*
72 * A routing update causes an increase of the serial number on the
73 * affected subtree. This allows for cached routes to be asynchronously
74 * tested when modifications are made to the destination cache as a
75 * result of redirects, path MTU changes, etc.
76 */
77
78 static void fib6_gc_timer_cb(unsigned long arg);
79
80 #define FOR_WALKERS(net, w) \
81 list_for_each_entry(w, &(net)->ipv6.fib6_walkers, lh)
82
83 static void fib6_walker_link(struct net *net, struct fib6_walker *w)
84 {
85 write_lock_bh(&net->ipv6.fib6_walker_lock);
86 list_add(&w->lh, &net->ipv6.fib6_walkers);
87 write_unlock_bh(&net->ipv6.fib6_walker_lock);
88 }
89
90 static void fib6_walker_unlink(struct net *net, struct fib6_walker *w)
91 {
92 write_lock_bh(&net->ipv6.fib6_walker_lock);
93 list_del(&w->lh);
94 write_unlock_bh(&net->ipv6.fib6_walker_lock);
95 }
96
97 static int fib6_new_sernum(struct net *net)
98 {
99 int new, old;
100
101 do {
102 old = atomic_read(&net->ipv6.fib6_sernum);
103 new = old < INT_MAX ? old + 1 : 1;
104 } while (atomic_cmpxchg(&net->ipv6.fib6_sernum,
105 old, new) != old);
106 return new;
107 }
108
109 enum {
110 FIB6_NO_SERNUM_CHANGE = 0,
111 };
112
113 /*
114 * Auxiliary address test functions for the radix tree.
115 *
116 * These assume a 32bit processor (although it will work on
117 * 64bit processors)
118 */
119
120 /*
121 * test bit
122 */
123 #if defined(__LITTLE_ENDIAN)
124 # define BITOP_BE32_SWIZZLE (0x1F & ~7)
125 #else
126 # define BITOP_BE32_SWIZZLE 0
127 #endif
128
129 static __be32 addr_bit_set(const void *token, int fn_bit)
130 {
131 const __be32 *addr = token;
132 /*
133 * Here,
134 * 1 << ((~fn_bit ^ BITOP_BE32_SWIZZLE) & 0x1f)
135 * is optimized version of
136 * htonl(1 << ((~fn_bit)&0x1F))
137 * See include/asm-generic/bitops/le.h.
138 */
139 return (__force __be32)(1 << ((~fn_bit ^ BITOP_BE32_SWIZZLE) & 0x1f)) &
140 addr[fn_bit >> 5];
141 }
142
143 static struct fib6_node *node_alloc(void)
144 {
145 struct fib6_node *fn;
146
147 fn = kmem_cache_zalloc(fib6_node_kmem, GFP_ATOMIC);
148
149 return fn;
150 }
151
152 static void node_free_immediate(struct fib6_node *fn)
153 {
154 kmem_cache_free(fib6_node_kmem, fn);
155 }
156
157 static void node_free_rcu(struct rcu_head *head)
158 {
159 struct fib6_node *fn = container_of(head, struct fib6_node, rcu);
160
161 kmem_cache_free(fib6_node_kmem, fn);
162 }
163
164 static void node_free(struct fib6_node *fn)
165 {
166 call_rcu(&fn->rcu, node_free_rcu);
167 }
168
169 void rt6_free_pcpu(struct rt6_info *non_pcpu_rt)
170 {
171 int cpu;
172
173 if (!non_pcpu_rt->rt6i_pcpu)
174 return;
175
176 for_each_possible_cpu(cpu) {
177 struct rt6_info **ppcpu_rt;
178 struct rt6_info *pcpu_rt;
179
180 ppcpu_rt = per_cpu_ptr(non_pcpu_rt->rt6i_pcpu, cpu);
181 pcpu_rt = *ppcpu_rt;
182 if (pcpu_rt) {
183 dst_dev_put(&pcpu_rt->dst);
184 dst_release(&pcpu_rt->dst);
185 *ppcpu_rt = NULL;
186 }
187 }
188
189 free_percpu(non_pcpu_rt->rt6i_pcpu);
190 non_pcpu_rt->rt6i_pcpu = NULL;
191 }
192 EXPORT_SYMBOL_GPL(rt6_free_pcpu);
193
194 static void fib6_free_table(struct fib6_table *table)
195 {
196 inetpeer_invalidate_tree(&table->tb6_peers);
197 kfree(table);
198 }
199
200 static void fib6_link_table(struct net *net, struct fib6_table *tb)
201 {
202 unsigned int h;
203
204 /*
205 * Initialize table lock at a single place to give lockdep a key,
206 * tables aren't visible prior to being linked to the list.
207 */
208 rwlock_init(&tb->tb6_lock);
209
210 h = tb->tb6_id & (FIB6_TABLE_HASHSZ - 1);
211
212 /*
213 * No protection necessary, this is the only list mutatation
214 * operation, tables never disappear once they exist.
215 */
216 hlist_add_head_rcu(&tb->tb6_hlist, &net->ipv6.fib_table_hash[h]);
217 }
218
219 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
220
221 static struct fib6_table *fib6_alloc_table(struct net *net, u32 id)
222 {
223 struct fib6_table *table;
224
225 table = kzalloc(sizeof(*table), GFP_ATOMIC);
226 if (table) {
227 table->tb6_id = id;
228 table->tb6_root.leaf = net->ipv6.ip6_null_entry;
229 table->tb6_root.fn_flags = RTN_ROOT | RTN_TL_ROOT | RTN_RTINFO;
230 inet_peer_base_init(&table->tb6_peers);
231 }
232
233 return table;
234 }
235
236 struct fib6_table *fib6_new_table(struct net *net, u32 id)
237 {
238 struct fib6_table *tb;
239
240 if (id == 0)
241 id = RT6_TABLE_MAIN;
242 tb = fib6_get_table(net, id);
243 if (tb)
244 return tb;
245
246 tb = fib6_alloc_table(net, id);
247 if (tb)
248 fib6_link_table(net, tb);
249
250 return tb;
251 }
252 EXPORT_SYMBOL_GPL(fib6_new_table);
253
254 struct fib6_table *fib6_get_table(struct net *net, u32 id)
255 {
256 struct fib6_table *tb;
257 struct hlist_head *head;
258 unsigned int h;
259
260 if (id == 0)
261 id = RT6_TABLE_MAIN;
262 h = id & (FIB6_TABLE_HASHSZ - 1);
263 rcu_read_lock();
264 head = &net->ipv6.fib_table_hash[h];
265 hlist_for_each_entry_rcu(tb, head, tb6_hlist) {
266 if (tb->tb6_id == id) {
267 rcu_read_unlock();
268 return tb;
269 }
270 }
271 rcu_read_unlock();
272
273 return NULL;
274 }
275 EXPORT_SYMBOL_GPL(fib6_get_table);
276
277 static void __net_init fib6_tables_init(struct net *net)
278 {
279 fib6_link_table(net, net->ipv6.fib6_main_tbl);
280 fib6_link_table(net, net->ipv6.fib6_local_tbl);
281 }
282 #else
283
284 struct fib6_table *fib6_new_table(struct net *net, u32 id)
285 {
286 return fib6_get_table(net, id);
287 }
288
289 struct fib6_table *fib6_get_table(struct net *net, u32 id)
290 {
291 return net->ipv6.fib6_main_tbl;
292 }
293
294 struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6,
295 int flags, pol_lookup_t lookup)
296 {
297 struct rt6_info *rt;
298
299 rt = lookup(net, net->ipv6.fib6_main_tbl, fl6, flags);
300 if (rt->dst.error == -EAGAIN) {
301 ip6_rt_put(rt);
302 rt = net->ipv6.ip6_null_entry;
303 dst_hold(&rt->dst);
304 }
305
306 return &rt->dst;
307 }
308
309 static void __net_init fib6_tables_init(struct net *net)
310 {
311 fib6_link_table(net, net->ipv6.fib6_main_tbl);
312 }
313
314 #endif
315
316 unsigned int fib6_tables_seq_read(struct net *net)
317 {
318 unsigned int h, fib_seq = 0;
319
320 rcu_read_lock();
321 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
322 struct hlist_head *head = &net->ipv6.fib_table_hash[h];
323 struct fib6_table *tb;
324
325 hlist_for_each_entry_rcu(tb, head, tb6_hlist) {
326 read_lock_bh(&tb->tb6_lock);
327 fib_seq += tb->fib_seq;
328 read_unlock_bh(&tb->tb6_lock);
329 }
330 }
331 rcu_read_unlock();
332
333 return fib_seq;
334 }
335
336 static int call_fib6_entry_notifier(struct notifier_block *nb, struct net *net,
337 enum fib_event_type event_type,
338 struct rt6_info *rt)
339 {
340 struct fib6_entry_notifier_info info = {
341 .rt = rt,
342 };
343
344 return call_fib6_notifier(nb, net, event_type, &info.info);
345 }
346
347 static int call_fib6_entry_notifiers(struct net *net,
348 enum fib_event_type event_type,
349 struct rt6_info *rt)
350 {
351 struct fib6_entry_notifier_info info = {
352 .rt = rt,
353 };
354
355 rt->rt6i_table->fib_seq++;
356 return call_fib6_notifiers(net, event_type, &info.info);
357 }
358
359 struct fib6_dump_arg {
360 struct net *net;
361 struct notifier_block *nb;
362 };
363
364 static void fib6_rt_dump(struct rt6_info *rt, struct fib6_dump_arg *arg)
365 {
366 if (rt == arg->net->ipv6.ip6_null_entry)
367 return;
368 call_fib6_entry_notifier(arg->nb, arg->net, FIB_EVENT_ENTRY_ADD, rt);
369 }
370
371 static int fib6_node_dump(struct fib6_walker *w)
372 {
373 struct rt6_info *rt;
374
375 for (rt = w->leaf; rt; rt = rt->dst.rt6_next)
376 fib6_rt_dump(rt, w->args);
377 w->leaf = NULL;
378 return 0;
379 }
380
381 static void fib6_table_dump(struct net *net, struct fib6_table *tb,
382 struct fib6_walker *w)
383 {
384 w->root = &tb->tb6_root;
385 read_lock_bh(&tb->tb6_lock);
386 fib6_walk(net, w);
387 read_unlock_bh(&tb->tb6_lock);
388 }
389
390 /* Called with rcu_read_lock() */
391 int fib6_tables_dump(struct net *net, struct notifier_block *nb)
392 {
393 struct fib6_dump_arg arg;
394 struct fib6_walker *w;
395 unsigned int h;
396
397 w = kzalloc(sizeof(*w), GFP_ATOMIC);
398 if (!w)
399 return -ENOMEM;
400
401 w->func = fib6_node_dump;
402 arg.net = net;
403 arg.nb = nb;
404 w->args = &arg;
405
406 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
407 struct hlist_head *head = &net->ipv6.fib_table_hash[h];
408 struct fib6_table *tb;
409
410 hlist_for_each_entry_rcu(tb, head, tb6_hlist)
411 fib6_table_dump(net, tb, w);
412 }
413
414 kfree(w);
415
416 return 0;
417 }
418
419 static int fib6_dump_node(struct fib6_walker *w)
420 {
421 int res;
422 struct rt6_info *rt;
423
424 for (rt = w->leaf; rt; rt = rt->dst.rt6_next) {
425 res = rt6_dump_route(rt, w->args);
426 if (res < 0) {
427 /* Frame is full, suspend walking */
428 w->leaf = rt;
429 return 1;
430 }
431
432 /* Multipath routes are dumped in one route with the
433 * RTA_MULTIPATH attribute. Jump 'rt' to point to the
434 * last sibling of this route (no need to dump the
435 * sibling routes again)
436 */
437 if (rt->rt6i_nsiblings)
438 rt = list_last_entry(&rt->rt6i_siblings,
439 struct rt6_info,
440 rt6i_siblings);
441 }
442 w->leaf = NULL;
443 return 0;
444 }
445
446 static void fib6_dump_end(struct netlink_callback *cb)
447 {
448 struct net *net = sock_net(cb->skb->sk);
449 struct fib6_walker *w = (void *)cb->args[2];
450
451 if (w) {
452 if (cb->args[4]) {
453 cb->args[4] = 0;
454 fib6_walker_unlink(net, w);
455 }
456 cb->args[2] = 0;
457 kfree(w);
458 }
459 cb->done = (void *)cb->args[3];
460 cb->args[1] = 3;
461 }
462
463 static int fib6_dump_done(struct netlink_callback *cb)
464 {
465 fib6_dump_end(cb);
466 return cb->done ? cb->done(cb) : 0;
467 }
468
469 static int fib6_dump_table(struct fib6_table *table, struct sk_buff *skb,
470 struct netlink_callback *cb)
471 {
472 struct net *net = sock_net(skb->sk);
473 struct fib6_walker *w;
474 int res;
475
476 w = (void *)cb->args[2];
477 w->root = &table->tb6_root;
478
479 if (cb->args[4] == 0) {
480 w->count = 0;
481 w->skip = 0;
482
483 read_lock_bh(&table->tb6_lock);
484 res = fib6_walk(net, w);
485 read_unlock_bh(&table->tb6_lock);
486 if (res > 0) {
487 cb->args[4] = 1;
488 cb->args[5] = w->root->fn_sernum;
489 }
490 } else {
491 if (cb->args[5] != w->root->fn_sernum) {
492 /* Begin at the root if the tree changed */
493 cb->args[5] = w->root->fn_sernum;
494 w->state = FWS_INIT;
495 w->node = w->root;
496 w->skip = w->count;
497 } else
498 w->skip = 0;
499
500 read_lock_bh(&table->tb6_lock);
501 res = fib6_walk_continue(w);
502 read_unlock_bh(&table->tb6_lock);
503 if (res <= 0) {
504 fib6_walker_unlink(net, w);
505 cb->args[4] = 0;
506 }
507 }
508
509 return res;
510 }
511
512 static int inet6_dump_fib(struct sk_buff *skb, struct netlink_callback *cb)
513 {
514 struct net *net = sock_net(skb->sk);
515 unsigned int h, s_h;
516 unsigned int e = 0, s_e;
517 struct rt6_rtnl_dump_arg arg;
518 struct fib6_walker *w;
519 struct fib6_table *tb;
520 struct hlist_head *head;
521 int res = 0;
522
523 s_h = cb->args[0];
524 s_e = cb->args[1];
525
526 w = (void *)cb->args[2];
527 if (!w) {
528 /* New dump:
529 *
530 * 1. hook callback destructor.
531 */
532 cb->args[3] = (long)cb->done;
533 cb->done = fib6_dump_done;
534
535 /*
536 * 2. allocate and initialize walker.
537 */
538 w = kzalloc(sizeof(*w), GFP_ATOMIC);
539 if (!w)
540 return -ENOMEM;
541 w->func = fib6_dump_node;
542 cb->args[2] = (long)w;
543 }
544
545 arg.skb = skb;
546 arg.cb = cb;
547 arg.net = net;
548 w->args = &arg;
549
550 rcu_read_lock();
551 for (h = s_h; h < FIB6_TABLE_HASHSZ; h++, s_e = 0) {
552 e = 0;
553 head = &net->ipv6.fib_table_hash[h];
554 hlist_for_each_entry_rcu(tb, head, tb6_hlist) {
555 if (e < s_e)
556 goto next;
557 res = fib6_dump_table(tb, skb, cb);
558 if (res != 0)
559 goto out;
560 next:
561 e++;
562 }
563 }
564 out:
565 rcu_read_unlock();
566 cb->args[1] = e;
567 cb->args[0] = h;
568
569 res = res < 0 ? res : skb->len;
570 if (res <= 0)
571 fib6_dump_end(cb);
572 return res;
573 }
574
575 /*
576 * Routing Table
577 *
578 * return the appropriate node for a routing tree "add" operation
579 * by either creating and inserting or by returning an existing
580 * node.
581 */
582
583 static struct fib6_node *fib6_add_1(struct fib6_node *root,
584 struct in6_addr *addr, int plen,
585 int offset, int allow_create,
586 int replace_required, int sernum,
587 struct netlink_ext_ack *extack)
588 {
589 struct fib6_node *fn, *in, *ln;
590 struct fib6_node *pn = NULL;
591 struct rt6key *key;
592 int bit;
593 __be32 dir = 0;
594
595 RT6_TRACE("fib6_add_1\n");
596
597 /* insert node in tree */
598
599 fn = root;
600
601 do {
602 key = (struct rt6key *)((u8 *)fn->leaf + offset);
603
604 /*
605 * Prefix match
606 */
607 if (plen < fn->fn_bit ||
608 !ipv6_prefix_equal(&key->addr, addr, fn->fn_bit)) {
609 if (!allow_create) {
610 if (replace_required) {
611 NL_SET_ERR_MSG(extack,
612 "Can not replace route - no match found");
613 pr_warn("Can't replace route, no match found\n");
614 return ERR_PTR(-ENOENT);
615 }
616 pr_warn("NLM_F_CREATE should be set when creating new route\n");
617 }
618 goto insert_above;
619 }
620
621 /*
622 * Exact match ?
623 */
624
625 if (plen == fn->fn_bit) {
626 /* clean up an intermediate node */
627 if (!(fn->fn_flags & RTN_RTINFO)) {
628 rt6_release(fn->leaf);
629 fn->leaf = NULL;
630 }
631
632 fn->fn_sernum = sernum;
633
634 return fn;
635 }
636
637 /*
638 * We have more bits to go
639 */
640
641 /* Try to walk down on tree. */
642 fn->fn_sernum = sernum;
643 dir = addr_bit_set(addr, fn->fn_bit);
644 pn = fn;
645 fn = dir ? fn->right : fn->left;
646 } while (fn);
647
648 if (!allow_create) {
649 /* We should not create new node because
650 * NLM_F_REPLACE was specified without NLM_F_CREATE
651 * I assume it is safe to require NLM_F_CREATE when
652 * REPLACE flag is used! Later we may want to remove the
653 * check for replace_required, because according
654 * to netlink specification, NLM_F_CREATE
655 * MUST be specified if new route is created.
656 * That would keep IPv6 consistent with IPv4
657 */
658 if (replace_required) {
659 NL_SET_ERR_MSG(extack,
660 "Can not replace route - no match found");
661 pr_warn("Can't replace route, no match found\n");
662 return ERR_PTR(-ENOENT);
663 }
664 pr_warn("NLM_F_CREATE should be set when creating new route\n");
665 }
666 /*
667 * We walked to the bottom of tree.
668 * Create new leaf node without children.
669 */
670
671 ln = node_alloc();
672
673 if (!ln)
674 return ERR_PTR(-ENOMEM);
675 ln->fn_bit = plen;
676
677 ln->parent = pn;
678 ln->fn_sernum = sernum;
679
680 if (dir)
681 pn->right = ln;
682 else
683 pn->left = ln;
684
685 return ln;
686
687
688 insert_above:
689 /*
690 * split since we don't have a common prefix anymore or
691 * we have a less significant route.
692 * we've to insert an intermediate node on the list
693 * this new node will point to the one we need to create
694 * and the current
695 */
696
697 pn = fn->parent;
698
699 /* find 1st bit in difference between the 2 addrs.
700
701 See comment in __ipv6_addr_diff: bit may be an invalid value,
702 but if it is >= plen, the value is ignored in any case.
703 */
704
705 bit = __ipv6_addr_diff(addr, &key->addr, sizeof(*addr));
706
707 /*
708 * (intermediate)[in]
709 * / \
710 * (new leaf node)[ln] (old node)[fn]
711 */
712 if (plen > bit) {
713 in = node_alloc();
714 ln = node_alloc();
715
716 if (!in || !ln) {
717 if (in)
718 node_free_immediate(in);
719 if (ln)
720 node_free_immediate(ln);
721 return ERR_PTR(-ENOMEM);
722 }
723
724 /*
725 * new intermediate node.
726 * RTN_RTINFO will
727 * be off since that an address that chooses one of
728 * the branches would not match less specific routes
729 * in the other branch
730 */
731
732 in->fn_bit = bit;
733
734 in->parent = pn;
735 in->leaf = fn->leaf;
736 atomic_inc(&in->leaf->rt6i_ref);
737
738 in->fn_sernum = sernum;
739
740 /* update parent pointer */
741 if (dir)
742 pn->right = in;
743 else
744 pn->left = in;
745
746 ln->fn_bit = plen;
747
748 ln->parent = in;
749 fn->parent = in;
750
751 ln->fn_sernum = sernum;
752
753 if (addr_bit_set(addr, bit)) {
754 in->right = ln;
755 in->left = fn;
756 } else {
757 in->left = ln;
758 in->right = fn;
759 }
760 } else { /* plen <= bit */
761
762 /*
763 * (new leaf node)[ln]
764 * / \
765 * (old node)[fn] NULL
766 */
767
768 ln = node_alloc();
769
770 if (!ln)
771 return ERR_PTR(-ENOMEM);
772
773 ln->fn_bit = plen;
774
775 ln->parent = pn;
776
777 ln->fn_sernum = sernum;
778
779 if (dir)
780 pn->right = ln;
781 else
782 pn->left = ln;
783
784 if (addr_bit_set(&key->addr, plen))
785 ln->right = fn;
786 else
787 ln->left = fn;
788
789 fn->parent = ln;
790 }
791 return ln;
792 }
793
794 static bool rt6_qualify_for_ecmp(struct rt6_info *rt)
795 {
796 return (rt->rt6i_flags & (RTF_GATEWAY|RTF_ADDRCONF|RTF_DYNAMIC)) ==
797 RTF_GATEWAY;
798 }
799
800 static void fib6_copy_metrics(u32 *mp, const struct mx6_config *mxc)
801 {
802 int i;
803
804 for (i = 0; i < RTAX_MAX; i++) {
805 if (test_bit(i, mxc->mx_valid))
806 mp[i] = mxc->mx[i];
807 }
808 }
809
810 static int fib6_commit_metrics(struct dst_entry *dst, struct mx6_config *mxc)
811 {
812 if (!mxc->mx)
813 return 0;
814
815 if (dst->flags & DST_HOST) {
816 u32 *mp = dst_metrics_write_ptr(dst);
817
818 if (unlikely(!mp))
819 return -ENOMEM;
820
821 fib6_copy_metrics(mp, mxc);
822 } else {
823 dst_init_metrics(dst, mxc->mx, false);
824
825 /* We've stolen mx now. */
826 mxc->mx = NULL;
827 }
828
829 return 0;
830 }
831
832 static void fib6_purge_rt(struct rt6_info *rt, struct fib6_node *fn,
833 struct net *net)
834 {
835 if (atomic_read(&rt->rt6i_ref) != 1) {
836 /* This route is used as dummy address holder in some split
837 * nodes. It is not leaked, but it still holds other resources,
838 * which must be released in time. So, scan ascendant nodes
839 * and replace dummy references to this route with references
840 * to still alive ones.
841 */
842 while (fn) {
843 if (!(fn->fn_flags & RTN_RTINFO) && fn->leaf == rt) {
844 fn->leaf = fib6_find_prefix(net, fn);
845 atomic_inc(&fn->leaf->rt6i_ref);
846 rt6_release(rt);
847 }
848 fn = fn->parent;
849 }
850 }
851 }
852
853 /*
854 * Insert routing information in a node.
855 */
856
857 static int fib6_add_rt2node(struct fib6_node *fn, struct rt6_info *rt,
858 struct nl_info *info, struct mx6_config *mxc)
859 {
860 struct rt6_info *iter = NULL;
861 struct rt6_info **ins;
862 struct rt6_info **fallback_ins = NULL;
863 int replace = (info->nlh &&
864 (info->nlh->nlmsg_flags & NLM_F_REPLACE));
865 int add = (!info->nlh ||
866 (info->nlh->nlmsg_flags & NLM_F_CREATE));
867 int found = 0;
868 bool rt_can_ecmp = rt6_qualify_for_ecmp(rt);
869 u16 nlflags = NLM_F_EXCL;
870 int err;
871
872 if (info->nlh && (info->nlh->nlmsg_flags & NLM_F_APPEND))
873 nlflags |= NLM_F_APPEND;
874
875 ins = &fn->leaf;
876
877 for (iter = fn->leaf; iter; iter = iter->dst.rt6_next) {
878 /*
879 * Search for duplicates
880 */
881
882 if (iter->rt6i_metric == rt->rt6i_metric) {
883 /*
884 * Same priority level
885 */
886 if (info->nlh &&
887 (info->nlh->nlmsg_flags & NLM_F_EXCL))
888 return -EEXIST;
889
890 nlflags &= ~NLM_F_EXCL;
891 if (replace) {
892 if (rt_can_ecmp == rt6_qualify_for_ecmp(iter)) {
893 found++;
894 break;
895 }
896 if (rt_can_ecmp)
897 fallback_ins = fallback_ins ?: ins;
898 goto next_iter;
899 }
900
901 if (rt6_duplicate_nexthop(iter, rt)) {
902 if (rt->rt6i_nsiblings)
903 rt->rt6i_nsiblings = 0;
904 if (!(iter->rt6i_flags & RTF_EXPIRES))
905 return -EEXIST;
906 if (!(rt->rt6i_flags & RTF_EXPIRES))
907 rt6_clean_expires(iter);
908 else
909 rt6_set_expires(iter, rt->dst.expires);
910 iter->rt6i_pmtu = rt->rt6i_pmtu;
911 return -EEXIST;
912 }
913 /* If we have the same destination and the same metric,
914 * but not the same gateway, then the route we try to
915 * add is sibling to this route, increment our counter
916 * of siblings, and later we will add our route to the
917 * list.
918 * Only static routes (which don't have flag
919 * RTF_EXPIRES) are used for ECMPv6.
920 *
921 * To avoid long list, we only had siblings if the
922 * route have a gateway.
923 */
924 if (rt_can_ecmp &&
925 rt6_qualify_for_ecmp(iter))
926 rt->rt6i_nsiblings++;
927 }
928
929 if (iter->rt6i_metric > rt->rt6i_metric)
930 break;
931
932 next_iter:
933 ins = &iter->dst.rt6_next;
934 }
935
936 if (fallback_ins && !found) {
937 /* No ECMP-able route found, replace first non-ECMP one */
938 ins = fallback_ins;
939 iter = *ins;
940 found++;
941 }
942
943 /* Reset round-robin state, if necessary */
944 if (ins == &fn->leaf)
945 fn->rr_ptr = NULL;
946
947 /* Link this route to others same route. */
948 if (rt->rt6i_nsiblings) {
949 unsigned int rt6i_nsiblings;
950 struct rt6_info *sibling, *temp_sibling;
951
952 /* Find the first route that have the same metric */
953 sibling = fn->leaf;
954 while (sibling) {
955 if (sibling->rt6i_metric == rt->rt6i_metric &&
956 rt6_qualify_for_ecmp(sibling)) {
957 list_add_tail(&rt->rt6i_siblings,
958 &sibling->rt6i_siblings);
959 break;
960 }
961 sibling = sibling->dst.rt6_next;
962 }
963 /* For each sibling in the list, increment the counter of
964 * siblings. BUG() if counters does not match, list of siblings
965 * is broken!
966 */
967 rt6i_nsiblings = 0;
968 list_for_each_entry_safe(sibling, temp_sibling,
969 &rt->rt6i_siblings, rt6i_siblings) {
970 sibling->rt6i_nsiblings++;
971 BUG_ON(sibling->rt6i_nsiblings != rt->rt6i_nsiblings);
972 rt6i_nsiblings++;
973 }
974 BUG_ON(rt6i_nsiblings != rt->rt6i_nsiblings);
975 }
976
977 /*
978 * insert node
979 */
980 if (!replace) {
981 if (!add)
982 pr_warn("NLM_F_CREATE should be set when creating new route\n");
983
984 add:
985 nlflags |= NLM_F_CREATE;
986 err = fib6_commit_metrics(&rt->dst, mxc);
987 if (err)
988 return err;
989
990 rt->dst.rt6_next = iter;
991 *ins = rt;
992 rcu_assign_pointer(rt->rt6i_node, fn);
993 atomic_inc(&rt->rt6i_ref);
994 call_fib6_entry_notifiers(info->nl_net, FIB_EVENT_ENTRY_ADD,
995 rt);
996 if (!info->skip_notify)
997 inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);
998 info->nl_net->ipv6.rt6_stats->fib_rt_entries++;
999
1000 if (!(fn->fn_flags & RTN_RTINFO)) {
1001 info->nl_net->ipv6.rt6_stats->fib_route_nodes++;
1002 fn->fn_flags |= RTN_RTINFO;
1003 }
1004
1005 } else {
1006 int nsiblings;
1007
1008 if (!found) {
1009 if (add)
1010 goto add;
1011 pr_warn("NLM_F_REPLACE set, but no existing node found!\n");
1012 return -ENOENT;
1013 }
1014
1015 err = fib6_commit_metrics(&rt->dst, mxc);
1016 if (err)
1017 return err;
1018
1019 *ins = rt;
1020 rcu_assign_pointer(rt->rt6i_node, fn);
1021 rt->dst.rt6_next = iter->dst.rt6_next;
1022 atomic_inc(&rt->rt6i_ref);
1023 call_fib6_entry_notifiers(info->nl_net, FIB_EVENT_ENTRY_REPLACE,
1024 rt);
1025 if (!info->skip_notify)
1026 inet6_rt_notify(RTM_NEWROUTE, rt, info, NLM_F_REPLACE);
1027 if (!(fn->fn_flags & RTN_RTINFO)) {
1028 info->nl_net->ipv6.rt6_stats->fib_route_nodes++;
1029 fn->fn_flags |= RTN_RTINFO;
1030 }
1031 nsiblings = iter->rt6i_nsiblings;
1032 iter->rt6i_node = NULL;
1033 fib6_purge_rt(iter, fn, info->nl_net);
1034 if (fn->rr_ptr == iter)
1035 fn->rr_ptr = NULL;
1036 rt6_release(iter);
1037
1038 if (nsiblings) {
1039 /* Replacing an ECMP route, remove all siblings */
1040 ins = &rt->dst.rt6_next;
1041 iter = *ins;
1042 while (iter) {
1043 if (iter->rt6i_metric > rt->rt6i_metric)
1044 break;
1045 if (rt6_qualify_for_ecmp(iter)) {
1046 *ins = iter->dst.rt6_next;
1047 iter->rt6i_node = NULL;
1048 fib6_purge_rt(iter, fn, info->nl_net);
1049 if (fn->rr_ptr == iter)
1050 fn->rr_ptr = NULL;
1051 rt6_release(iter);
1052 nsiblings--;
1053 } else {
1054 ins = &iter->dst.rt6_next;
1055 }
1056 iter = *ins;
1057 }
1058 WARN_ON(nsiblings != 0);
1059 }
1060 }
1061
1062 return 0;
1063 }
1064
1065 static void fib6_start_gc(struct net *net, struct rt6_info *rt)
1066 {
1067 if (!timer_pending(&net->ipv6.ip6_fib_timer) &&
1068 (rt->rt6i_flags & (RTF_EXPIRES | RTF_CACHE)))
1069 mod_timer(&net->ipv6.ip6_fib_timer,
1070 jiffies + net->ipv6.sysctl.ip6_rt_gc_interval);
1071 }
1072
1073 void fib6_force_start_gc(struct net *net)
1074 {
1075 if (!timer_pending(&net->ipv6.ip6_fib_timer))
1076 mod_timer(&net->ipv6.ip6_fib_timer,
1077 jiffies + net->ipv6.sysctl.ip6_rt_gc_interval);
1078 }
1079
1080 /*
1081 * Add routing information to the routing tree.
1082 * <destination addr>/<source addr>
1083 * with source addr info in sub-trees
1084 */
1085
1086 int fib6_add(struct fib6_node *root, struct rt6_info *rt,
1087 struct nl_info *info, struct mx6_config *mxc,
1088 struct netlink_ext_ack *extack)
1089 {
1090 struct fib6_node *fn, *pn = NULL;
1091 int err = -ENOMEM;
1092 int allow_create = 1;
1093 int replace_required = 0;
1094 int sernum = fib6_new_sernum(info->nl_net);
1095
1096 if (WARN_ON_ONCE(!atomic_read(&rt->dst.__refcnt)))
1097 return -EINVAL;
1098
1099 if (info->nlh) {
1100 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
1101 allow_create = 0;
1102 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1103 replace_required = 1;
1104 }
1105 if (!allow_create && !replace_required)
1106 pr_warn("RTM_NEWROUTE with no NLM_F_CREATE or NLM_F_REPLACE\n");
1107
1108 fn = fib6_add_1(root, &rt->rt6i_dst.addr, rt->rt6i_dst.plen,
1109 offsetof(struct rt6_info, rt6i_dst), allow_create,
1110 replace_required, sernum, extack);
1111 if (IS_ERR(fn)) {
1112 err = PTR_ERR(fn);
1113 fn = NULL;
1114 goto out;
1115 }
1116
1117 pn = fn;
1118
1119 #ifdef CONFIG_IPV6_SUBTREES
1120 if (rt->rt6i_src.plen) {
1121 struct fib6_node *sn;
1122
1123 if (!fn->subtree) {
1124 struct fib6_node *sfn;
1125
1126 /*
1127 * Create subtree.
1128 *
1129 * fn[main tree]
1130 * |
1131 * sfn[subtree root]
1132 * \
1133 * sn[new leaf node]
1134 */
1135
1136 /* Create subtree root node */
1137 sfn = node_alloc();
1138 if (!sfn)
1139 goto failure;
1140
1141 sfn->leaf = info->nl_net->ipv6.ip6_null_entry;
1142 atomic_inc(&info->nl_net->ipv6.ip6_null_entry->rt6i_ref);
1143 sfn->fn_flags = RTN_ROOT;
1144 sfn->fn_sernum = sernum;
1145
1146 /* Now add the first leaf node to new subtree */
1147
1148 sn = fib6_add_1(sfn, &rt->rt6i_src.addr,
1149 rt->rt6i_src.plen,
1150 offsetof(struct rt6_info, rt6i_src),
1151 allow_create, replace_required, sernum,
1152 extack);
1153
1154 if (IS_ERR(sn)) {
1155 /* If it is failed, discard just allocated
1156 root, and then (in failure) stale node
1157 in main tree.
1158 */
1159 node_free_immediate(sfn);
1160 err = PTR_ERR(sn);
1161 goto failure;
1162 }
1163
1164 /* Now link new subtree to main tree */
1165 sfn->parent = fn;
1166 fn->subtree = sfn;
1167 } else {
1168 sn = fib6_add_1(fn->subtree, &rt->rt6i_src.addr,
1169 rt->rt6i_src.plen,
1170 offsetof(struct rt6_info, rt6i_src),
1171 allow_create, replace_required, sernum,
1172 extack);
1173
1174 if (IS_ERR(sn)) {
1175 err = PTR_ERR(sn);
1176 goto failure;
1177 }
1178 }
1179
1180 if (!fn->leaf) {
1181 fn->leaf = rt;
1182 atomic_inc(&rt->rt6i_ref);
1183 }
1184 fn = sn;
1185 }
1186 #endif
1187
1188 err = fib6_add_rt2node(fn, rt, info, mxc);
1189 if (!err) {
1190 fib6_start_gc(info->nl_net, rt);
1191 if (!(rt->rt6i_flags & RTF_CACHE))
1192 fib6_prune_clones(info->nl_net, pn);
1193 }
1194
1195 out:
1196 if (err) {
1197 #ifdef CONFIG_IPV6_SUBTREES
1198 /*
1199 * If fib6_add_1 has cleared the old leaf pointer in the
1200 * super-tree leaf node we have to find a new one for it.
1201 */
1202 if (pn != fn && pn->leaf == rt) {
1203 pn->leaf = NULL;
1204 atomic_dec(&rt->rt6i_ref);
1205 }
1206 if (pn != fn && !pn->leaf && !(pn->fn_flags & RTN_RTINFO)) {
1207 pn->leaf = fib6_find_prefix(info->nl_net, pn);
1208 #if RT6_DEBUG >= 2
1209 if (!pn->leaf) {
1210 WARN_ON(pn->leaf == NULL);
1211 pn->leaf = info->nl_net->ipv6.ip6_null_entry;
1212 }
1213 #endif
1214 atomic_inc(&pn->leaf->rt6i_ref);
1215 }
1216 #endif
1217 goto failure;
1218 }
1219 return err;
1220
1221 failure:
1222 /* fn->leaf could be NULL if fn is an intermediate node and we
1223 * failed to add the new route to it in both subtree creation
1224 * failure and fib6_add_rt2node() failure case.
1225 * In both cases, fib6_repair_tree() should be called to fix
1226 * fn->leaf.
1227 */
1228 if (fn && !(fn->fn_flags & (RTN_RTINFO|RTN_ROOT)))
1229 fib6_repair_tree(info->nl_net, fn);
1230 /* Always release dst as dst->__refcnt is guaranteed
1231 * to be taken before entering this function
1232 */
1233 dst_release_immediate(&rt->dst);
1234 return err;
1235 }
1236
1237 /*
1238 * Routing tree lookup
1239 *
1240 */
1241
1242 struct lookup_args {
1243 int offset; /* key offset on rt6_info */
1244 const struct in6_addr *addr; /* search key */
1245 };
1246
1247 static struct fib6_node *fib6_lookup_1(struct fib6_node *root,
1248 struct lookup_args *args)
1249 {
1250 struct fib6_node *fn;
1251 __be32 dir;
1252
1253 if (unlikely(args->offset == 0))
1254 return NULL;
1255
1256 /*
1257 * Descend on a tree
1258 */
1259
1260 fn = root;
1261
1262 for (;;) {
1263 struct fib6_node *next;
1264
1265 dir = addr_bit_set(args->addr, fn->fn_bit);
1266
1267 next = dir ? fn->right : fn->left;
1268
1269 if (next) {
1270 fn = next;
1271 continue;
1272 }
1273 break;
1274 }
1275
1276 while (fn) {
1277 if (FIB6_SUBTREE(fn) || fn->fn_flags & RTN_RTINFO) {
1278 struct rt6key *key;
1279
1280 key = (struct rt6key *) ((u8 *) fn->leaf +
1281 args->offset);
1282
1283 if (ipv6_prefix_equal(&key->addr, args->addr, key->plen)) {
1284 #ifdef CONFIG_IPV6_SUBTREES
1285 if (fn->subtree) {
1286 struct fib6_node *sfn;
1287 sfn = fib6_lookup_1(fn->subtree,
1288 args + 1);
1289 if (!sfn)
1290 goto backtrack;
1291 fn = sfn;
1292 }
1293 #endif
1294 if (fn->fn_flags & RTN_RTINFO)
1295 return fn;
1296 }
1297 }
1298 #ifdef CONFIG_IPV6_SUBTREES
1299 backtrack:
1300 #endif
1301 if (fn->fn_flags & RTN_ROOT)
1302 break;
1303
1304 fn = fn->parent;
1305 }
1306
1307 return NULL;
1308 }
1309
1310 struct fib6_node *fib6_lookup(struct fib6_node *root, const struct in6_addr *daddr,
1311 const struct in6_addr *saddr)
1312 {
1313 struct fib6_node *fn;
1314 struct lookup_args args[] = {
1315 {
1316 .offset = offsetof(struct rt6_info, rt6i_dst),
1317 .addr = daddr,
1318 },
1319 #ifdef CONFIG_IPV6_SUBTREES
1320 {
1321 .offset = offsetof(struct rt6_info, rt6i_src),
1322 .addr = saddr,
1323 },
1324 #endif
1325 {
1326 .offset = 0, /* sentinel */
1327 }
1328 };
1329
1330 fn = fib6_lookup_1(root, daddr ? args : args + 1);
1331 if (!fn || fn->fn_flags & RTN_TL_ROOT)
1332 fn = root;
1333
1334 return fn;
1335 }
1336
1337 /*
1338 * Get node with specified destination prefix (and source prefix,
1339 * if subtrees are used)
1340 */
1341
1342
1343 static struct fib6_node *fib6_locate_1(struct fib6_node *root,
1344 const struct in6_addr *addr,
1345 int plen, int offset)
1346 {
1347 struct fib6_node *fn;
1348
1349 for (fn = root; fn ; ) {
1350 struct rt6key *key = (struct rt6key *)((u8 *)fn->leaf + offset);
1351
1352 /*
1353 * Prefix match
1354 */
1355 if (plen < fn->fn_bit ||
1356 !ipv6_prefix_equal(&key->addr, addr, fn->fn_bit))
1357 return NULL;
1358
1359 if (plen == fn->fn_bit)
1360 return fn;
1361
1362 /*
1363 * We have more bits to go
1364 */
1365 if (addr_bit_set(addr, fn->fn_bit))
1366 fn = fn->right;
1367 else
1368 fn = fn->left;
1369 }
1370 return NULL;
1371 }
1372
1373 struct fib6_node *fib6_locate(struct fib6_node *root,
1374 const struct in6_addr *daddr, int dst_len,
1375 const struct in6_addr *saddr, int src_len)
1376 {
1377 struct fib6_node *fn;
1378
1379 fn = fib6_locate_1(root, daddr, dst_len,
1380 offsetof(struct rt6_info, rt6i_dst));
1381
1382 #ifdef CONFIG_IPV6_SUBTREES
1383 if (src_len) {
1384 WARN_ON(saddr == NULL);
1385 if (fn && fn->subtree)
1386 fn = fib6_locate_1(fn->subtree, saddr, src_len,
1387 offsetof(struct rt6_info, rt6i_src));
1388 }
1389 #endif
1390
1391 if (fn && fn->fn_flags & RTN_RTINFO)
1392 return fn;
1393
1394 return NULL;
1395 }
1396
1397
1398 /*
1399 * Deletion
1400 *
1401 */
1402
1403 static struct rt6_info *fib6_find_prefix(struct net *net, struct fib6_node *fn)
1404 {
1405 if (fn->fn_flags & RTN_ROOT)
1406 return net->ipv6.ip6_null_entry;
1407
1408 while (fn) {
1409 if (fn->left)
1410 return fn->left->leaf;
1411 if (fn->right)
1412 return fn->right->leaf;
1413
1414 fn = FIB6_SUBTREE(fn);
1415 }
1416 return NULL;
1417 }
1418
1419 /*
1420 * Called to trim the tree of intermediate nodes when possible. "fn"
1421 * is the node we want to try and remove.
1422 */
1423
1424 static struct fib6_node *fib6_repair_tree(struct net *net,
1425 struct fib6_node *fn)
1426 {
1427 int children;
1428 int nstate;
1429 struct fib6_node *child, *pn;
1430 struct fib6_walker *w;
1431 int iter = 0;
1432
1433 for (;;) {
1434 RT6_TRACE("fixing tree: plen=%d iter=%d\n", fn->fn_bit, iter);
1435 iter++;
1436
1437 WARN_ON(fn->fn_flags & RTN_RTINFO);
1438 WARN_ON(fn->fn_flags & RTN_TL_ROOT);
1439 WARN_ON(fn->leaf);
1440
1441 children = 0;
1442 child = NULL;
1443 if (fn->right)
1444 child = fn->right, children |= 1;
1445 if (fn->left)
1446 child = fn->left, children |= 2;
1447
1448 if (children == 3 || FIB6_SUBTREE(fn)
1449 #ifdef CONFIG_IPV6_SUBTREES
1450 /* Subtree root (i.e. fn) may have one child */
1451 || (children && fn->fn_flags & RTN_ROOT)
1452 #endif
1453 ) {
1454 fn->leaf = fib6_find_prefix(net, fn);
1455 #if RT6_DEBUG >= 2
1456 if (!fn->leaf) {
1457 WARN_ON(!fn->leaf);
1458 fn->leaf = net->ipv6.ip6_null_entry;
1459 }
1460 #endif
1461 atomic_inc(&fn->leaf->rt6i_ref);
1462 return fn->parent;
1463 }
1464
1465 pn = fn->parent;
1466 #ifdef CONFIG_IPV6_SUBTREES
1467 if (FIB6_SUBTREE(pn) == fn) {
1468 WARN_ON(!(fn->fn_flags & RTN_ROOT));
1469 FIB6_SUBTREE(pn) = NULL;
1470 nstate = FWS_L;
1471 } else {
1472 WARN_ON(fn->fn_flags & RTN_ROOT);
1473 #endif
1474 if (pn->right == fn)
1475 pn->right = child;
1476 else if (pn->left == fn)
1477 pn->left = child;
1478 #if RT6_DEBUG >= 2
1479 else
1480 WARN_ON(1);
1481 #endif
1482 if (child)
1483 child->parent = pn;
1484 nstate = FWS_R;
1485 #ifdef CONFIG_IPV6_SUBTREES
1486 }
1487 #endif
1488
1489 read_lock(&net->ipv6.fib6_walker_lock);
1490 FOR_WALKERS(net, w) {
1491 if (!child) {
1492 if (w->root == fn) {
1493 w->root = w->node = NULL;
1494 RT6_TRACE("W %p adjusted by delroot 1\n", w);
1495 } else if (w->node == fn) {
1496 RT6_TRACE("W %p adjusted by delnode 1, s=%d/%d\n", w, w->state, nstate);
1497 w->node = pn;
1498 w->state = nstate;
1499 }
1500 } else {
1501 if (w->root == fn) {
1502 w->root = child;
1503 RT6_TRACE("W %p adjusted by delroot 2\n", w);
1504 }
1505 if (w->node == fn) {
1506 w->node = child;
1507 if (children&2) {
1508 RT6_TRACE("W %p adjusted by delnode 2, s=%d\n", w, w->state);
1509 w->state = w->state >= FWS_R ? FWS_U : FWS_INIT;
1510 } else {
1511 RT6_TRACE("W %p adjusted by delnode 2, s=%d\n", w, w->state);
1512 w->state = w->state >= FWS_C ? FWS_U : FWS_INIT;
1513 }
1514 }
1515 }
1516 }
1517 read_unlock(&net->ipv6.fib6_walker_lock);
1518
1519 node_free(fn);
1520 if (pn->fn_flags & RTN_RTINFO || FIB6_SUBTREE(pn))
1521 return pn;
1522
1523 rt6_release(pn->leaf);
1524 pn->leaf = NULL;
1525 fn = pn;
1526 }
1527 }
1528
1529 static void fib6_del_route(struct fib6_node *fn, struct rt6_info **rtp,
1530 struct nl_info *info)
1531 {
1532 struct fib6_walker *w;
1533 struct rt6_info *rt = *rtp;
1534 struct net *net = info->nl_net;
1535
1536 RT6_TRACE("fib6_del_route\n");
1537
1538 /* Unlink it */
1539 *rtp = rt->dst.rt6_next;
1540 rt->rt6i_node = NULL;
1541 net->ipv6.rt6_stats->fib_rt_entries--;
1542 net->ipv6.rt6_stats->fib_discarded_routes++;
1543
1544 /* Reset round-robin state, if necessary */
1545 if (fn->rr_ptr == rt)
1546 fn->rr_ptr = NULL;
1547
1548 /* Remove this entry from other siblings */
1549 if (rt->rt6i_nsiblings) {
1550 struct rt6_info *sibling, *next_sibling;
1551
1552 list_for_each_entry_safe(sibling, next_sibling,
1553 &rt->rt6i_siblings, rt6i_siblings)
1554 sibling->rt6i_nsiblings--;
1555 rt->rt6i_nsiblings = 0;
1556 list_del_init(&rt->rt6i_siblings);
1557 }
1558
1559 /* Adjust walkers */
1560 read_lock(&net->ipv6.fib6_walker_lock);
1561 FOR_WALKERS(net, w) {
1562 if (w->state == FWS_C && w->leaf == rt) {
1563 RT6_TRACE("walker %p adjusted by delroute\n", w);
1564 w->leaf = rt->dst.rt6_next;
1565 if (!w->leaf)
1566 w->state = FWS_U;
1567 }
1568 }
1569 read_unlock(&net->ipv6.fib6_walker_lock);
1570
1571 rt->dst.rt6_next = NULL;
1572
1573 /* If it was last route, expunge its radix tree node */
1574 if (!fn->leaf) {
1575 fn->fn_flags &= ~RTN_RTINFO;
1576 net->ipv6.rt6_stats->fib_route_nodes--;
1577 fn = fib6_repair_tree(net, fn);
1578 }
1579
1580 fib6_purge_rt(rt, fn, net);
1581
1582 call_fib6_entry_notifiers(net, FIB_EVENT_ENTRY_DEL, rt);
1583 if (!info->skip_notify)
1584 inet6_rt_notify(RTM_DELROUTE, rt, info, 0);
1585 rt6_release(rt);
1586 }
1587
1588 int fib6_del(struct rt6_info *rt, struct nl_info *info)
1589 {
1590 struct fib6_node *fn = rcu_dereference_protected(rt->rt6i_node,
1591 lockdep_is_held(&rt->rt6i_table->tb6_lock));
1592 struct net *net = info->nl_net;
1593 struct rt6_info **rtp;
1594
1595 #if RT6_DEBUG >= 2
1596 if (rt->dst.obsolete > 0) {
1597 WARN_ON(fn);
1598 return -ENOENT;
1599 }
1600 #endif
1601 if (!fn || rt == net->ipv6.ip6_null_entry)
1602 return -ENOENT;
1603
1604 WARN_ON(!(fn->fn_flags & RTN_RTINFO));
1605
1606 if (!(rt->rt6i_flags & RTF_CACHE)) {
1607 struct fib6_node *pn = fn;
1608 #ifdef CONFIG_IPV6_SUBTREES
1609 /* clones of this route might be in another subtree */
1610 if (rt->rt6i_src.plen) {
1611 while (!(pn->fn_flags & RTN_ROOT))
1612 pn = pn->parent;
1613 pn = pn->parent;
1614 }
1615 #endif
1616 fib6_prune_clones(info->nl_net, pn);
1617 }
1618
1619 /*
1620 * Walk the leaf entries looking for ourself
1621 */
1622
1623 for (rtp = &fn->leaf; *rtp; rtp = &(*rtp)->dst.rt6_next) {
1624 if (*rtp == rt) {
1625 fib6_del_route(fn, rtp, info);
1626 return 0;
1627 }
1628 }
1629 return -ENOENT;
1630 }
1631
1632 /*
1633 * Tree traversal function.
1634 *
1635 * Certainly, it is not interrupt safe.
1636 * However, it is internally reenterable wrt itself and fib6_add/fib6_del.
1637 * It means, that we can modify tree during walking
1638 * and use this function for garbage collection, clone pruning,
1639 * cleaning tree when a device goes down etc. etc.
1640 *
1641 * It guarantees that every node will be traversed,
1642 * and that it will be traversed only once.
1643 *
1644 * Callback function w->func may return:
1645 * 0 -> continue walking.
1646 * positive value -> walking is suspended (used by tree dumps,
1647 * and probably by gc, if it will be split to several slices)
1648 * negative value -> terminate walking.
1649 *
1650 * The function itself returns:
1651 * 0 -> walk is complete.
1652 * >0 -> walk is incomplete (i.e. suspended)
1653 * <0 -> walk is terminated by an error.
1654 */
1655
1656 static int fib6_walk_continue(struct fib6_walker *w)
1657 {
1658 struct fib6_node *fn, *pn;
1659
1660 for (;;) {
1661 fn = w->node;
1662 if (!fn)
1663 return 0;
1664
1665 if (w->prune && fn != w->root &&
1666 fn->fn_flags & RTN_RTINFO && w->state < FWS_C) {
1667 w->state = FWS_C;
1668 w->leaf = fn->leaf;
1669 }
1670 switch (w->state) {
1671 #ifdef CONFIG_IPV6_SUBTREES
1672 case FWS_S:
1673 if (FIB6_SUBTREE(fn)) {
1674 w->node = FIB6_SUBTREE(fn);
1675 continue;
1676 }
1677 w->state = FWS_L;
1678 #endif
1679 case FWS_L:
1680 if (fn->left) {
1681 w->node = fn->left;
1682 w->state = FWS_INIT;
1683 continue;
1684 }
1685 w->state = FWS_R;
1686 case FWS_R:
1687 if (fn->right) {
1688 w->node = fn->right;
1689 w->state = FWS_INIT;
1690 continue;
1691 }
1692 w->state = FWS_C;
1693 w->leaf = fn->leaf;
1694 case FWS_C:
1695 if (w->leaf && fn->fn_flags & RTN_RTINFO) {
1696 int err;
1697
1698 if (w->skip) {
1699 w->skip--;
1700 goto skip;
1701 }
1702
1703 err = w->func(w);
1704 if (err)
1705 return err;
1706
1707 w->count++;
1708 continue;
1709 }
1710 skip:
1711 w->state = FWS_U;
1712 case FWS_U:
1713 if (fn == w->root)
1714 return 0;
1715 pn = fn->parent;
1716 w->node = pn;
1717 #ifdef CONFIG_IPV6_SUBTREES
1718 if (FIB6_SUBTREE(pn) == fn) {
1719 WARN_ON(!(fn->fn_flags & RTN_ROOT));
1720 w->state = FWS_L;
1721 continue;
1722 }
1723 #endif
1724 if (pn->left == fn) {
1725 w->state = FWS_R;
1726 continue;
1727 }
1728 if (pn->right == fn) {
1729 w->state = FWS_C;
1730 w->leaf = w->node->leaf;
1731 continue;
1732 }
1733 #if RT6_DEBUG >= 2
1734 WARN_ON(1);
1735 #endif
1736 }
1737 }
1738 }
1739
1740 static int fib6_walk(struct net *net, struct fib6_walker *w)
1741 {
1742 int res;
1743
1744 w->state = FWS_INIT;
1745 w->node = w->root;
1746
1747 fib6_walker_link(net, w);
1748 res = fib6_walk_continue(w);
1749 if (res <= 0)
1750 fib6_walker_unlink(net, w);
1751 return res;
1752 }
1753
1754 static int fib6_clean_node(struct fib6_walker *w)
1755 {
1756 int res;
1757 struct rt6_info *rt;
1758 struct fib6_cleaner *c = container_of(w, struct fib6_cleaner, w);
1759 struct nl_info info = {
1760 .nl_net = c->net,
1761 };
1762
1763 if (c->sernum != FIB6_NO_SERNUM_CHANGE &&
1764 w->node->fn_sernum != c->sernum)
1765 w->node->fn_sernum = c->sernum;
1766
1767 if (!c->func) {
1768 WARN_ON_ONCE(c->sernum == FIB6_NO_SERNUM_CHANGE);
1769 w->leaf = NULL;
1770 return 0;
1771 }
1772
1773 for (rt = w->leaf; rt; rt = rt->dst.rt6_next) {
1774 res = c->func(rt, c->arg);
1775 if (res < 0) {
1776 w->leaf = rt;
1777 res = fib6_del(rt, &info);
1778 if (res) {
1779 #if RT6_DEBUG >= 2
1780 pr_debug("%s: del failed: rt=%p@%p err=%d\n",
1781 __func__, rt,
1782 rcu_access_pointer(rt->rt6i_node),
1783 res);
1784 #endif
1785 continue;
1786 }
1787 return 0;
1788 }
1789 WARN_ON(res != 0);
1790 }
1791 w->leaf = rt;
1792 return 0;
1793 }
1794
1795 /*
1796 * Convenient frontend to tree walker.
1797 *
1798 * func is called on each route.
1799 * It may return -1 -> delete this route.
1800 * 0 -> continue walking
1801 *
1802 * prune==1 -> only immediate children of node (certainly,
1803 * ignoring pure split nodes) will be scanned.
1804 */
1805
1806 static void fib6_clean_tree(struct net *net, struct fib6_node *root,
1807 int (*func)(struct rt6_info *, void *arg),
1808 bool prune, int sernum, void *arg)
1809 {
1810 struct fib6_cleaner c;
1811
1812 c.w.root = root;
1813 c.w.func = fib6_clean_node;
1814 c.w.prune = prune;
1815 c.w.count = 0;
1816 c.w.skip = 0;
1817 c.func = func;
1818 c.sernum = sernum;
1819 c.arg = arg;
1820 c.net = net;
1821
1822 fib6_walk(net, &c.w);
1823 }
1824
1825 static void __fib6_clean_all(struct net *net,
1826 int (*func)(struct rt6_info *, void *),
1827 int sernum, void *arg)
1828 {
1829 struct fib6_table *table;
1830 struct hlist_head *head;
1831 unsigned int h;
1832
1833 rcu_read_lock();
1834 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
1835 head = &net->ipv6.fib_table_hash[h];
1836 hlist_for_each_entry_rcu(table, head, tb6_hlist) {
1837 write_lock_bh(&table->tb6_lock);
1838 fib6_clean_tree(net, &table->tb6_root,
1839 func, false, sernum, arg);
1840 write_unlock_bh(&table->tb6_lock);
1841 }
1842 }
1843 rcu_read_unlock();
1844 }
1845
1846 void fib6_clean_all(struct net *net, int (*func)(struct rt6_info *, void *),
1847 void *arg)
1848 {
1849 __fib6_clean_all(net, func, FIB6_NO_SERNUM_CHANGE, arg);
1850 }
1851
1852 static int fib6_prune_clone(struct rt6_info *rt, void *arg)
1853 {
1854 if (rt->rt6i_flags & RTF_CACHE) {
1855 RT6_TRACE("pruning clone %p\n", rt);
1856 return -1;
1857 }
1858
1859 return 0;
1860 }
1861
1862 static void fib6_prune_clones(struct net *net, struct fib6_node *fn)
1863 {
1864 fib6_clean_tree(net, fn, fib6_prune_clone, true,
1865 FIB6_NO_SERNUM_CHANGE, NULL);
1866 }
1867
1868 static void fib6_flush_trees(struct net *net)
1869 {
1870 int new_sernum = fib6_new_sernum(net);
1871
1872 __fib6_clean_all(net, NULL, new_sernum, NULL);
1873 }
1874
1875 /*
1876 * Garbage collection
1877 */
1878
1879 struct fib6_gc_args
1880 {
1881 int timeout;
1882 int more;
1883 };
1884
1885 static int fib6_age(struct rt6_info *rt, void *arg)
1886 {
1887 struct fib6_gc_args *gc_args = arg;
1888 unsigned long now = jiffies;
1889
1890 /*
1891 * check addrconf expiration here.
1892 * Routes are expired even if they are in use.
1893 *
1894 * Also age clones. Note, that clones are aged out
1895 * only if they are not in use now.
1896 */
1897
1898 if (rt->rt6i_flags & RTF_EXPIRES && rt->dst.expires) {
1899 if (time_after(now, rt->dst.expires)) {
1900 RT6_TRACE("expiring %p\n", rt);
1901 return -1;
1902 }
1903 gc_args->more++;
1904 } else if (rt->rt6i_flags & RTF_CACHE) {
1905 if (time_after_eq(now, rt->dst.lastuse + gc_args->timeout))
1906 rt->dst.obsolete = DST_OBSOLETE_KILL;
1907 if (atomic_read(&rt->dst.__refcnt) == 1 &&
1908 rt->dst.obsolete == DST_OBSOLETE_KILL) {
1909 RT6_TRACE("aging clone %p\n", rt);
1910 return -1;
1911 } else if (rt->rt6i_flags & RTF_GATEWAY) {
1912 struct neighbour *neigh;
1913 __u8 neigh_flags = 0;
1914
1915 neigh = dst_neigh_lookup(&rt->dst, &rt->rt6i_gateway);
1916 if (neigh) {
1917 neigh_flags = neigh->flags;
1918 neigh_release(neigh);
1919 }
1920 if (!(neigh_flags & NTF_ROUTER)) {
1921 RT6_TRACE("purging route %p via non-router but gateway\n",
1922 rt);
1923 return -1;
1924 }
1925 }
1926 gc_args->more++;
1927 }
1928
1929 return 0;
1930 }
1931
1932 void fib6_run_gc(unsigned long expires, struct net *net, bool force)
1933 {
1934 struct fib6_gc_args gc_args;
1935 unsigned long now;
1936
1937 if (force) {
1938 spin_lock_bh(&net->ipv6.fib6_gc_lock);
1939 } else if (!spin_trylock_bh(&net->ipv6.fib6_gc_lock)) {
1940 mod_timer(&net->ipv6.ip6_fib_timer, jiffies + HZ);
1941 return;
1942 }
1943 gc_args.timeout = expires ? (int)expires :
1944 net->ipv6.sysctl.ip6_rt_gc_interval;
1945 gc_args.more = 0;
1946
1947 fib6_clean_all(net, fib6_age, &gc_args);
1948 now = jiffies;
1949 net->ipv6.ip6_rt_last_gc = now;
1950
1951 if (gc_args.more)
1952 mod_timer(&net->ipv6.ip6_fib_timer,
1953 round_jiffies(now
1954 + net->ipv6.sysctl.ip6_rt_gc_interval));
1955 else
1956 del_timer(&net->ipv6.ip6_fib_timer);
1957 spin_unlock_bh(&net->ipv6.fib6_gc_lock);
1958 }
1959
1960 static void fib6_gc_timer_cb(unsigned long arg)
1961 {
1962 fib6_run_gc(0, (struct net *)arg, true);
1963 }
1964
1965 static int __net_init fib6_net_init(struct net *net)
1966 {
1967 size_t size = sizeof(struct hlist_head) * FIB6_TABLE_HASHSZ;
1968 int err;
1969
1970 err = fib6_notifier_init(net);
1971 if (err)
1972 return err;
1973
1974 spin_lock_init(&net->ipv6.fib6_gc_lock);
1975 rwlock_init(&net->ipv6.fib6_walker_lock);
1976 INIT_LIST_HEAD(&net->ipv6.fib6_walkers);
1977 setup_timer(&net->ipv6.ip6_fib_timer, fib6_gc_timer_cb, (unsigned long)net);
1978
1979 net->ipv6.rt6_stats = kzalloc(sizeof(*net->ipv6.rt6_stats), GFP_KERNEL);
1980 if (!net->ipv6.rt6_stats)
1981 goto out_timer;
1982
1983 /* Avoid false sharing : Use at least a full cache line */
1984 size = max_t(size_t, size, L1_CACHE_BYTES);
1985
1986 net->ipv6.fib_table_hash = kzalloc(size, GFP_KERNEL);
1987 if (!net->ipv6.fib_table_hash)
1988 goto out_rt6_stats;
1989
1990 net->ipv6.fib6_main_tbl = kzalloc(sizeof(*net->ipv6.fib6_main_tbl),
1991 GFP_KERNEL);
1992 if (!net->ipv6.fib6_main_tbl)
1993 goto out_fib_table_hash;
1994
1995 net->ipv6.fib6_main_tbl->tb6_id = RT6_TABLE_MAIN;
1996 net->ipv6.fib6_main_tbl->tb6_root.leaf = net->ipv6.ip6_null_entry;
1997 net->ipv6.fib6_main_tbl->tb6_root.fn_flags =
1998 RTN_ROOT | RTN_TL_ROOT | RTN_RTINFO;
1999 inet_peer_base_init(&net->ipv6.fib6_main_tbl->tb6_peers);
2000
2001 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
2002 net->ipv6.fib6_local_tbl = kzalloc(sizeof(*net->ipv6.fib6_local_tbl),
2003 GFP_KERNEL);
2004 if (!net->ipv6.fib6_local_tbl)
2005 goto out_fib6_main_tbl;
2006 net->ipv6.fib6_local_tbl->tb6_id = RT6_TABLE_LOCAL;
2007 net->ipv6.fib6_local_tbl->tb6_root.leaf = net->ipv6.ip6_null_entry;
2008 net->ipv6.fib6_local_tbl->tb6_root.fn_flags =
2009 RTN_ROOT | RTN_TL_ROOT | RTN_RTINFO;
2010 inet_peer_base_init(&net->ipv6.fib6_local_tbl->tb6_peers);
2011 #endif
2012 fib6_tables_init(net);
2013
2014 return 0;
2015
2016 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
2017 out_fib6_main_tbl:
2018 kfree(net->ipv6.fib6_main_tbl);
2019 #endif
2020 out_fib_table_hash:
2021 kfree(net->ipv6.fib_table_hash);
2022 out_rt6_stats:
2023 kfree(net->ipv6.rt6_stats);
2024 out_timer:
2025 fib6_notifier_exit(net);
2026 return -ENOMEM;
2027 }
2028
2029 static void fib6_net_exit(struct net *net)
2030 {
2031 unsigned int i;
2032
2033 rt6_ifdown(net, NULL);
2034 del_timer_sync(&net->ipv6.ip6_fib_timer);
2035
2036 for (i = 0; i < FIB6_TABLE_HASHSZ; i++) {
2037 struct hlist_head *head = &net->ipv6.fib_table_hash[i];
2038 struct hlist_node *tmp;
2039 struct fib6_table *tb;
2040
2041 hlist_for_each_entry_safe(tb, tmp, head, tb6_hlist) {
2042 hlist_del(&tb->tb6_hlist);
2043 fib6_free_table(tb);
2044 }
2045 }
2046
2047 kfree(net->ipv6.fib_table_hash);
2048 kfree(net->ipv6.rt6_stats);
2049 fib6_notifier_exit(net);
2050 }
2051
2052 static struct pernet_operations fib6_net_ops = {
2053 .init = fib6_net_init,
2054 .exit = fib6_net_exit,
2055 };
2056
2057 int __init fib6_init(void)
2058 {
2059 int ret = -ENOMEM;
2060
2061 fib6_node_kmem = kmem_cache_create("fib6_nodes",
2062 sizeof(struct fib6_node),
2063 0, SLAB_HWCACHE_ALIGN,
2064 NULL);
2065 if (!fib6_node_kmem)
2066 goto out;
2067
2068 ret = register_pernet_subsys(&fib6_net_ops);
2069 if (ret)
2070 goto out_kmem_cache_create;
2071
2072 ret = __rtnl_register(PF_INET6, RTM_GETROUTE, NULL, inet6_dump_fib,
2073 0);
2074 if (ret)
2075 goto out_unregister_subsys;
2076
2077 __fib6_flush_trees = fib6_flush_trees;
2078 out:
2079 return ret;
2080
2081 out_unregister_subsys:
2082 unregister_pernet_subsys(&fib6_net_ops);
2083 out_kmem_cache_create:
2084 kmem_cache_destroy(fib6_node_kmem);
2085 goto out;
2086 }
2087
2088 void fib6_gc_cleanup(void)
2089 {
2090 unregister_pernet_subsys(&fib6_net_ops);
2091 kmem_cache_destroy(fib6_node_kmem);
2092 }
2093
2094 #ifdef CONFIG_PROC_FS
2095
2096 struct ipv6_route_iter {
2097 struct seq_net_private p;
2098 struct fib6_walker w;
2099 loff_t skip;
2100 struct fib6_table *tbl;
2101 int sernum;
2102 };
2103
2104 static int ipv6_route_seq_show(struct seq_file *seq, void *v)
2105 {
2106 struct rt6_info *rt = v;
2107 struct ipv6_route_iter *iter = seq->private;
2108
2109 seq_printf(seq, "%pi6 %02x ", &rt->rt6i_dst.addr, rt->rt6i_dst.plen);
2110
2111 #ifdef CONFIG_IPV6_SUBTREES
2112 seq_printf(seq, "%pi6 %02x ", &rt->rt6i_src.addr, rt->rt6i_src.plen);
2113 #else
2114 seq_puts(seq, "00000000000000000000000000000000 00 ");
2115 #endif
2116 if (rt->rt6i_flags & RTF_GATEWAY)
2117 seq_printf(seq, "%pi6", &rt->rt6i_gateway);
2118 else
2119 seq_puts(seq, "00000000000000000000000000000000");
2120
2121 seq_printf(seq, " %08x %08x %08x %08x %8s\n",
2122 rt->rt6i_metric, atomic_read(&rt->dst.__refcnt),
2123 rt->dst.__use, rt->rt6i_flags,
2124 rt->dst.dev ? rt->dst.dev->name : "");
2125 iter->w.leaf = NULL;
2126 return 0;
2127 }
2128
2129 static int ipv6_route_yield(struct fib6_walker *w)
2130 {
2131 struct ipv6_route_iter *iter = w->args;
2132
2133 if (!iter->skip)
2134 return 1;
2135
2136 do {
2137 iter->w.leaf = iter->w.leaf->dst.rt6_next;
2138 iter->skip--;
2139 if (!iter->skip && iter->w.leaf)
2140 return 1;
2141 } while (iter->w.leaf);
2142
2143 return 0;
2144 }
2145
2146 static void ipv6_route_seq_setup_walk(struct ipv6_route_iter *iter,
2147 struct net *net)
2148 {
2149 memset(&iter->w, 0, sizeof(iter->w));
2150 iter->w.func = ipv6_route_yield;
2151 iter->w.root = &iter->tbl->tb6_root;
2152 iter->w.state = FWS_INIT;
2153 iter->w.node = iter->w.root;
2154 iter->w.args = iter;
2155 iter->sernum = iter->w.root->fn_sernum;
2156 INIT_LIST_HEAD(&iter->w.lh);
2157 fib6_walker_link(net, &iter->w);
2158 }
2159
2160 static struct fib6_table *ipv6_route_seq_next_table(struct fib6_table *tbl,
2161 struct net *net)
2162 {
2163 unsigned int h;
2164 struct hlist_node *node;
2165
2166 if (tbl) {
2167 h = (tbl->tb6_id & (FIB6_TABLE_HASHSZ - 1)) + 1;
2168 node = rcu_dereference_bh(hlist_next_rcu(&tbl->tb6_hlist));
2169 } else {
2170 h = 0;
2171 node = NULL;
2172 }
2173
2174 while (!node && h < FIB6_TABLE_HASHSZ) {
2175 node = rcu_dereference_bh(
2176 hlist_first_rcu(&net->ipv6.fib_table_hash[h++]));
2177 }
2178 return hlist_entry_safe(node, struct fib6_table, tb6_hlist);
2179 }
2180
2181 static void ipv6_route_check_sernum(struct ipv6_route_iter *iter)
2182 {
2183 if (iter->sernum != iter->w.root->fn_sernum) {
2184 iter->sernum = iter->w.root->fn_sernum;
2185 iter->w.state = FWS_INIT;
2186 iter->w.node = iter->w.root;
2187 WARN_ON(iter->w.skip);
2188 iter->w.skip = iter->w.count;
2189 }
2190 }
2191
2192 static void *ipv6_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2193 {
2194 int r;
2195 struct rt6_info *n;
2196 struct net *net = seq_file_net(seq);
2197 struct ipv6_route_iter *iter = seq->private;
2198
2199 if (!v)
2200 goto iter_table;
2201
2202 n = ((struct rt6_info *)v)->dst.rt6_next;
2203 if (n) {
2204 ++*pos;
2205 return n;
2206 }
2207
2208 iter_table:
2209 ipv6_route_check_sernum(iter);
2210 read_lock(&iter->tbl->tb6_lock);
2211 r = fib6_walk_continue(&iter->w);
2212 read_unlock(&iter->tbl->tb6_lock);
2213 if (r > 0) {
2214 if (v)
2215 ++*pos;
2216 return iter->w.leaf;
2217 } else if (r < 0) {
2218 fib6_walker_unlink(net, &iter->w);
2219 return NULL;
2220 }
2221 fib6_walker_unlink(net, &iter->w);
2222
2223 iter->tbl = ipv6_route_seq_next_table(iter->tbl, net);
2224 if (!iter->tbl)
2225 return NULL;
2226
2227 ipv6_route_seq_setup_walk(iter, net);
2228 goto iter_table;
2229 }
2230
2231 static void *ipv6_route_seq_start(struct seq_file *seq, loff_t *pos)
2232 __acquires(RCU_BH)
2233 {
2234 struct net *net = seq_file_net(seq);
2235 struct ipv6_route_iter *iter = seq->private;
2236
2237 rcu_read_lock_bh();
2238 iter->tbl = ipv6_route_seq_next_table(NULL, net);
2239 iter->skip = *pos;
2240
2241 if (iter->tbl) {
2242 ipv6_route_seq_setup_walk(iter, net);
2243 return ipv6_route_seq_next(seq, NULL, pos);
2244 } else {
2245 return NULL;
2246 }
2247 }
2248
2249 static bool ipv6_route_iter_active(struct ipv6_route_iter *iter)
2250 {
2251 struct fib6_walker *w = &iter->w;
2252 return w->node && !(w->state == FWS_U && w->node == w->root);
2253 }
2254
2255 static void ipv6_route_seq_stop(struct seq_file *seq, void *v)
2256 __releases(RCU_BH)
2257 {
2258 struct net *net = seq_file_net(seq);
2259 struct ipv6_route_iter *iter = seq->private;
2260
2261 if (ipv6_route_iter_active(iter))
2262 fib6_walker_unlink(net, &iter->w);
2263
2264 rcu_read_unlock_bh();
2265 }
2266
2267 static const struct seq_operations ipv6_route_seq_ops = {
2268 .start = ipv6_route_seq_start,
2269 .next = ipv6_route_seq_next,
2270 .stop = ipv6_route_seq_stop,
2271 .show = ipv6_route_seq_show
2272 };
2273
2274 int ipv6_route_open(struct inode *inode, struct file *file)
2275 {
2276 return seq_open_net(inode, file, &ipv6_route_seq_ops,
2277 sizeof(struct ipv6_route_iter));
2278 }
2279
2280 #endif /* CONFIG_PROC_FS */