projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
mlxsw: spectrum_router: Make FIB node retrieval family agnostic
[GitHub/LineageOS/android_kernel_motorola_exynos9610.git] / drivers / net / ethernet / mellanox / mlxsw / spectrum_router.c
CommitLineData
464dce18
IS		 1/*
2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
c723c735 6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
464dce18
IS		 7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the names of the copyright holders nor the names of its
17 * contributors may be used to endorse or promote products derived from
18 * this software without specific prior written permission.
19 *
20 * Alternatively, this software may be distributed under the terms of the
21 * GNU General Public License ("GPL") version 2 as published by the Free
22 * Software Foundation.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
35 */
36
37#include <linux/kernel.h>
38#include <linux/types.h>
5e9c16cc
JP		 39#include <linux/rhashtable.h>
40#include <linux/bitops.h>
41#include <linux/in6.h>
c723c735 42#include <linux/notifier.h>
df6dd79b 43#include <linux/inetdevice.h>
9db032bb 44#include <linux/netdevice.h>
03ea01e9 45#include <linux/if_bridge.h>
c723c735 46#include <net/netevent.h>
6cf3c971
JP		 47#include <net/neighbour.h>
48#include <net/arp.h>
b45f64d1 49#include <net/ip_fib.h>
5d7bfd14 50#include <net/fib_rules.h>
57837885 51#include <net/l3mdev.h>
5ea1237f 52#include <net/addrconf.h>
d5eb89cf
AS		 53#include <net/ndisc.h>
54#include <net/ipv6.h>
464dce18
IS		 55
56#include "spectrum.h"
57#include "core.h"
58#include "reg.h"
e0c0afd8
AS		 59#include "spectrum_cnt.h"
60#include "spectrum_dpipe.h"
61#include "spectrum_router.h"
464dce18 62
9011b677
IS		 63struct mlxsw_sp_vr;
64struct mlxsw_sp_lpm_tree;
e4f3c1c1 65struct mlxsw_sp_rif_ops;
9011b677
IS		 66
67struct mlxsw_sp_router {
68 struct mlxsw_sp *mlxsw_sp;
5f9efffb 69 struct mlxsw_sp_rif **rifs;
9011b677
IS		 70 struct mlxsw_sp_vr *vrs;
71 struct rhashtable neigh_ht;
72 struct rhashtable nexthop_group_ht;
73 struct rhashtable nexthop_ht;
74 struct {
75 struct mlxsw_sp_lpm_tree *trees;
76 unsigned int tree_count;
77 } lpm;
78 struct {
79 struct delayed_work dw;
80 unsigned long interval; /* ms */
81 } neighs_update;
82 struct delayed_work nexthop_probe_dw;
83#define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
84 struct list_head nexthop_neighs_list;
85 bool aborted;
7e39d115 86 struct notifier_block fib_nb;
e4f3c1c1 87 const struct mlxsw_sp_rif_ops **rif_ops_arr;
9011b677
IS		 88};
89
4724ba56
IS		 90struct mlxsw_sp_rif {
91 struct list_head nexthop_list;
92 struct list_head neigh_list;
93 struct net_device *dev;
a1107487 94 struct mlxsw_sp_fid *fid;
4724ba56
IS		 95 unsigned char addr[ETH_ALEN];
96 int mtu;
bf95233e 97 u16 rif_index;
6913229e 98 u16 vr_id;
e4f3c1c1
IS		 99 const struct mlxsw_sp_rif_ops *ops;
100 struct mlxsw_sp *mlxsw_sp;
101
e0c0afd8
AS		 102 unsigned int counter_ingress;
103 bool counter_ingress_valid;
104 unsigned int counter_egress;
105 bool counter_egress_valid;
4724ba56
IS		 106};
107
e4f3c1c1
IS		 108struct mlxsw_sp_rif_params {
109 struct net_device *dev;
110 union {
111 u16 system_port;
112 u16 lag_id;
113 };
114 u16 vid;
115 bool lag;
116};
117
4d93ceeb
IS		 118struct mlxsw_sp_rif_subport {
119 struct mlxsw_sp_rif common;
120 union {
121 u16 system_port;
122 u16 lag_id;
123 };
124 u16 vid;
125 bool lag;
126};
127
e4f3c1c1
IS		 128struct mlxsw_sp_rif_ops {
129 enum mlxsw_sp_rif_type type;
130 size_t rif_size;
131
132 void (*setup)(struct mlxsw_sp_rif *rif,
133 const struct mlxsw_sp_rif_params *params);
134 int (*configure)(struct mlxsw_sp_rif *rif);
135 void (*deconfigure)(struct mlxsw_sp_rif *rif);
136 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
137};
138
e0c0afd8
AS		 139static unsigned int *
140mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
141 enum mlxsw_sp_rif_counter_dir dir)
142{
143 switch (dir) {
144 case MLXSW_SP_RIF_COUNTER_EGRESS:
145 return &rif->counter_egress;
146 case MLXSW_SP_RIF_COUNTER_INGRESS:
147 return &rif->counter_ingress;
148 }
149 return NULL;
150}
151
152static bool
153mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
154 enum mlxsw_sp_rif_counter_dir dir)
155{
156 switch (dir) {
157 case MLXSW_SP_RIF_COUNTER_EGRESS:
158 return rif->counter_egress_valid;
159 case MLXSW_SP_RIF_COUNTER_INGRESS:
160 return rif->counter_ingress_valid;
161 }
162 return false;
163}
164
165static void
166mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
167 enum mlxsw_sp_rif_counter_dir dir,
168 bool valid)
169{
170 switch (dir) {
171 case MLXSW_SP_RIF_COUNTER_EGRESS:
172 rif->counter_egress_valid = valid;
173 break;
174 case MLXSW_SP_RIF_COUNTER_INGRESS:
175 rif->counter_ingress_valid = valid;
176 break;
177 }
178}
179
180static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
181 unsigned int counter_index, bool enable,
182 enum mlxsw_sp_rif_counter_dir dir)
183{
184 char ritr_pl[MLXSW_REG_RITR_LEN];
185 bool is_egress = false;
186 int err;
187
188 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
189 is_egress = true;
190 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
191 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
192 if (err)
193 return err;
194
195 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
196 is_egress);
197 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
198}
199
200int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
201 struct mlxsw_sp_rif *rif,
202 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
203{
204 char ricnt_pl[MLXSW_REG_RICNT_LEN];
205 unsigned int *p_counter_index;
206 bool valid;
207 int err;
208
209 valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
210 if (!valid)
211 return -EINVAL;
212
213 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
214 if (!p_counter_index)
215 return -EINVAL;
216 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
217 MLXSW_REG_RICNT_OPCODE_NOP);
218 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
219 if (err)
220 return err;
221 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
222 return 0;
223}
224
225static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
226 unsigned int counter_index)
227{
228 char ricnt_pl[MLXSW_REG_RICNT_LEN];
229
230 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
231 MLXSW_REG_RICNT_OPCODE_CLEAR);
232 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
233}
234
235int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
236 struct mlxsw_sp_rif *rif,
237 enum mlxsw_sp_rif_counter_dir dir)
238{
239 unsigned int *p_counter_index;
240 int err;
241
242 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
243 if (!p_counter_index)
244 return -EINVAL;
245 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
246 p_counter_index);
247 if (err)
248 return err;
249
250 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
251 if (err)
252 goto err_counter_clear;
253
254 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
255 *p_counter_index, true, dir);
256 if (err)
257 goto err_counter_edit;
258 mlxsw_sp_rif_counter_valid_set(rif, dir, true);
259 return 0;
260
261err_counter_edit:
262err_counter_clear:
263 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
264 *p_counter_index);
265 return err;
266}
267
268void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
269 struct mlxsw_sp_rif *rif,
270 enum mlxsw_sp_rif_counter_dir dir)
271{
272 unsigned int *p_counter_index;
273
6b1206bb
AS		 274 if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
275 return;
276
e0c0afd8
AS		 277 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
278 if (WARN_ON(!p_counter_index))
279 return;
280 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
281 *p_counter_index, false, dir);
282 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
283 *p_counter_index);
284 mlxsw_sp_rif_counter_valid_set(rif, dir, false);
285}
286
e4f3c1c1
IS		 287static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
288{
289 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
290 struct devlink *devlink;
291
292 devlink = priv_to_devlink(mlxsw_sp->core);
293 if (!devlink_dpipe_table_counter_enabled(devlink,
294 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
295 return;
296 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
297}
298
299static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
300{
301 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
302
303 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
304}
305
4724ba56
IS		 306static struct mlxsw_sp_rif *
307mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
308 const struct net_device *dev);
309
9011b677
IS		 310#define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE)
311
312struct mlxsw_sp_prefix_usage {
313 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
314};
315
53342023
JP		 316#define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
317 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
318
6b75c480
JP		 319static bool
320mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
321 struct mlxsw_sp_prefix_usage *prefix_usage2)
322{
323 unsigned char prefix;
324
325 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
326 if (!test_bit(prefix, prefix_usage2->b))
327 return false;
328 }
329 return true;
330}
331
53342023
JP		 332static bool
333mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
334 struct mlxsw_sp_prefix_usage *prefix_usage2)
335{
336 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
337}
338
6b75c480
JP		 339static bool
340mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
341{
342 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
343
344 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
345}
346
347static void
348mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
349 struct mlxsw_sp_prefix_usage *prefix_usage2)
350{
351 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
352}
353
5e9c16cc
JP		 354static void
355mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
356 unsigned char prefix_len)
357{
358 set_bit(prefix_len, prefix_usage->b);
359}
360
361static void
362mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
363 unsigned char prefix_len)
364{
365 clear_bit(prefix_len, prefix_usage->b);
366}
367
368struct mlxsw_sp_fib_key {
369 unsigned char addr[sizeof(struct in6_addr)];
370 unsigned char prefix_len;
371};
372
61c503f9
JP		 373enum mlxsw_sp_fib_entry_type {
374 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
375 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
376 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
377};
378
a7ff87ac 379struct mlxsw_sp_nexthop_group;
9011b677 380struct mlxsw_sp_fib;
a7ff87ac 381
9aecce1c
IS		 382struct mlxsw_sp_fib_node {
383 struct list_head entry_list;
b45f64d1 384 struct list_head list;
9aecce1c 385 struct rhash_head ht_node;
76610ebb 386 struct mlxsw_sp_fib *fib;
5e9c16cc 387 struct mlxsw_sp_fib_key key;
9aecce1c
IS		 388};
389
390struct mlxsw_sp_fib_entry_params {
391 u32 tb_id;
392 u32 prio;
393 u8 tos;
394 u8 type;
395};
396
397struct mlxsw_sp_fib_entry {
398 struct list_head list;
399 struct mlxsw_sp_fib_node *fib_node;
61c503f9 400 enum mlxsw_sp_fib_entry_type type;
a7ff87ac
JP		 401 struct list_head nexthop_group_node;
402 struct mlxsw_sp_nexthop_group *nh_group;
9aecce1c 403 struct mlxsw_sp_fib_entry_params params;
013b20f9 404 bool offloaded;
5e9c16cc
JP		 405};
406
9011b677
IS		 407enum mlxsw_sp_l3proto {
408 MLXSW_SP_L3_PROTO_IPV4,
409 MLXSW_SP_L3_PROTO_IPV6,
410};
411
412struct mlxsw_sp_lpm_tree {
413 u8 id; /* tree ID */
414 unsigned int ref_count;
415 enum mlxsw_sp_l3proto proto;
416 struct mlxsw_sp_prefix_usage prefix_usage;
417};
418
5e9c16cc
JP		 419struct mlxsw_sp_fib {
420 struct rhashtable ht;
9aecce1c 421 struct list_head node_list;
76610ebb
IS		 422 struct mlxsw_sp_vr *vr;
423 struct mlxsw_sp_lpm_tree *lpm_tree;
5e9c16cc
JP		 424 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
425 struct mlxsw_sp_prefix_usage prefix_usage;
76610ebb 426 enum mlxsw_sp_l3proto proto;
5e9c16cc
JP		 427};
428
9011b677
IS		 429struct mlxsw_sp_vr {
430 u16 id; /* virtual router ID */
431 u32 tb_id; /* kernel fib table id */
432 unsigned int rif_count;
433 struct mlxsw_sp_fib *fib4;
434};
435
9aecce1c 436static const struct rhashtable_params mlxsw_sp_fib_ht_params;
5e9c16cc 437
76610ebb
IS		 438static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
439 enum mlxsw_sp_l3proto proto)
5e9c16cc
JP		 440{
441 struct mlxsw_sp_fib *fib;
442 int err;
443
444 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
445 if (!fib)
446 return ERR_PTR(-ENOMEM);
447 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
448 if (err)
449 goto err_rhashtable_init;
9aecce1c 450 INIT_LIST_HEAD(&fib->node_list);
76610ebb
IS		 451 fib->proto = proto;
452 fib->vr = vr;
5e9c16cc
JP		 453 return fib;
454
455err_rhashtable_init:
456 kfree(fib);
457 return ERR_PTR(err);
458}
459
460static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
461{
9aecce1c 462 WARN_ON(!list_empty(&fib->node_list));
76610ebb 463 WARN_ON(fib->lpm_tree);
5e9c16cc
JP		 464 rhashtable_destroy(&fib->ht);
465 kfree(fib);
466}
467
53342023 468static struct mlxsw_sp_lpm_tree *
382dbb40 469mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
53342023
JP		 470{
471 static struct mlxsw_sp_lpm_tree *lpm_tree;
472 int i;
473
9011b677
IS		 474 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
475 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
382dbb40
IS		 476 if (lpm_tree->ref_count == 0)
477 return lpm_tree;
53342023
JP		 478 }
479 return NULL;
480}
481
482static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
483 struct mlxsw_sp_lpm_tree *lpm_tree)
484{
485 char ralta_pl[MLXSW_REG_RALTA_LEN];
486
1a9234e6
IS		 487 mlxsw_reg_ralta_pack(ralta_pl, true,
488 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
489 lpm_tree->id);
53342023
JP		 490 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
491}
492
493static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
494 struct mlxsw_sp_lpm_tree *lpm_tree)
495{
496 char ralta_pl[MLXSW_REG_RALTA_LEN];
497
1a9234e6
IS		 498 mlxsw_reg_ralta_pack(ralta_pl, false,
499 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
500 lpm_tree->id);
53342023
JP		 501 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
502}
503
504static int
505mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
506 struct mlxsw_sp_prefix_usage *prefix_usage,
507 struct mlxsw_sp_lpm_tree *lpm_tree)
508{
509 char ralst_pl[MLXSW_REG_RALST_LEN];
510 u8 root_bin = 0;
511 u8 prefix;
512 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
513
514 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
515 root_bin = prefix;
516
517 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
518 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
519 if (prefix == 0)
520 continue;
521 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
522 MLXSW_REG_RALST_BIN_NO_CHILD);
523 last_prefix = prefix;
524 }
525 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
526}
527
528static struct mlxsw_sp_lpm_tree *
529mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
530 struct mlxsw_sp_prefix_usage *prefix_usage,
382dbb40 531 enum mlxsw_sp_l3proto proto)
53342023
JP		 532{
533 struct mlxsw_sp_lpm_tree *lpm_tree;
534 int err;
535
382dbb40 536 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
53342023
JP		 537 if (!lpm_tree)
538 return ERR_PTR(-EBUSY);
539 lpm_tree->proto = proto;
540 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
541 if (err)
542 return ERR_PTR(err);
543
544 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
545 lpm_tree);
546 if (err)
547 goto err_left_struct_set;
2083d367
JP		 548 memcpy(&lpm_tree->prefix_usage, prefix_usage,
549 sizeof(lpm_tree->prefix_usage));
53342023
JP		 550 return lpm_tree;
551
552err_left_struct_set:
553 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
554 return ERR_PTR(err);
555}
556
557static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
558 struct mlxsw_sp_lpm_tree *lpm_tree)
559{
560 return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
561}
562
563static struct mlxsw_sp_lpm_tree *
564mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
565 struct mlxsw_sp_prefix_usage *prefix_usage,
382dbb40 566 enum mlxsw_sp_l3proto proto)
53342023
JP		 567{
568 struct mlxsw_sp_lpm_tree *lpm_tree;
569 int i;
570
9011b677
IS		 571 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
572 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
8b99becd
JP		 573 if (lpm_tree->ref_count != 0 &&
574 lpm_tree->proto == proto &&
53342023
JP		 575 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
576 prefix_usage))
577 goto inc_ref_count;
578 }
579 lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
382dbb40 580 proto);
53342023
JP		 581 if (IS_ERR(lpm_tree))
582 return lpm_tree;
583
584inc_ref_count:
585 lpm_tree->ref_count++;
586 return lpm_tree;
587}
588
589static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
590 struct mlxsw_sp_lpm_tree *lpm_tree)
591{
592 if (--lpm_tree->ref_count == 0)
593 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
594 return 0;
595}
596
d7a60306 597#define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
8494ab06
IS		 598
599static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
53342023
JP		 600{
601 struct mlxsw_sp_lpm_tree *lpm_tree;
8494ab06 602 u64 max_trees;
53342023
JP		 603 int i;
604
8494ab06
IS		 605 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
606 return -EIO;
607
608 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
9011b677
IS		 609 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
610 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
8494ab06
IS		 611 sizeof(struct mlxsw_sp_lpm_tree),
612 GFP_KERNEL);
9011b677 613 if (!mlxsw_sp->router->lpm.trees)
8494ab06
IS		 614 return -ENOMEM;
615
9011b677
IS		 616 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
617 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
53342023
JP		 618 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
619 }
8494ab06
IS		 620
621 return 0;
622}
623
624static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
625{
9011b677 626 kfree(mlxsw_sp->router->lpm.trees);
53342023
JP		 627}
628
76610ebb
IS		 629static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
630{
631 return !!vr->fib4;
632}
633
6b75c480
JP		 634static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
635{
636 struct mlxsw_sp_vr *vr;
637 int i;
638
c1a38311 639 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
9011b677 640 vr = &mlxsw_sp->router->vrs[i];
76610ebb 641 if (!mlxsw_sp_vr_is_used(vr))
6b75c480
JP		 642 return vr;
643 }
644 return NULL;
645}
646
647static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
76610ebb 648 const struct mlxsw_sp_fib *fib)
6b75c480
JP		 649{
650 char raltb_pl[MLXSW_REG_RALTB_LEN];
651
76610ebb
IS		 652 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
653 (enum mlxsw_reg_ralxx_protocol) fib->proto,
654 fib->lpm_tree->id);
6b75c480
JP		 655 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
656}
657
658static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
76610ebb 659 const struct mlxsw_sp_fib *fib)
6b75c480
JP		 660{
661 char raltb_pl[MLXSW_REG_RALTB_LEN];
662
663 /* Bind to tree 0 which is default */
76610ebb
IS		 664 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
665 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
6b75c480
JP		 666 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
667}
668
669static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
670{
671 /* For our purpose, squash main and local table into one */
672 if (tb_id == RT_TABLE_LOCAL)
673 tb_id = RT_TABLE_MAIN;
674 return tb_id;
675}
676
677static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
76610ebb 678 u32 tb_id)
6b75c480
JP		 679{
680 struct mlxsw_sp_vr *vr;
681 int i;
682
683 tb_id = mlxsw_sp_fix_tb_id(tb_id);
9497c042 684
c1a38311 685 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
9011b677 686 vr = &mlxsw_sp->router->vrs[i];
76610ebb 687 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
6b75c480
JP		 688 return vr;
689 }
690 return NULL;
691}
692
76610ebb
IS		 693static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
694 enum mlxsw_sp_l3proto proto)
695{
696 switch (proto) {
697 case MLXSW_SP_L3_PROTO_IPV4:
698 return vr->fib4;
699 case MLXSW_SP_L3_PROTO_IPV6:
700 BUG_ON(1);
701 }
702 return NULL;
703}
704
6b75c480 705static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
76610ebb 706 u32 tb_id)
6b75c480 707{
6b75c480 708 struct mlxsw_sp_vr *vr;
6b75c480
JP		 709
710 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
711 if (!vr)
712 return ERR_PTR(-EBUSY);
76610ebb
IS		 713 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
714 if (IS_ERR(vr->fib4))
715 return ERR_CAST(vr->fib4);
6b75c480 716 vr->tb_id = tb_id;
6b75c480 717 return vr;
6b75c480
JP		 718}
719
76610ebb 720static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
6b75c480 721{
76610ebb
IS		 722 mlxsw_sp_fib_destroy(vr->fib4);
723 vr->fib4 = NULL;
6b75c480
JP		 724}
725
726static int
76610ebb 727mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
6b75c480
JP		 728 struct mlxsw_sp_prefix_usage *req_prefix_usage)
729{
76610ebb 730 struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
f7df4923
IS		 731 struct mlxsw_sp_lpm_tree *new_tree;
732 int err;
6b75c480 733
f7df4923 734 if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
6b75c480
JP		 735 return 0;
736
f7df4923 737 new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
76610ebb 738 fib->proto);
f7df4923 739 if (IS_ERR(new_tree)) {
6b75c480
JP		 740 /* We failed to get a tree according to the required
741 * prefix usage. However, the current tree might be still good
742 * for us if our requirement is subset of the prefixes used
743 * in the tree.
744 */
745 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
f7df4923 746 &lpm_tree->prefix_usage))
6b75c480 747 return 0;
f7df4923 748 return PTR_ERR(new_tree);
6b75c480
JP		 749 }
750
f7df4923 751 /* Prevent packet loss by overwriting existing binding */
76610ebb
IS		 752 fib->lpm_tree = new_tree;
753 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
f7df4923
IS		 754 if (err)
755 goto err_tree_bind;
756 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
757
758 return 0;
759
760err_tree_bind:
76610ebb 761 fib->lpm_tree = lpm_tree;
f7df4923
IS		 762 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
763 return err;
6b75c480
JP		 764}
765
76610ebb 766static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
6b75c480
JP		 767{
768 struct mlxsw_sp_vr *vr;
6b75c480
JP		 769
770 tb_id = mlxsw_sp_fix_tb_id(tb_id);
76610ebb
IS		 771 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
772 if (!vr)
773 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
6b75c480
JP		 774 return vr;
775}
776
76610ebb 777static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
6b75c480 778{
6913229e 779 if (!vr->rif_count && list_empty(&vr->fib4->node_list))
76610ebb 780 mlxsw_sp_vr_destroy(vr);
6b75c480
JP		 781}
782
9497c042 783static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
6b75c480
JP		 784{
785 struct mlxsw_sp_vr *vr;
c1a38311 786 u64 max_vrs;
6b75c480
JP		 787 int i;
788
c1a38311 789 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
9497c042
NF		 790 return -EIO;
791
c1a38311 792 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
9011b677
IS		 793 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
794 GFP_KERNEL);
795 if (!mlxsw_sp->router->vrs)
9497c042
NF		 796 return -ENOMEM;
797
c1a38311 798 for (i = 0; i < max_vrs; i++) {
9011b677 799 vr = &mlxsw_sp->router->vrs[i];
6b75c480
JP		 800 vr->id = i;
801 }
9497c042
NF		 802
803 return 0;
804}
805
ac571de9
IS		 806static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
807
9497c042
NF		 808static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
809{
3057224e
IS		 810 /* At this stage we're guaranteed not to have new incoming
811 * FIB notifications and the work queue is free from FIBs
812 * sitting on top of mlxsw netdevs. However, we can still
813 * have other FIBs queued. Flush the queue before flushing
814 * the device's tables. No need for locks, as we're the only
815 * writer.
816 */
817 mlxsw_core_flush_owq();
ac571de9 818 mlxsw_sp_router_fib_flush(mlxsw_sp);
9011b677 819 kfree(mlxsw_sp->router->vrs);
6b75c480
JP		 820}
821
6cf3c971 822struct mlxsw_sp_neigh_key {
33b1341c 823 struct neighbour *n;
6cf3c971
JP		 824};
825
826struct mlxsw_sp_neigh_entry {
9665b745 827 struct list_head rif_list_node;
6cf3c971
JP		 828 struct rhash_head ht_node;
829 struct mlxsw_sp_neigh_key key;
830 u16 rif;
5c8802f1 831 bool connected;
a6bf9e93 832 unsigned char ha[ETH_ALEN];
a7ff87ac
JP		 833 struct list_head nexthop_list; /* list of nexthops using
834 * this neigh entry
835 */
b2157149 836 struct list_head nexthop_neighs_list_node;
6cf3c971
JP		 837};
838
839static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
840 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
841 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
842 .key_len = sizeof(struct mlxsw_sp_neigh_key),
843};
844
6cf3c971 845static struct mlxsw_sp_neigh_entry *
5c8802f1
IS		 846mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
847 u16 rif)
6cf3c971
JP		 848{
849 struct mlxsw_sp_neigh_entry *neigh_entry;
850
5c8802f1 851 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
6cf3c971
JP		 852 if (!neigh_entry)
853 return NULL;
5c8802f1 854
33b1341c 855 neigh_entry->key.n = n;
6cf3c971 856 neigh_entry->rif = rif;
a7ff87ac 857 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
5c8802f1 858
6cf3c971
JP		 859 return neigh_entry;
860}
861
5c8802f1 862static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
6cf3c971
JP		 863{
864 kfree(neigh_entry);
865}
866
5c8802f1
IS		 867static int
868mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
869 struct mlxsw_sp_neigh_entry *neigh_entry)
6cf3c971 870{
9011b677 871 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
5c8802f1
IS		 872 &neigh_entry->ht_node,
873 mlxsw_sp_neigh_ht_params);
874}
6cf3c971 875
5c8802f1
IS		 876static void
877mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
878 struct mlxsw_sp_neigh_entry *neigh_entry)
879{
9011b677 880 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
5c8802f1
IS		 881 &neigh_entry->ht_node,
882 mlxsw_sp_neigh_ht_params);
6cf3c971
JP		 883}
884
5c8802f1
IS		 885static struct mlxsw_sp_neigh_entry *
886mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
6cf3c971 887{
6cf3c971 888 struct mlxsw_sp_neigh_entry *neigh_entry;
bf95233e 889 struct mlxsw_sp_rif *rif;
6cf3c971
JP		 890 int err;
891
bf95233e
AS		 892 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
893 if (!rif)
5c8802f1 894 return ERR_PTR(-EINVAL);
6cf3c971 895
bf95233e 896 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
6cf3c971 897 if (!neigh_entry)
5c8802f1
IS		 898 return ERR_PTR(-ENOMEM);
899
6cf3c971
JP		 900 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
901 if (err)
902 goto err_neigh_entry_insert;
5c8802f1 903
bf95233e 904 list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
9665b745 905
5c8802f1 906 return neigh_entry;
6cf3c971
JP		 907
908err_neigh_entry_insert:
5c8802f1
IS		 909 mlxsw_sp_neigh_entry_free(neigh_entry);
910 return ERR_PTR(err);
6cf3c971
JP		 911}
912
5c8802f1
IS		 913static void
914mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
915 struct mlxsw_sp_neigh_entry *neigh_entry)
6cf3c971 916{
9665b745 917 list_del(&neigh_entry->rif_list_node);
5c8802f1
IS		 918 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
919 mlxsw_sp_neigh_entry_free(neigh_entry);
920}
6cf3c971 921
5c8802f1
IS		 922static struct mlxsw_sp_neigh_entry *
923mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
924{
925 struct mlxsw_sp_neigh_key key;
6cf3c971 926
5c8802f1 927 key.n = n;
9011b677 928 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
5c8802f1 929 &key, mlxsw_sp_neigh_ht_params);
6cf3c971
JP		 930}
931
c723c735
YG		 932static void
933mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
934{
a6c9b5d1 935 unsigned long interval;
c723c735 936
a6c9b5d1
AS		 937 interval = min_t(unsigned long,
938 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
939 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
9011b677 940 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
c723c735
YG		 941}
942
943static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
944 char *rauhtd_pl,
945 int ent_index)
946{
947 struct net_device *dev;
948 struct neighbour *n;
949 __be32 dipn;
950 u32 dip;
951 u16 rif;
952
953 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
954
5f9efffb 955 if (!mlxsw_sp->router->rifs[rif]) {
c723c735
YG		 956 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
957 return;
958 }
959
960 dipn = htonl(dip);
5f9efffb 961 dev = mlxsw_sp->router->rifs[rif]->dev;
c723c735
YG		 962 n = neigh_lookup(&arp_tbl, &dipn, dev);
963 if (!n) {
964 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
965 &dip);
966 return;
967 }
968
969 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
970 neigh_event_send(n, NULL);
971 neigh_release(n);
972}
973
60f040ca
AS		 974static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
975 char *rauhtd_pl,
976 int rec_index)
977{
978 struct net_device *dev;
979 struct neighbour *n;
980 struct in6_addr dip;
981 u16 rif;
982
983 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
984 (char *) &dip);
985
986 if (!mlxsw_sp->router->rifs[rif]) {
987 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
988 return;
989 }
990
991 dev = mlxsw_sp->router->rifs[rif]->dev;
992 n = neigh_lookup(&nd_tbl, &dip, dev);
993 if (!n) {
994 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n",
995 &dip);
996 return;
997 }
998
999 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
1000 neigh_event_send(n, NULL);
1001 neigh_release(n);
1002}
1003
c723c735
YG		 1004static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1005 char *rauhtd_pl,
1006 int rec_index)
1007{
1008 u8 num_entries;
1009 int i;
1010
1011 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1012 rec_index);
1013 /* Hardware starts counting at 0, so add 1. */
1014 num_entries++;
1015
1016 /* Each record consists of several neighbour entries. */
1017 for (i = 0; i < num_entries; i++) {
1018 int ent_index;
1019
1020 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
1021 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
1022 ent_index);
1023 }
1024
1025}
1026
60f040ca
AS		 1027static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1028 char *rauhtd_pl,
1029 int rec_index)
1030{
1031 /* One record contains one entry. */
1032 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
1033 rec_index);
1034}
1035
c723c735
YG		 1036static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
1037 char *rauhtd_pl, int rec_index)
1038{
1039 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
1040 case MLXSW_REG_RAUHTD_TYPE_IPV4:
1041 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
1042 rec_index);
1043 break;
1044 case MLXSW_REG_RAUHTD_TYPE_IPV6:
60f040ca
AS		 1045 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
1046 rec_index);
c723c735
YG		 1047 break;
1048 }
1049}
1050
42cdb338
AS		 1051static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1052{
1053 u8 num_rec, last_rec_index, num_entries;
1054
1055 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1056 last_rec_index = num_rec - 1;
1057
1058 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1059 return false;
1060 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1061 MLXSW_REG_RAUHTD_TYPE_IPV6)
1062 return true;
1063
1064 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1065 last_rec_index);
1066 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1067 return true;
1068 return false;
1069}
1070
60f040ca
AS		 1071static int
1072__mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
1073 char *rauhtd_pl,
1074 enum mlxsw_reg_rauhtd_type type)
c723c735 1075{
60f040ca
AS		 1076 int i, num_rec;
1077 int err;
c723c735
YG		 1078
1079 /* Make sure the neighbour's netdev isn't removed in the
1080 * process.
1081 */
1082 rtnl_lock();
1083 do {
60f040ca 1084 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
c723c735
YG		 1085 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1086 rauhtd_pl);
1087 if (err) {
1088 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1089 break;
1090 }
1091 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1092 for (i = 0; i < num_rec; i++)
1093 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1094 i);
42cdb338 1095 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
c723c735
YG		 1096 rtnl_unlock();
1097
60f040ca
AS		 1098 return err;
1099}
1100
1101static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1102{
1103 enum mlxsw_reg_rauhtd_type type;
1104 char *rauhtd_pl;
1105 int err;
1106
1107 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1108 if (!rauhtd_pl)
1109 return -ENOMEM;
1110
1111 type = MLXSW_REG_RAUHTD_TYPE_IPV4;
1112 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1113 if (err)
1114 goto out;
1115
1116 type = MLXSW_REG_RAUHTD_TYPE_IPV6;
1117 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1118out:
c723c735 1119 kfree(rauhtd_pl);
b2157149
YG		 1120 return err;
1121}
1122
1123static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1124{
1125 struct mlxsw_sp_neigh_entry *neigh_entry;
1126
1127 /* Take RTNL mutex here to prevent lists from changes */
1128 rtnl_lock();
9011b677 1129 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
8a0b7275 1130 nexthop_neighs_list_node)
b2157149
YG		 1131 /* If this neigh have nexthops, make the kernel think this neigh
1132 * is active regardless of the traffic.
1133 */
8a0b7275 1134 neigh_event_send(neigh_entry->key.n, NULL);
b2157149
YG		 1135 rtnl_unlock();
1136}
1137
1138static void
1139mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1140{
9011b677 1141 unsigned long interval = mlxsw_sp->router->neighs_update.interval;
b2157149 1142
9011b677 1143 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
b2157149
YG		 1144 msecs_to_jiffies(interval));
1145}
1146
1147static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1148{
9011b677 1149 struct mlxsw_sp_router *router;
b2157149
YG		 1150 int err;
1151
9011b677
IS		 1152 router = container_of(work, struct mlxsw_sp_router,
1153 neighs_update.dw.work);
1154 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
b2157149 1155 if (err)
9011b677 1156 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
b2157149 1157
9011b677 1158 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
b2157149 1159
9011b677 1160 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
c723c735
YG		 1161}
1162
0b2361d9
YG		 1163static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1164{
1165 struct mlxsw_sp_neigh_entry *neigh_entry;
9011b677 1166 struct mlxsw_sp_router *router;
0b2361d9 1167
9011b677
IS		 1168 router = container_of(work, struct mlxsw_sp_router,
1169 nexthop_probe_dw.work);
0b2361d9
YG		 1170 /* Iterate over nexthop neighbours, find those who are unresolved and
1171 * send arp on them. This solves the chicken-egg problem when
1172 * the nexthop wouldn't get offloaded until the neighbor is resolved
1173 * but it wouldn't get resolved ever in case traffic is flowing in HW
1174 * using different nexthop.
1175 *
1176 * Take RTNL mutex here to prevent lists from changes.
1177 */
1178 rtnl_lock();
9011b677 1179 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
8a0b7275 1180 nexthop_neighs_list_node)
01b1aa35 1181 if (!neigh_entry->connected)
33b1341c 1182 neigh_event_send(neigh_entry->key.n, NULL);
0b2361d9
YG		 1183 rtnl_unlock();
1184
9011b677 1185 mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
0b2361d9
YG		 1186 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1187}
1188
a7ff87ac
JP		 1189static void
1190mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1191 struct mlxsw_sp_neigh_entry *neigh_entry,
1192 bool removing);
1193
5c8802f1
IS		 1194static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1195{
1196 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1197 MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1198}
1199
1200static void
1201mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1202 struct mlxsw_sp_neigh_entry *neigh_entry,
1203 enum mlxsw_reg_rauht_op op)
a6bf9e93 1204{
33b1341c 1205 struct neighbour *n = neigh_entry->key.n;
5c8802f1 1206 u32 dip = ntohl(*((__be32 *) n->primary_key));
a6bf9e93 1207 char rauht_pl[MLXSW_REG_RAUHT_LEN];
5c8802f1
IS		 1208
1209 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1210 dip);
1211 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1212}
1213
d5eb89cf
AS		 1214static void
1215mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
1216 struct mlxsw_sp_neigh_entry *neigh_entry,
1217 enum mlxsw_reg_rauht_op op)
1218{
1219 struct neighbour *n = neigh_entry->key.n;
1220 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1221 const char *dip = n->primary_key;
1222
1223 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1224 dip);
1225 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1226}
1227
1228static bool mlxsw_sp_neigh_ipv6_ignore(struct neighbour *n)
1229{
1230 /* Packets with a link-local destination address are trapped
1231 * after LPM lookup and never reach the neighbour table, so
1232 * there is no need to program such neighbours to the device.
1233 */
1234 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
1235 IPV6_ADDR_LINKLOCAL)
1236 return true;
1237 return false;
1238}
1239
5c8802f1
IS		 1240static void
1241mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1242 struct mlxsw_sp_neigh_entry *neigh_entry,
1243 bool adding)
1244{
1245 if (!adding && !neigh_entry->connected)
1246 return;
1247 neigh_entry->connected = adding;
d5eb89cf 1248 if (neigh_entry->key.n->tbl == &arp_tbl) {
5c8802f1
IS		 1249 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1250 mlxsw_sp_rauht_op(adding));
d5eb89cf
AS		 1251 } else if (neigh_entry->key.n->tbl == &nd_tbl) {
1252 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry->key.n))
1253 return;
1254 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
1255 mlxsw_sp_rauht_op(adding));
1256 } else {
5c8802f1 1257 WARN_ON_ONCE(1);
d5eb89cf 1258 }
5c8802f1
IS		 1259}
1260
1261struct mlxsw_sp_neigh_event_work {
1262 struct work_struct work;
1263 struct mlxsw_sp *mlxsw_sp;
1264 struct neighbour *n;
1265};
1266
1267static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1268{
1269 struct mlxsw_sp_neigh_event_work *neigh_work =
1270 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1271 struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1272 struct mlxsw_sp_neigh_entry *neigh_entry;
1273 struct neighbour *n = neigh_work->n;
1274 unsigned char ha[ETH_ALEN];
a6bf9e93 1275 bool entry_connected;
93a87e5e 1276 u8 nud_state, dead;
a6bf9e93 1277
5c8802f1
IS		 1278 /* If these parameters are changed after we release the lock,
1279 * then we are guaranteed to receive another event letting us
1280 * know about it.
1281 */
a6bf9e93 1282 read_lock_bh(&n->lock);
5c8802f1 1283 memcpy(ha, n->ha, ETH_ALEN);
a6bf9e93 1284 nud_state = n->nud_state;
93a87e5e 1285 dead = n->dead;
a6bf9e93
YG		 1286 read_unlock_bh(&n->lock);
1287
5c8802f1 1288 rtnl_lock();
93a87e5e 1289 entry_connected = nud_state & NUD_VALID && !dead;
5c8802f1
IS		 1290 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1291 if (!entry_connected && !neigh_entry)
1292 goto out;
1293 if (!neigh_entry) {
1294 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1295 if (IS_ERR(neigh_entry))
1296 goto out;
a6bf9e93
YG		 1297 }
1298
5c8802f1
IS		 1299 memcpy(neigh_entry->ha, ha, ETH_ALEN);
1300 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1301 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1302
1303 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1304 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1305
1306out:
1307 rtnl_unlock();
a6bf9e93 1308 neigh_release(n);
5c8802f1 1309 kfree(neigh_work);
a6bf9e93
YG		 1310}
1311
e7322638
JP		 1312int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1313 unsigned long event, void *ptr)
c723c735 1314{
5c8802f1 1315 struct mlxsw_sp_neigh_event_work *neigh_work;
c723c735
YG		 1316 struct mlxsw_sp_port *mlxsw_sp_port;
1317 struct mlxsw_sp *mlxsw_sp;
1318 unsigned long interval;
1319 struct neigh_parms *p;
a6bf9e93 1320 struct neighbour *n;
c723c735
YG		 1321
1322 switch (event) {
1323 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1324 p = ptr;
1325
1326 /* We don't care about changes in the default table. */
a6c9b5d1 1327 if (!p->dev || (p->tbl != &arp_tbl && p->tbl != &nd_tbl))
c723c735
YG		 1328 return NOTIFY_DONE;
1329
1330 /* We are in atomic context and can't take RTNL mutex,
1331 * so use RCU variant to walk the device chain.
1332 */
1333 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1334 if (!mlxsw_sp_port)
1335 return NOTIFY_DONE;
1336
1337 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1338 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
9011b677 1339 mlxsw_sp->router->neighs_update.interval = interval;
c723c735
YG		 1340
1341 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1342 break;
a6bf9e93
YG		 1343 case NETEVENT_NEIGH_UPDATE:
1344 n = ptr;
a6bf9e93 1345
d5eb89cf 1346 if (n->tbl != &arp_tbl && n->tbl != &nd_tbl)
a6bf9e93
YG		 1347 return NOTIFY_DONE;
1348
5c8802f1 1349 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
a6bf9e93
YG		 1350 if (!mlxsw_sp_port)
1351 return NOTIFY_DONE;
1352
5c8802f1
IS		 1353 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1354 if (!neigh_work) {
a6bf9e93 1355 mlxsw_sp_port_dev_put(mlxsw_sp_port);
5c8802f1 1356 return NOTIFY_BAD;
a6bf9e93 1357 }
5c8802f1
IS		 1358
1359 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1360 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1361 neigh_work->n = n;
a6bf9e93
YG		 1362
1363 /* Take a reference to ensure the neighbour won't be
1364 * destructed until we drop the reference in delayed
1365 * work.
1366 */
1367 neigh_clone(n);
5c8802f1
IS		 1368 mlxsw_core_schedule_work(&neigh_work->work);
1369 mlxsw_sp_port_dev_put(mlxsw_sp_port);
a6bf9e93 1370 break;
c723c735
YG		 1371 }
1372
1373 return NOTIFY_DONE;
1374}
1375
6cf3c971
JP		 1376static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1377{
c723c735
YG		 1378 int err;
1379
9011b677 1380 err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
c723c735
YG		 1381 &mlxsw_sp_neigh_ht_params);
1382 if (err)
1383 return err;
1384
1385 /* Initialize the polling interval according to the default
1386 * table.
1387 */
1388 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1389
0b2361d9 1390 /* Create the delayed works for the activity_update */
9011b677 1391 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
c723c735 1392 mlxsw_sp_router_neighs_update_work);
9011b677 1393 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
0b2361d9 1394 mlxsw_sp_router_probe_unresolved_nexthops);
9011b677
IS		 1395 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1396 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
c723c735 1397 return 0;
6cf3c971
JP		 1398}
1399
1400static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1401{
9011b677
IS		 1402 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1403 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1404 rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
6cf3c971
JP		 1405}
1406
9665b745 1407static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
bf95233e 1408 const struct mlxsw_sp_rif *rif)
9665b745
IS		 1409{
1410 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1411
1412 mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
bf95233e 1413 rif->rif_index, rif->addr);
9665b745
IS		 1414 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1415}
1416
1417static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
bf95233e 1418 struct mlxsw_sp_rif *rif)
9665b745
IS		 1419{
1420 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1421
bf95233e
AS		 1422 mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif);
1423 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
9665b745
IS		 1424 rif_list_node)
1425 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1426}
1427
c53b8e1b
IS		 1428struct mlxsw_sp_nexthop_key {
1429 struct fib_nh *fib_nh;
1430};
1431
a7ff87ac
JP		 1432struct mlxsw_sp_nexthop {
1433 struct list_head neigh_list_node; /* member of neigh entry list */
9665b745 1434 struct list_head rif_list_node;
a7ff87ac
JP		 1435 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1436 * this belongs to
1437 */
c53b8e1b
IS		 1438 struct rhash_head ht_node;
1439 struct mlxsw_sp_nexthop_key key;
58adf2c4 1440 unsigned char gw_addr[sizeof(struct in6_addr)];
bf95233e 1441 struct mlxsw_sp_rif *rif;
a7ff87ac
JP		 1442 u8 should_offload:1, /* set indicates this neigh is connected and
1443 * should be put to KVD linear area of this group.
1444 */
1445 offloaded:1, /* set in case the neigh is actually put into
1446 * KVD linear area of this group.
1447 */
1448 update:1; /* set indicates that MAC of this neigh should be
1449 * updated in HW
1450 */
1451 struct mlxsw_sp_neigh_entry *neigh_entry;
1452};
1453
e9ad5e7d
IS		 1454struct mlxsw_sp_nexthop_group_key {
1455 struct fib_info *fi;
1456};
1457
a7ff87ac 1458struct mlxsw_sp_nexthop_group {
e9ad5e7d 1459 struct rhash_head ht_node;
a7ff87ac 1460 struct list_head fib_list; /* list of fib entries that use this group */
58adf2c4 1461 struct neigh_table *neigh_tbl;
e9ad5e7d 1462 struct mlxsw_sp_nexthop_group_key key;
b3e8d1eb
IS		 1463 u8 adj_index_valid:1,
1464 gateway:1; /* routes using the group use a gateway */
a7ff87ac
JP		 1465 u32 adj_index;
1466 u16 ecmp_size;
1467 u16 count;
1468 struct mlxsw_sp_nexthop nexthops[0];
bf95233e 1469#define nh_rif nexthops[0].rif
a7ff87ac
JP		 1470};
1471
e9ad5e7d
IS		 1472static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1473 .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1474 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1475 .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1476};
1477
1478static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1479 struct mlxsw_sp_nexthop_group *nh_grp)
1480{
9011b677 1481 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
e9ad5e7d
IS		 1482 &nh_grp->ht_node,
1483 mlxsw_sp_nexthop_group_ht_params);
1484}
1485
1486static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1487 struct mlxsw_sp_nexthop_group *nh_grp)
1488{
9011b677 1489 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
e9ad5e7d
IS		 1490 &nh_grp->ht_node,
1491 mlxsw_sp_nexthop_group_ht_params);
1492}
1493
1494static struct mlxsw_sp_nexthop_group *
1495mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1496 struct mlxsw_sp_nexthop_group_key key)
1497{
9011b677 1498 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, &key,
e9ad5e7d
IS		 1499 mlxsw_sp_nexthop_group_ht_params);
1500}
1501
c53b8e1b
IS		 1502static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1503 .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1504 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1505 .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1506};
1507
1508static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1509 struct mlxsw_sp_nexthop *nh)
1510{
9011b677 1511 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
c53b8e1b
IS		 1512 &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1513}
1514
1515static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1516 struct mlxsw_sp_nexthop *nh)
1517{
9011b677 1518 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
c53b8e1b
IS		 1519 mlxsw_sp_nexthop_ht_params);
1520}
1521
ad178c8e
IS		 1522static struct mlxsw_sp_nexthop *
1523mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1524 struct mlxsw_sp_nexthop_key key)
1525{
9011b677 1526 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
ad178c8e
IS		 1527 mlxsw_sp_nexthop_ht_params);
1528}
1529
a7ff87ac 1530static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
76610ebb 1531 const struct mlxsw_sp_fib *fib,
a7ff87ac
JP		 1532 u32 adj_index, u16 ecmp_size,
1533 u32 new_adj_index,
1534 u16 new_ecmp_size)
1535{
1536 char raleu_pl[MLXSW_REG_RALEU_LEN];
1537
1a9234e6 1538 mlxsw_reg_raleu_pack(raleu_pl,
76610ebb
IS		 1539 (enum mlxsw_reg_ralxx_protocol) fib->proto,
1540 fib->vr->id, adj_index, ecmp_size, new_adj_index,
1a9234e6 1541 new_ecmp_size);
a7ff87ac
JP		 1542 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1543}
1544
1545static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1546 struct mlxsw_sp_nexthop_group *nh_grp,
1547 u32 old_adj_index, u16 old_ecmp_size)
1548{
1549 struct mlxsw_sp_fib_entry *fib_entry;
76610ebb 1550 struct mlxsw_sp_fib *fib = NULL;
a7ff87ac
JP		 1551 int err;
1552
1553 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
76610ebb 1554 if (fib == fib_entry->fib_node->fib)
a7ff87ac 1555 continue;
76610ebb
IS		 1556 fib = fib_entry->fib_node->fib;
1557 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
a7ff87ac
JP		 1558 old_adj_index,
1559 old_ecmp_size,
1560 nh_grp->adj_index,
1561 nh_grp->ecmp_size);
1562 if (err)
1563 return err;
1564 }
1565 return 0;
1566}
1567
1568static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1569 struct mlxsw_sp_nexthop *nh)
1570{
1571 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1572 char ratr_pl[MLXSW_REG_RATR_LEN];
1573
1574 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1575 true, adj_index, neigh_entry->rif);
1576 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1577 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1578}
1579
1580static int
1581mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
a59b7e02
IS		 1582 struct mlxsw_sp_nexthop_group *nh_grp,
1583 bool reallocate)
a7ff87ac
JP		 1584{
1585 u32 adj_index = nh_grp->adj_index; /* base */
1586 struct mlxsw_sp_nexthop *nh;
1587 int i;
1588 int err;
1589
1590 for (i = 0; i < nh_grp->count; i++) {
1591 nh = &nh_grp->nexthops[i];
1592
1593 if (!nh->should_offload) {
1594 nh->offloaded = 0;
1595 continue;
1596 }
1597
a59b7e02 1598 if (nh->update || reallocate) {
a7ff87ac
JP		 1599 err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1600 adj_index, nh);
1601 if (err)
1602 return err;
1603 nh->update = 0;
1604 nh->offloaded = 1;
1605 }
1606 adj_index++;
1607 }
1608 return 0;
1609}
1610
1611static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1612 struct mlxsw_sp_fib_entry *fib_entry);
1613
1614static int
1615mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1616 struct mlxsw_sp_nexthop_group *nh_grp)
1617{
1618 struct mlxsw_sp_fib_entry *fib_entry;
1619 int err;
1620
1621 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1622 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1623 if (err)
1624 return err;
1625 }
1626 return 0;
1627}
1628
1629static void
1630mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1631 struct mlxsw_sp_nexthop_group *nh_grp)
1632{
1633 struct mlxsw_sp_nexthop *nh;
1634 bool offload_change = false;
1635 u32 adj_index;
1636 u16 ecmp_size = 0;
1637 bool old_adj_index_valid;
1638 u32 old_adj_index;
1639 u16 old_ecmp_size;
a7ff87ac
JP		 1640 int i;
1641 int err;
1642
b3e8d1eb
IS		 1643 if (!nh_grp->gateway) {
1644 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1645 return;
1646 }
1647
a7ff87ac
JP		 1648 for (i = 0; i < nh_grp->count; i++) {
1649 nh = &nh_grp->nexthops[i];
1650
1651 if (nh->should_offload ^ nh->offloaded) {
1652 offload_change = true;
1653 if (nh->should_offload)
1654 nh->update = 1;
1655 }
1656 if (nh->should_offload)
1657 ecmp_size++;
1658 }
1659 if (!offload_change) {
1660 /* Nothing was added or removed, so no need to reallocate. Just
1661 * update MAC on existing adjacency indexes.
1662 */
a59b7e02
IS		 1663 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1664 false);
a7ff87ac
JP		 1665 if (err) {
1666 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1667 goto set_trap;
1668 }
1669 return;
1670 }
1671 if (!ecmp_size)
1672 /* No neigh of this group is connected so we just set
1673 * the trap and let everthing flow through kernel.
1674 */
1675 goto set_trap;
1676
13124443
AS		 1677 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
1678 if (err) {
a7ff87ac
JP		 1679 /* We ran out of KVD linear space, just set the
1680 * trap and let everything flow through kernel.
1681 */
1682 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1683 goto set_trap;
1684 }
a7ff87ac
JP		 1685 old_adj_index_valid = nh_grp->adj_index_valid;
1686 old_adj_index = nh_grp->adj_index;
1687 old_ecmp_size = nh_grp->ecmp_size;
1688 nh_grp->adj_index_valid = 1;
1689 nh_grp->adj_index = adj_index;
1690 nh_grp->ecmp_size = ecmp_size;
a59b7e02 1691 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
a7ff87ac
JP		 1692 if (err) {
1693 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1694 goto set_trap;
1695 }
1696
1697 if (!old_adj_index_valid) {
1698 /* The trap was set for fib entries, so we have to call
1699 * fib entry update to unset it and use adjacency index.
1700 */
1701 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1702 if (err) {
1703 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1704 goto set_trap;
1705 }
1706 return;
1707 }
1708
1709 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1710 old_adj_index, old_ecmp_size);
1711 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1712 if (err) {
1713 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1714 goto set_trap;
1715 }
1716 return;
1717
1718set_trap:
1719 old_adj_index_valid = nh_grp->adj_index_valid;
1720 nh_grp->adj_index_valid = 0;
1721 for (i = 0; i < nh_grp->count; i++) {
1722 nh = &nh_grp->nexthops[i];
1723 nh->offloaded = 0;
1724 }
1725 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1726 if (err)
1727 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1728 if (old_adj_index_valid)
1729 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1730}
1731
1732static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1733 bool removing)
1734{
1735 if (!removing && !nh->should_offload)
1736 nh->should_offload = 1;
1737 else if (removing && nh->offloaded)
1738 nh->should_offload = 0;
1739 nh->update = 1;
1740}
1741
1742static void
1743mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1744 struct mlxsw_sp_neigh_entry *neigh_entry,
1745 bool removing)
1746{
1747 struct mlxsw_sp_nexthop *nh;
1748
a7ff87ac
JP		 1749 list_for_each_entry(nh, &neigh_entry->nexthop_list,
1750 neigh_list_node) {
1751 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1752 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1753 }
a7ff87ac
JP		 1754}
1755
9665b745 1756static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
bf95233e 1757 struct mlxsw_sp_rif *rif)
9665b745 1758{
bf95233e 1759 if (nh->rif)
9665b745
IS		 1760 return;
1761
bf95233e
AS		 1762 nh->rif = rif;
1763 list_add(&nh->rif_list_node, &rif->nexthop_list);
9665b745
IS		 1764}
1765
1766static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1767{
bf95233e 1768 if (!nh->rif)
9665b745
IS		 1769 return;
1770
1771 list_del(&nh->rif_list_node);
bf95233e 1772 nh->rif = NULL;
9665b745
IS		 1773}
1774
a8c97014
IS		 1775static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1776 struct mlxsw_sp_nexthop *nh)
a7ff87ac
JP		 1777{
1778 struct mlxsw_sp_neigh_entry *neigh_entry;
a7ff87ac 1779 struct neighbour *n;
93a87e5e 1780 u8 nud_state, dead;
c53b8e1b
IS		 1781 int err;
1782
ad178c8e 1783 if (!nh->nh_grp->gateway || nh->neigh_entry)
b8399a1e
IS		 1784 return 0;
1785
33b1341c
JP		 1786 /* Take a reference of neigh here ensuring that neigh would
1787 * not be detructed before the nexthop entry is finished.
1788 * The reference is taken either in neigh_lookup() or
fd76d910 1789 * in neigh_create() in case n is not found.
33b1341c 1790 */
58adf2c4 1791 n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
33b1341c 1792 if (!n) {
58adf2c4
IS		 1793 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
1794 nh->rif->dev);
a8c97014
IS		 1795 if (IS_ERR(n))
1796 return PTR_ERR(n);
a7ff87ac 1797 neigh_event_send(n, NULL);
33b1341c
JP		 1798 }
1799 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1800 if (!neigh_entry) {
5c8802f1
IS		 1801 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1802 if (IS_ERR(neigh_entry)) {
c53b8e1b
IS		 1803 err = -EINVAL;
1804 goto err_neigh_entry_create;
5c8802f1 1805 }
a7ff87ac 1806 }
b2157149
YG		 1807
1808 /* If that is the first nexthop connected to that neigh, add to
1809 * nexthop_neighs_list
1810 */
1811 if (list_empty(&neigh_entry->nexthop_list))
1812 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
9011b677 1813 &mlxsw_sp->router->nexthop_neighs_list);
b2157149 1814
a7ff87ac
JP		 1815 nh->neigh_entry = neigh_entry;
1816 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1817 read_lock_bh(&n->lock);
1818 nud_state = n->nud_state;
93a87e5e 1819 dead = n->dead;
a7ff87ac 1820 read_unlock_bh(&n->lock);
93a87e5e 1821 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
a7ff87ac
JP		 1822
1823 return 0;
c53b8e1b
IS		 1824
1825err_neigh_entry_create:
1826 neigh_release(n);
c53b8e1b 1827 return err;
a7ff87ac
JP		 1828}
1829
a8c97014
IS		 1830static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1831 struct mlxsw_sp_nexthop *nh)
a7ff87ac
JP		 1832{
1833 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
a8c97014 1834 struct neighbour *n;
a7ff87ac 1835
b8399a1e 1836 if (!neigh_entry)
a8c97014
IS		 1837 return;
1838 n = neigh_entry->key.n;
b8399a1e 1839
58312125 1840 __mlxsw_sp_nexthop_neigh_update(nh, true);
a7ff87ac 1841 list_del(&nh->neigh_list_node);
e58be79e 1842 nh->neigh_entry = NULL;
b2157149
YG		 1843
1844 /* If that is the last nexthop connected to that neigh, remove from
1845 * nexthop_neighs_list
1846 */
e58be79e
IS		 1847 if (list_empty(&neigh_entry->nexthop_list))
1848 list_del(&neigh_entry->nexthop_neighs_list_node);
b2157149 1849
5c8802f1
IS		 1850 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1851 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1852
1853 neigh_release(n);
a8c97014 1854}
c53b8e1b 1855
a8c97014
IS		 1856static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1857 struct mlxsw_sp_nexthop_group *nh_grp,
1858 struct mlxsw_sp_nexthop *nh,
1859 struct fib_nh *fib_nh)
1860{
1861 struct net_device *dev = fib_nh->nh_dev;
df6dd79b 1862 struct in_device *in_dev;
bf95233e 1863 struct mlxsw_sp_rif *rif;
a8c97014
IS		 1864 int err;
1865
1866 nh->nh_grp = nh_grp;
1867 nh->key.fib_nh = fib_nh;
58adf2c4 1868 memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
a8c97014
IS		 1869 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1870 if (err)
1871 return err;
1872
97989ee0
IS		 1873 if (!dev)
1874 return 0;
1875
df6dd79b
IS		 1876 in_dev = __in_dev_get_rtnl(dev);
1877 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1878 fib_nh->nh_flags & RTNH_F_LINKDOWN)
1879 return 0;
1880
bf95233e
AS		 1881 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1882 if (!rif)
a8c97014 1883 return 0;
bf95233e 1884 mlxsw_sp_nexthop_rif_init(nh, rif);
a8c97014
IS		 1885
1886 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1887 if (err)
1888 goto err_nexthop_neigh_init;
1889
1890 return 0;
1891
1892err_nexthop_neigh_init:
a4e75b76 1893 mlxsw_sp_nexthop_rif_fini(nh);
a8c97014
IS		 1894 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1895 return err;
1896}
1897
1898static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1899 struct mlxsw_sp_nexthop *nh)
1900{
1901 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
9665b745 1902 mlxsw_sp_nexthop_rif_fini(nh);
c53b8e1b 1903 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
a7ff87ac
JP		 1904}
1905
ad178c8e
IS		 1906static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1907 unsigned long event, struct fib_nh *fib_nh)
1908{
1909 struct mlxsw_sp_nexthop_key key;
1910 struct mlxsw_sp_nexthop *nh;
bf95233e 1911 struct mlxsw_sp_rif *rif;
ad178c8e 1912
9011b677 1913 if (mlxsw_sp->router->aborted)
ad178c8e
IS		 1914 return;
1915
1916 key.fib_nh = fib_nh;
1917 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1918 if (WARN_ON_ONCE(!nh))
1919 return;
1920
bf95233e
AS		 1921 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1922 if (!rif)
ad178c8e
IS		 1923 return;
1924
1925 switch (event) {
1926 case FIB_EVENT_NH_ADD:
bf95233e 1927 mlxsw_sp_nexthop_rif_init(nh, rif);
ad178c8e
IS		 1928 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1929 break;
1930 case FIB_EVENT_NH_DEL:
1931 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
9665b745 1932 mlxsw_sp_nexthop_rif_fini(nh);
ad178c8e
IS		 1933 break;
1934 }
1935
1936 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1937}
1938
9665b745 1939static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
bf95233e 1940 struct mlxsw_sp_rif *rif)
9665b745
IS		 1941{
1942 struct mlxsw_sp_nexthop *nh, *tmp;
1943
bf95233e 1944 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
9665b745
IS		 1945 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1946 mlxsw_sp_nexthop_rif_fini(nh);
1947 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1948 }
1949}
1950
a7ff87ac
JP		 1951static struct mlxsw_sp_nexthop_group *
1952mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1953{
1954 struct mlxsw_sp_nexthop_group *nh_grp;
1955 struct mlxsw_sp_nexthop *nh;
1956 struct fib_nh *fib_nh;
1957 size_t alloc_size;
1958 int i;
1959 int err;
1960
1961 alloc_size = sizeof(*nh_grp) +
1962 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1963 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1964 if (!nh_grp)
1965 return ERR_PTR(-ENOMEM);
1966 INIT_LIST_HEAD(&nh_grp->fib_list);
58adf2c4
IS		 1967 nh_grp->neigh_tbl = &arp_tbl;
1968
b3e8d1eb 1969 nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
a7ff87ac 1970 nh_grp->count = fi->fib_nhs;
e9ad5e7d 1971 nh_grp->key.fi = fi;
7387dbbc 1972 fib_info_hold(fi);
a7ff87ac
JP		 1973 for (i = 0; i < nh_grp->count; i++) {
1974 nh = &nh_grp->nexthops[i];
1975 fib_nh = &fi->fib_nh[i];
1976 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1977 if (err)
1978 goto err_nexthop_init;
1979 }
e9ad5e7d
IS		 1980 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1981 if (err)
1982 goto err_nexthop_group_insert;
a7ff87ac
JP		 1983 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1984 return nh_grp;
1985
e9ad5e7d 1986err_nexthop_group_insert:
a7ff87ac 1987err_nexthop_init:
df6dd79b
IS		 1988 for (i--; i >= 0; i--) {
1989 nh = &nh_grp->nexthops[i];
a7ff87ac 1990 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
df6dd79b 1991 }
7387dbbc 1992 fib_info_put(nh_grp->key.fi);
a7ff87ac
JP		 1993 kfree(nh_grp);
1994 return ERR_PTR(err);
1995}
1996
1997static void
1998mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1999 struct mlxsw_sp_nexthop_group *nh_grp)
2000{
2001 struct mlxsw_sp_nexthop *nh;
2002 int i;
2003
e9ad5e7d 2004 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
a7ff87ac
JP		 2005 for (i = 0; i < nh_grp->count; i++) {
2006 nh = &nh_grp->nexthops[i];
2007 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
2008 }
58312125
IS		 2009 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2010 WARN_ON_ONCE(nh_grp->adj_index_valid);
7387dbbc 2011 fib_info_put(nh_grp->key.fi);
a7ff87ac
JP		 2012 kfree(nh_grp);
2013}
2014
a7ff87ac
JP		 2015static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
2016 struct mlxsw_sp_fib_entry *fib_entry,
2017 struct fib_info *fi)
2018{
e9ad5e7d 2019 struct mlxsw_sp_nexthop_group_key key;
a7ff87ac
JP		 2020 struct mlxsw_sp_nexthop_group *nh_grp;
2021
e9ad5e7d
IS		 2022 key.fi = fi;
2023 nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
a7ff87ac
JP		 2024 if (!nh_grp) {
2025 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
2026 if (IS_ERR(nh_grp))
2027 return PTR_ERR(nh_grp);
2028 }
2029 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
2030 fib_entry->nh_group = nh_grp;
2031 return 0;
2032}
2033
2034static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
2035 struct mlxsw_sp_fib_entry *fib_entry)
2036{
2037 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2038
2039 list_del(&fib_entry->nexthop_group_node);
2040 if (!list_empty(&nh_grp->fib_list))
2041 return;
2042 mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
2043}
2044
013b20f9
IS		 2045static bool
2046mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2047{
2048 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
2049
9aecce1c
IS		 2050 if (fib_entry->params.tos)
2051 return false;
2052
013b20f9
IS		 2053 switch (fib_entry->type) {
2054 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2055 return !!nh_group->adj_index_valid;
2056 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
70ad3506 2057 return !!nh_group->nh_rif;
013b20f9
IS		 2058 default:
2059 return false;
2060 }
2061}
2062
2063static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2064{
2065 fib_entry->offloaded = true;
2066
76610ebb 2067 switch (fib_entry->fib_node->fib->proto) {
013b20f9
IS		 2068 case MLXSW_SP_L3_PROTO_IPV4:
2069 fib_info_offload_inc(fib_entry->nh_group->key.fi);
2070 break;
2071 case MLXSW_SP_L3_PROTO_IPV6:
2072 WARN_ON_ONCE(1);
2073 }
2074}
2075
2076static void
2077mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2078{
76610ebb 2079 switch (fib_entry->fib_node->fib->proto) {
013b20f9
IS		 2080 case MLXSW_SP_L3_PROTO_IPV4:
2081 fib_info_offload_dec(fib_entry->nh_group->key.fi);
2082 break;
2083 case MLXSW_SP_L3_PROTO_IPV6:
2084 WARN_ON_ONCE(1);
2085 }
2086
2087 fib_entry->offloaded = false;
2088}
2089
2090static void
2091mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2092 enum mlxsw_reg_ralue_op op, int err)
2093{
2094 switch (op) {
2095 case MLXSW_REG_RALUE_OP_WRITE_DELETE:
2096 if (!fib_entry->offloaded)
2097 return;
2098 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
2099 case MLXSW_REG_RALUE_OP_WRITE_WRITE:
2100 if (err)
2101 return;
2102 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
2103 !fib_entry->offloaded)
2104 mlxsw_sp_fib_entry_offload_set(fib_entry);
2105 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
2106 fib_entry->offloaded)
2107 mlxsw_sp_fib_entry_offload_unset(fib_entry);
2108 return;
2109 default:
2110 return;
2111 }
2112}
2113
a7ff87ac
JP		 2114static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
2115 struct mlxsw_sp_fib_entry *fib_entry,
2116 enum mlxsw_reg_ralue_op op)
2117{
2118 char ralue_pl[MLXSW_REG_RALUE_LEN];
76610ebb 2119 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
9aecce1c 2120 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
a7ff87ac
JP		 2121 enum mlxsw_reg_ralue_trap_action trap_action;
2122 u16 trap_id = 0;
2123 u32 adjacency_index = 0;
2124 u16 ecmp_size = 0;
2125
2126 /* In case the nexthop group adjacency index is valid, use it
2127 * with provided ECMP size. Otherwise, setup trap and pass
2128 * traffic to kernel.
2129 */
4b411477 2130 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
a7ff87ac
JP		 2131 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2132 adjacency_index = fib_entry->nh_group->adj_index;
2133 ecmp_size = fib_entry->nh_group->ecmp_size;
2134 } else {
2135 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2136 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2137 }
2138
1a9234e6 2139 mlxsw_reg_ralue_pack4(ralue_pl,
76610ebb
IS		 2140 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2141 fib->vr->id, fib_entry->fib_node->key.prefix_len,
9aecce1c 2142 *p_dip);
a7ff87ac
JP		 2143 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
2144 adjacency_index, ecmp_size);
2145 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2146}
2147
61c503f9
JP		 2148static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
2149 struct mlxsw_sp_fib_entry *fib_entry,
2150 enum mlxsw_reg_ralue_op op)
2151{
bf95233e 2152 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
76610ebb 2153 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
70ad3506 2154 enum mlxsw_reg_ralue_trap_action trap_action;
61c503f9 2155 char ralue_pl[MLXSW_REG_RALUE_LEN];
9aecce1c 2156 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
70ad3506 2157 u16 trap_id = 0;
bf95233e 2158 u16 rif_index = 0;
70ad3506
IS		 2159
2160 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2161 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
bf95233e 2162 rif_index = rif->rif_index;
70ad3506
IS		 2163 } else {
2164 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2165 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2166 }
61c503f9 2167
1a9234e6 2168 mlxsw_reg_ralue_pack4(ralue_pl,
76610ebb
IS		 2169 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2170 fib->vr->id, fib_entry->fib_node->key.prefix_len,
9aecce1c 2171 *p_dip);
bf95233e
AS		 2172 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
2173 rif_index);
61c503f9
JP		 2174 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2175}
2176
2177static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
2178 struct mlxsw_sp_fib_entry *fib_entry,
2179 enum mlxsw_reg_ralue_op op)
2180{
76610ebb 2181 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
61c503f9 2182 char ralue_pl[MLXSW_REG_RALUE_LEN];
9aecce1c 2183 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
61c503f9 2184
1a9234e6 2185 mlxsw_reg_ralue_pack4(ralue_pl,
76610ebb
IS		 2186 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2187 fib->vr->id, fib_entry->fib_node->key.prefix_len,
9aecce1c 2188 *p_dip);
61c503f9
JP		 2189 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2190 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2191}
2192
2193static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
2194 struct mlxsw_sp_fib_entry *fib_entry,
2195 enum mlxsw_reg_ralue_op op)
2196{
2197 switch (fib_entry->type) {
2198 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
a7ff87ac 2199 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
61c503f9
JP		 2200 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2201 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
2202 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
2203 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
2204 }
2205 return -EINVAL;
2206}
2207
2208static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2209 struct mlxsw_sp_fib_entry *fib_entry,
2210 enum mlxsw_reg_ralue_op op)
2211{
013b20f9
IS		 2212 int err = -EINVAL;
2213
76610ebb 2214 switch (fib_entry->fib_node->fib->proto) {
61c503f9 2215 case MLXSW_SP_L3_PROTO_IPV4:
013b20f9
IS		 2216 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
2217 break;
61c503f9 2218 case MLXSW_SP_L3_PROTO_IPV6:
013b20f9 2219 return err;
61c503f9 2220 }
013b20f9
IS		 2221 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
2222 return err;
61c503f9
JP		 2223}
2224
2225static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
2226 struct mlxsw_sp_fib_entry *fib_entry)
2227{
7146da31
JP		 2228 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2229 MLXSW_REG_RALUE_OP_WRITE_WRITE);
61c503f9
JP		 2230}
2231
2232static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
2233 struct mlxsw_sp_fib_entry *fib_entry)
2234{
2235 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2236 MLXSW_REG_RALUE_OP_WRITE_DELETE);
2237}
2238
61c503f9 2239static int
013b20f9
IS		 2240mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
2241 const struct fib_entry_notifier_info *fen_info,
2242 struct mlxsw_sp_fib_entry *fib_entry)
61c503f9 2243{
b45f64d1 2244 struct fib_info *fi = fen_info->fi;
61c503f9 2245
97989ee0
IS		 2246 switch (fen_info->type) {
2247 case RTN_BROADCAST: /* fall through */
2248 case RTN_LOCAL:
61c503f9
JP		 2249 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2250 return 0;
97989ee0
IS		 2251 case RTN_UNREACHABLE: /* fall through */
2252 case RTN_BLACKHOLE: /* fall through */
2253 case RTN_PROHIBIT:
2254 /* Packets hitting these routes need to be trapped, but
2255 * can do so with a lower priority than packets directed
2256 * at the host, so use action type local instead of trap.
2257 */
61c503f9 2258 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
97989ee0
IS		 2259 return 0;
2260 case RTN_UNICAST:
2261 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
2262 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2263 else
2264 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
2265 return 0;
2266 default:
2267 return -EINVAL;
2268 }
a7ff87ac
JP		 2269}
2270
5b004412 2271static struct mlxsw_sp_fib_entry *
9aecce1c
IS		 2272mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
2273 struct mlxsw_sp_fib_node *fib_node,
2274 const struct fib_entry_notifier_info *fen_info)
61c503f9 2275{
61c503f9 2276 struct mlxsw_sp_fib_entry *fib_entry;
61c503f9
JP		 2277 int err;
2278
9aecce1c 2279 fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
61c503f9
JP		 2280 if (!fib_entry) {
2281 err = -ENOMEM;
9aecce1c 2282 goto err_fib_entry_alloc;
61c503f9 2283 }
61c503f9 2284
013b20f9 2285 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
61c503f9 2286 if (err)
013b20f9 2287 goto err_fib4_entry_type_set;
61c503f9 2288
9aecce1c 2289 err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
b8399a1e
IS		 2290 if (err)
2291 goto err_nexthop_group_get;
2292
9aecce1c
IS		 2293 fib_entry->params.prio = fen_info->fi->fib_priority;
2294 fib_entry->params.tb_id = fen_info->tb_id;
2295 fib_entry->params.type = fen_info->type;
2296 fib_entry->params.tos = fen_info->tos;
2297
2298 fib_entry->fib_node = fib_node;
2299
5b004412
JP		 2300 return fib_entry;
2301
b8399a1e 2302err_nexthop_group_get:
013b20f9 2303err_fib4_entry_type_set:
9aecce1c
IS		 2304 kfree(fib_entry);
2305err_fib_entry_alloc:
5b004412
JP		 2306 return ERR_PTR(err);
2307}
2308
9aecce1c
IS		 2309static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2310 struct mlxsw_sp_fib_entry *fib_entry)
2311{
2312 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
2313 kfree(fib_entry);
2314}
2315
2316static struct mlxsw_sp_fib_node *
160e22aa
IS		 2317mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2318 size_t addr_len, unsigned char prefix_len);
9aecce1c 2319
5b004412 2320static struct mlxsw_sp_fib_entry *
9aecce1c
IS		 2321mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
2322 const struct fib_entry_notifier_info *fen_info)
5b004412 2323{
9aecce1c
IS		 2324 struct mlxsw_sp_fib_entry *fib_entry;
2325 struct mlxsw_sp_fib_node *fib_node;
160e22aa
IS		 2326 struct mlxsw_sp_fib *fib;
2327 struct mlxsw_sp_vr *vr;
2328
2329 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
2330 if (!vr)
2331 return NULL;
2332 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
5b004412 2333
160e22aa
IS		 2334 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2335 sizeof(fen_info->dst),
2336 fen_info->dst_len);
2337 if (!fib_node)
9aecce1c
IS		 2338 return NULL;
2339
2340 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2341 if (fib_entry->params.tb_id == fen_info->tb_id &&
2342 fib_entry->params.tos == fen_info->tos &&
2343 fib_entry->params.type == fen_info->type &&
2344 fib_entry->nh_group->key.fi == fen_info->fi) {
2345 return fib_entry;
2346 }
2347 }
2348
2349 return NULL;
2350}
2351
2352static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
2353 .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
2354 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
2355 .key_len = sizeof(struct mlxsw_sp_fib_key),
2356 .automatic_shrinking = true,
2357};
2358
2359static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
2360 struct mlxsw_sp_fib_node *fib_node)
2361{
2362 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
2363 mlxsw_sp_fib_ht_params);
2364}
2365
2366static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
2367 struct mlxsw_sp_fib_node *fib_node)
2368{
2369 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
2370 mlxsw_sp_fib_ht_params);
2371}
2372
2373static struct mlxsw_sp_fib_node *
2374mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2375 size_t addr_len, unsigned char prefix_len)
2376{
2377 struct mlxsw_sp_fib_key key;
2378
2379 memset(&key, 0, sizeof(key));
2380 memcpy(key.addr, addr, addr_len);
2381 key.prefix_len = prefix_len;
2382 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2383}
2384
2385static struct mlxsw_sp_fib_node *
76610ebb 2386mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
9aecce1c
IS		 2387 size_t addr_len, unsigned char prefix_len)
2388{
2389 struct mlxsw_sp_fib_node *fib_node;
2390
2391 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2392 if (!fib_node)
5b004412
JP		 2393 return NULL;
2394
9aecce1c 2395 INIT_LIST_HEAD(&fib_node->entry_list);
76610ebb 2396 list_add(&fib_node->list, &fib->node_list);
9aecce1c
IS		 2397 memcpy(fib_node->key.addr, addr, addr_len);
2398 fib_node->key.prefix_len = prefix_len;
9aecce1c
IS		 2399
2400 return fib_node;
2401}
2402
2403static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2404{
9aecce1c
IS		 2405 list_del(&fib_node->list);
2406 WARN_ON(!list_empty(&fib_node->entry_list));
2407 kfree(fib_node);
2408}
2409
2410static bool
2411mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2412 const struct mlxsw_sp_fib_entry *fib_entry)
2413{
2414 return list_first_entry(&fib_node->entry_list,
2415 struct mlxsw_sp_fib_entry, list) == fib_entry;
2416}
2417
2418static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2419{
2420 unsigned char prefix_len = fib_node->key.prefix_len;
76610ebb 2421 struct mlxsw_sp_fib *fib = fib_node->fib;
9aecce1c
IS		 2422
2423 if (fib->prefix_ref_count[prefix_len]++ == 0)
2424 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2425}
2426
2427static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2428{
2429 unsigned char prefix_len = fib_node->key.prefix_len;
76610ebb 2430 struct mlxsw_sp_fib *fib = fib_node->fib;
9aecce1c
IS		 2431
2432 if (--fib->prefix_ref_count[prefix_len] == 0)
2433 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
5b004412
JP		 2434}
2435
76610ebb
IS		 2436static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2437 struct mlxsw_sp_fib_node *fib_node,
2438 struct mlxsw_sp_fib *fib)
2439{
2440 struct mlxsw_sp_prefix_usage req_prefix_usage;
2441 struct mlxsw_sp_lpm_tree *lpm_tree;
2442 int err;
2443
2444 err = mlxsw_sp_fib_node_insert(fib, fib_node);
2445 if (err)
2446 return err;
2447 fib_node->fib = fib;
2448
2449 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2450 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2451
2452 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2453 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2454 &req_prefix_usage);
2455 if (err)
2456 goto err_tree_check;
2457 } else {
2458 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2459 fib->proto);
2460 if (IS_ERR(lpm_tree))
2461 return PTR_ERR(lpm_tree);
2462 fib->lpm_tree = lpm_tree;
2463 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2464 if (err)
2465 goto err_tree_bind;
2466 }
2467
2468 mlxsw_sp_fib_node_prefix_inc(fib_node);
2469
2470 return 0;
2471
2472err_tree_bind:
2473 fib->lpm_tree = NULL;
2474 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2475err_tree_check:
2476 fib_node->fib = NULL;
2477 mlxsw_sp_fib_node_remove(fib, fib_node);
2478 return err;
2479}
2480
2481static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2482 struct mlxsw_sp_fib_node *fib_node)
2483{
2484 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2485 struct mlxsw_sp_fib *fib = fib_node->fib;
2486
2487 mlxsw_sp_fib_node_prefix_dec(fib_node);
2488
2489 if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2490 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2491 fib->lpm_tree = NULL;
2492 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2493 } else {
2494 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2495 }
2496
2497 fib_node->fib = NULL;
2498 mlxsw_sp_fib_node_remove(fib, fib_node);
2499}
2500
9aecce1c 2501static struct mlxsw_sp_fib_node *
731ea1ca
IS		 2502mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
2503 size_t addr_len, unsigned char prefix_len,
2504 enum mlxsw_sp_l3proto proto)
5b004412 2505{
9aecce1c 2506 struct mlxsw_sp_fib_node *fib_node;
76610ebb 2507 struct mlxsw_sp_fib *fib;
9aecce1c
IS		 2508 struct mlxsw_sp_vr *vr;
2509 int err;
2510
731ea1ca 2511 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
9aecce1c
IS		 2512 if (IS_ERR(vr))
2513 return ERR_CAST(vr);
731ea1ca 2514 fib = mlxsw_sp_vr_fib(vr, proto);
9aecce1c 2515
731ea1ca 2516 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
9aecce1c
IS		 2517 if (fib_node)
2518 return fib_node;
5b004412 2519
731ea1ca 2520 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
9aecce1c
IS		 2521 if (!fib_node) {
2522 err = -ENOMEM;
2523 goto err_fib_node_create;
5b004412 2524 }
9aecce1c 2525
76610ebb
IS		 2526 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2527 if (err)
2528 goto err_fib_node_init;
2529
9aecce1c
IS		 2530 return fib_node;
2531
76610ebb
IS		 2532err_fib_node_init:
2533 mlxsw_sp_fib_node_destroy(fib_node);
9aecce1c 2534err_fib_node_create:
76610ebb 2535 mlxsw_sp_vr_put(vr);
9aecce1c 2536 return ERR_PTR(err);
5b004412
JP		 2537}
2538
731ea1ca
IS		 2539static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
2540 struct mlxsw_sp_fib_node *fib_node)
5b004412 2541{
76610ebb 2542 struct mlxsw_sp_vr *vr = fib_node->fib->vr;
5b004412 2543
9aecce1c
IS		 2544 if (!list_empty(&fib_node->entry_list))
2545 return;
76610ebb 2546 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
9aecce1c 2547 mlxsw_sp_fib_node_destroy(fib_node);
76610ebb 2548 mlxsw_sp_vr_put(vr);
61c503f9
JP		 2549}
2550
9aecce1c
IS		 2551static struct mlxsw_sp_fib_entry *
2552mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2553 const struct mlxsw_sp_fib_entry_params *params)
61c503f9 2554{
61c503f9 2555 struct mlxsw_sp_fib_entry *fib_entry;
9aecce1c
IS		 2556
2557 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2558 if (fib_entry->params.tb_id > params->tb_id)
2559 continue;
2560 if (fib_entry->params.tb_id != params->tb_id)
2561 break;
2562 if (fib_entry->params.tos > params->tos)
2563 continue;
2564 if (fib_entry->params.prio >= params->prio ||
2565 fib_entry->params.tos < params->tos)
2566 return fib_entry;
2567 }
2568
2569 return NULL;
2570}
2571
4283bce5
IS		 2572static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2573 struct mlxsw_sp_fib_entry *new_entry)
2574{
2575 struct mlxsw_sp_fib_node *fib_node;
2576
2577 if (WARN_ON(!fib_entry))
2578 return -EINVAL;
2579
2580 fib_node = fib_entry->fib_node;
2581 list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2582 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2583 fib_entry->params.tos != new_entry->params.tos ||
2584 fib_entry->params.prio != new_entry->params.prio)
2585 break;
2586 }
2587
2588 list_add_tail(&new_entry->list, &fib_entry->list);
2589 return 0;
2590}
2591
9aecce1c
IS		 2592static int
2593mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
4283bce5 2594 struct mlxsw_sp_fib_entry *new_entry,
599cf8f9 2595 bool replace, bool append)
9aecce1c
IS		 2596{
2597 struct mlxsw_sp_fib_entry *fib_entry;
2598
2599 fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2600
4283bce5
IS		 2601 if (append)
2602 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
599cf8f9
IS		 2603 if (replace && WARN_ON(!fib_entry))
2604 return -EINVAL;
4283bce5 2605
599cf8f9
IS		 2606 /* Insert new entry before replaced one, so that we can later
2607 * remove the second.
2608 */
9aecce1c
IS		 2609 if (fib_entry) {
2610 list_add_tail(&new_entry->list, &fib_entry->list);
2611 } else {
2612 struct mlxsw_sp_fib_entry *last;
2613
2614 list_for_each_entry(last, &fib_node->entry_list, list) {
2615 if (new_entry->params.tb_id > last->params.tb_id)
2616 break;
2617 fib_entry = last;
2618 }
2619
2620 if (fib_entry)
2621 list_add(&new_entry->list, &fib_entry->list);
2622 else
2623 list_add(&new_entry->list, &fib_node->entry_list);
2624 }
2625
2626 return 0;
2627}
2628
2629static void
2630mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2631{
2632 list_del(&fib_entry->list);
2633}
2634
2635static int
2636mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2637 const struct mlxsw_sp_fib_node *fib_node,
2638 struct mlxsw_sp_fib_entry *fib_entry)
2639{
2640 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2641 return 0;
2642
2643 /* To prevent packet loss, overwrite the previously offloaded
2644 * entry.
2645 */
2646 if (!list_is_singular(&fib_node->entry_list)) {
2647 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2648 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2649
2650 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2651 }
2652
2653 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2654}
2655
2656static void
2657mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2658 const struct mlxsw_sp_fib_node *fib_node,
2659 struct mlxsw_sp_fib_entry *fib_entry)
2660{
2661 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2662 return;
2663
2664 /* Promote the next entry by overwriting the deleted entry */
2665 if (!list_is_singular(&fib_node->entry_list)) {
2666 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2667 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2668
2669 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2670 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2671 return;
2672 }
2673
2674 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2675}
2676
2677static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
4283bce5 2678 struct mlxsw_sp_fib_entry *fib_entry,
599cf8f9 2679 bool replace, bool append)
9aecce1c
IS		 2680{
2681 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2682 int err;
2683
599cf8f9
IS		 2684 err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2685 append);
9aecce1c
IS		 2686 if (err)
2687 return err;
2688
2689 err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2690 if (err)
2691 goto err_fib4_node_entry_add;
2692
9aecce1c
IS		 2693 return 0;
2694
2695err_fib4_node_entry_add:
2696 mlxsw_sp_fib4_node_list_remove(fib_entry);
2697 return err;
2698}
2699
2700static void
2701mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2702 struct mlxsw_sp_fib_entry *fib_entry)
2703{
2704 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2705
9aecce1c
IS		 2706 mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2707 mlxsw_sp_fib4_node_list_remove(fib_entry);
2708}
2709
599cf8f9
IS		 2710static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2711 struct mlxsw_sp_fib_entry *fib_entry,
2712 bool replace)
2713{
2714 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2715 struct mlxsw_sp_fib_entry *replaced;
2716
2717 if (!replace)
2718 return;
2719
2720 /* We inserted the new entry before replaced one */
2721 replaced = list_next_entry(fib_entry, list);
2722
2723 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2724 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
731ea1ca 2725 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
599cf8f9
IS		 2726}
2727
9aecce1c
IS		 2728static int
2729mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
4283bce5 2730 const struct fib_entry_notifier_info *fen_info,
599cf8f9 2731 bool replace, bool append)
9aecce1c
IS		 2732{
2733 struct mlxsw_sp_fib_entry *fib_entry;
2734 struct mlxsw_sp_fib_node *fib_node;
61c503f9
JP		 2735 int err;
2736
9011b677 2737 if (mlxsw_sp->router->aborted)
b45f64d1
JP		 2738 return 0;
2739
731ea1ca
IS		 2740 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
2741 &fen_info->dst, sizeof(fen_info->dst),
2742 fen_info->dst_len,
2743 MLXSW_SP_L3_PROTO_IPV4);
9aecce1c
IS		 2744 if (IS_ERR(fib_node)) {
2745 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2746 return PTR_ERR(fib_node);
b45f64d1 2747 }
61c503f9 2748
9aecce1c
IS		 2749 fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2750 if (IS_ERR(fib_entry)) {
2751 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2752 err = PTR_ERR(fib_entry);
2753 goto err_fib4_entry_create;
2754 }
5b004412 2755
599cf8f9
IS		 2756 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2757 append);
b45f64d1 2758 if (err) {
9aecce1c
IS		 2759 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2760 goto err_fib4_node_entry_link;
b45f64d1 2761 }
9aecce1c 2762
599cf8f9
IS		 2763 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2764
61c503f9
JP		 2765 return 0;
2766
9aecce1c
IS		 2767err_fib4_node_entry_link:
2768 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2769err_fib4_entry_create:
731ea1ca 2770 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
61c503f9
JP		 2771 return err;
2772}
2773
37956d78
JP		 2774static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2775 struct fib_entry_notifier_info *fen_info)
61c503f9 2776{
61c503f9 2777 struct mlxsw_sp_fib_entry *fib_entry;
9aecce1c 2778 struct mlxsw_sp_fib_node *fib_node;
61c503f9 2779
9011b677 2780 if (mlxsw_sp->router->aborted)
37956d78 2781 return;
b45f64d1 2782
9aecce1c
IS		 2783 fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2784 if (WARN_ON(!fib_entry))
37956d78 2785 return;
9aecce1c 2786 fib_node = fib_entry->fib_node;
5b004412 2787
9aecce1c
IS		 2788 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2789 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
731ea1ca 2790 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
61c503f9 2791}
b45f64d1
JP		 2792
2793static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2794{
2795 char ralta_pl[MLXSW_REG_RALTA_LEN];
2796 char ralst_pl[MLXSW_REG_RALST_LEN];
b5d90e6d 2797 int i, err;
b45f64d1
JP		 2798
2799 mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2800 MLXSW_SP_LPM_TREE_MIN);
2801 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2802 if (err)
2803 return err;
2804
2805 mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2806 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2807 if (err)
2808 return err;
2809
b5d90e6d 2810 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
9011b677 2811 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
b5d90e6d
IS		 2812 char raltb_pl[MLXSW_REG_RALTB_LEN];
2813 char ralue_pl[MLXSW_REG_RALUE_LEN];
b45f64d1 2814
b5d90e6d
IS		 2815 if (!mlxsw_sp_vr_is_used(vr))
2816 continue;
2817
2818 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2819 MLXSW_REG_RALXX_PROTOCOL_IPV4,
2820 MLXSW_SP_LPM_TREE_MIN);
2821 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2822 raltb_pl);
2823 if (err)
2824 return err;
2825
2826 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2827 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2828 0);
2829 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2830 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2831 ralue_pl);
2832 if (err)
2833 return err;
2834 }
2835
2836 return 0;
b45f64d1
JP		 2837}
2838
9aecce1c
IS		 2839static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2840 struct mlxsw_sp_fib_node *fib_node)
2841{
2842 struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2843
2844 list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2845 bool do_break = &tmp->list == &fib_node->entry_list;
2846
2847 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2848 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
731ea1ca 2849 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
9aecce1c
IS		 2850 /* Break when entry list is empty and node was freed.
2851 * Otherwise, we'll access freed memory in the next
2852 * iteration.
2853 */
2854 if (do_break)
2855 break;
2856 }
2857}
2858
2859static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2860 struct mlxsw_sp_fib_node *fib_node)
2861{
76610ebb 2862 switch (fib_node->fib->proto) {
9aecce1c
IS		 2863 case MLXSW_SP_L3_PROTO_IPV4:
2864 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2865 break;
2866 case MLXSW_SP_L3_PROTO_IPV6:
2867 WARN_ON_ONCE(1);
2868 break;
2869 }
2870}
2871
76610ebb
IS		 2872static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2873 struct mlxsw_sp_vr *vr,
2874 enum mlxsw_sp_l3proto proto)
b45f64d1 2875{
76610ebb 2876 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
9aecce1c 2877 struct mlxsw_sp_fib_node *fib_node, *tmp;
76610ebb
IS		 2878
2879 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2880 bool do_break = &tmp->list == &fib->node_list;
2881
2882 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2883 if (do_break)
2884 break;
2885 }
2886}
2887
2888static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2889{
b45f64d1 2890 int i;
b45f64d1 2891
c1a38311 2892 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
9011b677 2893 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
ac571de9 2894
76610ebb 2895 if (!mlxsw_sp_vr_is_used(vr))
b45f64d1 2896 continue;
76610ebb 2897 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
b45f64d1 2898 }
ac571de9
IS		 2899}
2900
2901static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2902{
2903 int err;
2904
9011b677 2905 if (mlxsw_sp->router->aborted)
d331d303
IS		 2906 return;
2907 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
ac571de9 2908 mlxsw_sp_router_fib_flush(mlxsw_sp);
9011b677 2909 mlxsw_sp->router->aborted = true;
b45f64d1
JP		 2910 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2911 if (err)
2912 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2913}
2914
3057224e 2915struct mlxsw_sp_fib_event_work {
a0e4761d 2916 struct work_struct work;
ad178c8e
IS		 2917 union {
2918 struct fib_entry_notifier_info fen_info;
5d7bfd14 2919 struct fib_rule_notifier_info fr_info;
ad178c8e
IS		 2920 struct fib_nh_notifier_info fnh_info;
2921 };
3057224e
IS		 2922 struct mlxsw_sp *mlxsw_sp;
2923 unsigned long event;
2924};
2925
2926static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
b45f64d1 2927{
3057224e 2928 struct mlxsw_sp_fib_event_work *fib_work =
a0e4761d 2929 container_of(work, struct mlxsw_sp_fib_event_work, work);
3057224e 2930 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
5d7bfd14 2931 struct fib_rule *rule;
599cf8f9 2932 bool replace, append;
b45f64d1
JP		 2933 int err;
2934
3057224e
IS		 2935 /* Protect internal structures from changes */
2936 rtnl_lock();
2937 switch (fib_work->event) {
599cf8f9 2938 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4283bce5 2939 case FIB_EVENT_ENTRY_APPEND: /* fall through */
b45f64d1 2940 case FIB_EVENT_ENTRY_ADD:
599cf8f9 2941 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4283bce5
IS		 2942 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2943 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
599cf8f9 2944 replace, append);
b45f64d1
JP		 2945 if (err)
2946 mlxsw_sp_router_fib4_abort(mlxsw_sp);
3057224e 2947 fib_info_put(fib_work->fen_info.fi);
b45f64d1
JP		 2948 break;
2949 case FIB_EVENT_ENTRY_DEL:
3057224e
IS		 2950 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2951 fib_info_put(fib_work->fen_info.fi);
b45f64d1
JP		 2952 break;
2953 case FIB_EVENT_RULE_ADD: /* fall through */
2954 case FIB_EVENT_RULE_DEL:
5d7bfd14 2955 rule = fib_work->fr_info.rule;
c7f6e665 2956 if (!fib4_rule_default(rule) && !rule->l3mdev)
5d7bfd14
IS		 2957 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2958 fib_rule_put(rule);
b45f64d1 2959 break;
ad178c8e
IS		 2960 case FIB_EVENT_NH_ADD: /* fall through */
2961 case FIB_EVENT_NH_DEL:
2962 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2963 fib_work->fnh_info.fib_nh);
2964 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2965 break;
b45f64d1 2966 }
3057224e
IS		 2967 rtnl_unlock();
2968 kfree(fib_work);
2969}
2970
2971/* Called with rcu_read_lock() */
2972static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2973 unsigned long event, void *ptr)
2974{
3057224e
IS		 2975 struct mlxsw_sp_fib_event_work *fib_work;
2976 struct fib_notifier_info *info = ptr;
7e39d115 2977 struct mlxsw_sp_router *router;
3057224e
IS		 2978
2979 if (!net_eq(info->net, &init_net))
2980 return NOTIFY_DONE;
2981
2982 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2983 if (WARN_ON(!fib_work))
2984 return NOTIFY_BAD;
2985
a0e4761d 2986 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
7e39d115
IS		 2987 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
2988 fib_work->mlxsw_sp = router->mlxsw_sp;
3057224e
IS		 2989 fib_work->event = event;
2990
2991 switch (event) {
599cf8f9 2992 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4283bce5 2993 case FIB_EVENT_ENTRY_APPEND: /* fall through */
3057224e
IS		 2994 case FIB_EVENT_ENTRY_ADD: /* fall through */
2995 case FIB_EVENT_ENTRY_DEL:
2996 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2997 /* Take referece on fib_info to prevent it from being
2998 * freed while work is queued. Release it afterwards.
2999 */
3000 fib_info_hold(fib_work->fen_info.fi);
3001 break;
5d7bfd14
IS		 3002 case FIB_EVENT_RULE_ADD: /* fall through */
3003 case FIB_EVENT_RULE_DEL:
3004 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
3005 fib_rule_get(fib_work->fr_info.rule);
3006 break;
ad178c8e
IS		 3007 case FIB_EVENT_NH_ADD: /* fall through */
3008 case FIB_EVENT_NH_DEL:
3009 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
3010 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
3011 break;
3057224e
IS		 3012 }
3013
a0e4761d 3014 mlxsw_core_schedule_work(&fib_work->work);
3057224e 3015
b45f64d1
JP		 3016 return NOTIFY_DONE;
3017}
3018
4724ba56
IS		 3019static struct mlxsw_sp_rif *
3020mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
3021 const struct net_device *dev)
3022{
3023 int i;
3024
3025 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5f9efffb
IS		 3026 if (mlxsw_sp->router->rifs[i] &&
3027 mlxsw_sp->router->rifs[i]->dev == dev)
3028 return mlxsw_sp->router->rifs[i];
4724ba56
IS		 3029
3030 return NULL;
3031}
3032
3033static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
3034{
3035 char ritr_pl[MLXSW_REG_RITR_LEN];
3036 int err;
3037
3038 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
3039 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3040 if (WARN_ON_ONCE(err))
3041 return err;
3042
3043 mlxsw_reg_ritr_enable_set(ritr_pl, false);
3044 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3045}
3046
3047static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
bf95233e 3048 struct mlxsw_sp_rif *rif)
4724ba56 3049{
bf95233e
AS		 3050 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
3051 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
3052 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
4724ba56
IS		 3053}
3054
5ea1237f
AS		 3055static bool
3056mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
3057 unsigned long event)
4724ba56 3058{
5ea1237f
AS		 3059 struct inet6_dev *inet6_dev;
3060 bool addr_list_empty = true;
3061 struct in_device *idev;
3062
4724ba56
IS		 3063 switch (event) {
3064 case NETDEV_UP:
bf95233e 3065 if (!rif)
4724ba56
IS		 3066 return true;
3067 return false;
3068 case NETDEV_DOWN:
5ea1237f
AS		 3069 idev = __in_dev_get_rtnl(dev);
3070 if (idev && idev->ifa_list)
3071 addr_list_empty = false;
3072
3073 inet6_dev = __in6_dev_get(dev);
3074 if (addr_list_empty && inet6_dev &&
3075 !list_empty(&inet6_dev->addr_list))
3076 addr_list_empty = false;
3077
3078 if (rif && addr_list_empty &&
bf95233e 3079 !netif_is_l3_slave(rif->dev))
4724ba56
IS		 3080 return true;
3081 /* It is possible we already removed the RIF ourselves
3082 * if it was assigned to a netdev that is now a bridge
3083 * or LAG slave.
3084 */
3085 return false;
3086 }
3087
3088 return false;
3089}
3090
e4f3c1c1
IS		 3091static enum mlxsw_sp_rif_type
3092mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
3093 const struct net_device *dev)
3094{
3095 enum mlxsw_sp_fid_type type;
3096
3097 /* RIF type is derived from the type of the underlying FID */
3098 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
3099 type = MLXSW_SP_FID_TYPE_8021Q;
3100 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
3101 type = MLXSW_SP_FID_TYPE_8021Q;
3102 else if (netif_is_bridge_master(dev))
3103 type = MLXSW_SP_FID_TYPE_8021D;
3104 else
3105 type = MLXSW_SP_FID_TYPE_RFID;
3106
3107 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
3108}
3109
de5ed99e 3110static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
4724ba56
IS		 3111{
3112 int i;
3113
de5ed99e
IS		 3114 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
3115 if (!mlxsw_sp->router->rifs[i]) {
3116 *p_rif_index = i;
3117 return 0;
3118 }
3119 }
4724ba56 3120
de5ed99e 3121 return -ENOBUFS;
4724ba56
IS		 3122}
3123
e4f3c1c1
IS		 3124static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
3125 u16 vr_id,
3126 struct net_device *l3_dev)
4724ba56 3127{
bf95233e 3128 struct mlxsw_sp_rif *rif;
4724ba56 3129
e4f3c1c1 3130 rif = kzalloc(rif_size, GFP_KERNEL);
bf95233e 3131 if (!rif)
4724ba56
IS		 3132 return NULL;
3133
bf95233e
AS		 3134 INIT_LIST_HEAD(&rif->nexthop_list);
3135 INIT_LIST_HEAD(&rif->neigh_list);
3136 ether_addr_copy(rif->addr, l3_dev->dev_addr);
3137 rif->mtu = l3_dev->mtu;
3138 rif->vr_id = vr_id;
3139 rif->dev = l3_dev;
3140 rif->rif_index = rif_index;
4724ba56 3141
bf95233e 3142 return rif;
4724ba56
IS		 3143}
3144
5f9efffb
IS		 3145struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
3146 u16 rif_index)
3147{
3148 return mlxsw_sp->router->rifs[rif_index];
3149}
3150
fd1b9d41
AS		 3151u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
3152{
3153 return rif->rif_index;
3154}
3155
3156int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
3157{
3158 return rif->dev->ifindex;
3159}
3160
4724ba56 3161static struct mlxsw_sp_rif *
e4f3c1c1
IS		 3162mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
3163 const struct mlxsw_sp_rif_params *params)
4724ba56 3164{
e4f3c1c1
IS		 3165 u32 tb_id = l3mdev_fib_table(params->dev);
3166 const struct mlxsw_sp_rif_ops *ops;
3167 enum mlxsw_sp_rif_type type;
bf95233e 3168 struct mlxsw_sp_rif *rif;
a1107487
IS		 3169 struct mlxsw_sp_fid *fid;
3170 struct mlxsw_sp_vr *vr;
3171 u16 rif_index;
4724ba56
IS		 3172 int err;
3173
e4f3c1c1
IS		 3174 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
3175 ops = mlxsw_sp->router->rif_ops_arr[type];
3176
c9ec53f0
IS		 3177 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
3178 if (IS_ERR(vr))
3179 return ERR_CAST(vr);
3180
de5ed99e
IS		 3181 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
3182 if (err)
3183 goto err_rif_index_alloc;
4724ba56 3184
e4f3c1c1 3185 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
a13a594d
IS		 3186 if (!rif) {
3187 err = -ENOMEM;
3188 goto err_rif_alloc;
3189 }
e4f3c1c1
IS		 3190 rif->mlxsw_sp = mlxsw_sp;
3191 rif->ops = ops;
a13a594d 3192
e4f3c1c1
IS		 3193 fid = ops->fid_get(rif);
3194 if (IS_ERR(fid)) {
3195 err = PTR_ERR(fid);
3196 goto err_fid_get;
4d93ceeb 3197 }
e4f3c1c1 3198 rif->fid = fid;
4d93ceeb 3199
e4f3c1c1
IS		 3200 if (ops->setup)
3201 ops->setup(rif, params);
3202
3203 err = ops->configure(rif);
4724ba56 3204 if (err)
e4f3c1c1 3205 goto err_configure;
4724ba56 3206
e4f3c1c1 3207 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, params->dev->dev_addr,
a1107487 3208 mlxsw_sp_fid_index(fid), true);
4724ba56
IS		 3209 if (err)
3210 goto err_rif_fdb_op;
3211
e4f3c1c1 3212 mlxsw_sp_rif_counters_alloc(rif);
a1107487 3213 mlxsw_sp_fid_rif_set(fid, rif);
5f9efffb 3214 mlxsw_sp->router->rifs[rif_index] = rif;
6913229e 3215 vr->rif_count++;
4724ba56 3216
bf95233e 3217 return rif;
4724ba56 3218
4724ba56 3219err_rif_fdb_op:
e4f3c1c1
IS		 3220 ops->deconfigure(rif);
3221err_configure:
a1107487
IS		 3222 mlxsw_sp_fid_put(fid);
3223err_fid_get:
e4f3c1c1
IS		 3224 kfree(rif);
3225err_rif_alloc:
de5ed99e 3226err_rif_index_alloc:
c9ec53f0 3227 mlxsw_sp_vr_put(vr);
4724ba56
IS		 3228 return ERR_PTR(err);
3229}
3230
e4f3c1c1 3231void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
4724ba56 3232{
e4f3c1c1
IS		 3233 const struct mlxsw_sp_rif_ops *ops = rif->ops;
3234 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
a1107487 3235 struct mlxsw_sp_fid *fid = rif->fid;
e4f3c1c1 3236 struct mlxsw_sp_vr *vr;
4724ba56 3237
bf95233e 3238 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
e4f3c1c1 3239 vr = &mlxsw_sp->router->vrs[rif->vr_id];
e0c0afd8 3240
6913229e 3241 vr->rif_count--;
e4f3c1c1 3242 mlxsw_sp->router->rifs[rif->rif_index] = NULL;
a1107487 3243 mlxsw_sp_fid_rif_set(fid, NULL);
e4f3c1c1
IS		 3244 mlxsw_sp_rif_counters_free(rif);
3245 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->dev->dev_addr,
3246 mlxsw_sp_fid_index(fid), false);
3247 ops->deconfigure(rif);
a1107487 3248 mlxsw_sp_fid_put(fid);
e4f3c1c1 3249 kfree(rif);
c9ec53f0 3250 mlxsw_sp_vr_put(vr);
4724ba56
IS		 3251}
3252
e4f3c1c1
IS		 3253static void
3254mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
3255 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
3256{
3257 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3258
3259 params->vid = mlxsw_sp_port_vlan->vid;
3260 params->lag = mlxsw_sp_port->lagged;
3261 if (params->lag)
3262 params->lag_id = mlxsw_sp_port->lag_id;
3263 else
3264 params->system_port = mlxsw_sp_port->local_port;
3265}
3266
7cbecf24 3267static int
a1107487 3268mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
7cbecf24 3269 struct net_device *l3_dev)
4724ba56 3270{
7cbecf24 3271 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
1b8f09a0 3272 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
7cbecf24 3273 u16 vid = mlxsw_sp_port_vlan->vid;
bf95233e 3274 struct mlxsw_sp_rif *rif;
a1107487 3275 struct mlxsw_sp_fid *fid;
03ea01e9 3276 int err;
4724ba56 3277
1b8f09a0 3278 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
bf95233e 3279 if (!rif) {
e4f3c1c1
IS		 3280 struct mlxsw_sp_rif_params params = {
3281 .dev = l3_dev,
3282 };
3283
3284 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
3285 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
bf95233e
AS		 3286 if (IS_ERR(rif))
3287 return PTR_ERR(rif);
4724ba56
IS		 3288 }
3289
a1107487 3290 /* FID was already created, just take a reference */
e4f3c1c1 3291 fid = rif->ops->fid_get(rif);
a1107487
IS		 3292 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
3293 if (err)
3294 goto err_fid_port_vid_map;
3295
7cbecf24 3296 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
03ea01e9
IS		 3297 if (err)
3298 goto err_port_vid_learning_set;
3299
7cbecf24 3300 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
03ea01e9
IS		 3301 BR_STATE_FORWARDING);
3302 if (err)
3303 goto err_port_vid_stp_set;
3304
a1107487 3305 mlxsw_sp_port_vlan->fid = fid;
4724ba56 3306
4724ba56 3307 return 0;
03ea01e9
IS		 3308
3309err_port_vid_stp_set:
7cbecf24 3310 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
03ea01e9 3311err_port_vid_learning_set:
a1107487
IS		 3312 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3313err_fid_port_vid_map:
3314 mlxsw_sp_fid_put(fid);
03ea01e9 3315 return err;
4724ba56
IS		 3316}
3317
a1107487
IS		 3318void
3319mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
4724ba56 3320{
ce95e154 3321 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
7cbecf24 3322 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
ce95e154 3323 u16 vid = mlxsw_sp_port_vlan->vid;
ce95e154 3324
a1107487
IS		 3325 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
3326 return;
4aafc368 3327
a1107487 3328 mlxsw_sp_port_vlan->fid = NULL;
7cbecf24
IS		 3329 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
3330 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
a1107487
IS		 3331 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3332 /* If router port holds the last reference on the rFID, then the
3333 * associated Sub-port RIF will be destroyed.
3334 */
3335 mlxsw_sp_fid_put(fid);
4724ba56
IS		 3336}
3337
7cbecf24
IS		 3338static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
3339 struct net_device *port_dev,
3340 unsigned long event, u16 vid)
4724ba56
IS		 3341{
3342 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
ce95e154 3343 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
4724ba56 3344
ce95e154 3345 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
7cbecf24
IS		 3346 if (WARN_ON(!mlxsw_sp_port_vlan))
3347 return -EINVAL;
4724ba56
IS		 3348
3349 switch (event) {
3350 case NETDEV_UP:
a1107487 3351 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
7cbecf24 3352 l3_dev);
4724ba56 3353 case NETDEV_DOWN:
a1107487 3354 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
4724ba56
IS		 3355 break;
3356 }
3357
3358 return 0;
3359}
3360
3361static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
3362 unsigned long event)
3363{
2b94e58d
JP		 3364 if (netif_is_bridge_port(port_dev) ||
3365 netif_is_lag_port(port_dev) ||
3366 netif_is_ovs_port(port_dev))
4724ba56
IS		 3367 return 0;
3368
7cbecf24 3369 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
4724ba56
IS		 3370}
3371
3372static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
3373 struct net_device *lag_dev,
3374 unsigned long event, u16 vid)
3375{
3376 struct net_device *port_dev;
3377 struct list_head *iter;
3378 int err;
3379
3380 netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
3381 if (mlxsw_sp_port_dev_check(port_dev)) {
7cbecf24
IS		 3382 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
3383 port_dev,
3384 event, vid);
4724ba56
IS		 3385 if (err)
3386 return err;
3387 }
3388 }
3389
3390 return 0;
3391}
3392
3393static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
3394 unsigned long event)
3395{
3396 if (netif_is_bridge_port(lag_dev))
3397 return 0;
3398
3399 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
3400}
3401
4724ba56 3402static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
4724ba56
IS		 3403 unsigned long event)
3404{
3405 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
e4f3c1c1
IS		 3406 struct mlxsw_sp_rif_params params = {
3407 .dev = l3_dev,
3408 };
a1107487 3409 struct mlxsw_sp_rif *rif;
4724ba56
IS		 3410
3411 switch (event) {
3412 case NETDEV_UP:
e4f3c1c1
IS		 3413 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
3414 if (IS_ERR(rif))
3415 return PTR_ERR(rif);
3416 break;
4724ba56 3417 case NETDEV_DOWN:
a1107487 3418 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
e4f3c1c1 3419 mlxsw_sp_rif_destroy(rif);
4724ba56
IS		 3420 break;
3421 }
3422
3423 return 0;
3424}
3425
3426static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3427 unsigned long event)
3428{
3429 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
4724ba56
IS		 3430 u16 vid = vlan_dev_vlan_id(vlan_dev);
3431
6b27c8ad
IS		 3432 if (netif_is_bridge_port(vlan_dev))
3433 return 0;
3434
4724ba56 3435 if (mlxsw_sp_port_dev_check(real_dev))
7cbecf24
IS		 3436 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
3437 event, vid);
4724ba56
IS		 3438 else if (netif_is_lag_master(real_dev))
3439 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3440 vid);
c57529e1 3441 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
a1107487 3442 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
4724ba56
IS		 3443
3444 return 0;
3445}
3446
b1e45526
IS		 3447static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
3448 unsigned long event)
3449{
3450 if (mlxsw_sp_port_dev_check(dev))
3451 return mlxsw_sp_inetaddr_port_event(dev, event);
3452 else if (netif_is_lag_master(dev))
3453 return mlxsw_sp_inetaddr_lag_event(dev, event);
3454 else if (netif_is_bridge_master(dev))
a1107487 3455 return mlxsw_sp_inetaddr_bridge_event(dev, event);
b1e45526
IS		 3456 else if (is_vlan_dev(dev))
3457 return mlxsw_sp_inetaddr_vlan_event(dev, event);
3458 else
3459 return 0;
3460}
3461
4724ba56
IS		 3462int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3463 unsigned long event, void *ptr)
3464{
3465 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3466 struct net_device *dev = ifa->ifa_dev->dev;
3467 struct mlxsw_sp *mlxsw_sp;
bf95233e 3468 struct mlxsw_sp_rif *rif;
4724ba56
IS		 3469 int err = 0;
3470
3471 mlxsw_sp = mlxsw_sp_lower_get(dev);
3472 if (!mlxsw_sp)
3473 goto out;
3474
bf95233e 3475 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5ea1237f 3476 if (!mlxsw_sp_rif_should_config(rif, dev, event))
4724ba56
IS		 3477 goto out;
3478
b1e45526 3479 err = __mlxsw_sp_inetaddr_event(dev, event);
4724ba56
IS		 3480out:
3481 return notifier_from_errno(err);
3482}
3483
5ea1237f
AS		 3484struct mlxsw_sp_inet6addr_event_work {
3485 struct work_struct work;
3486 struct net_device *dev;
3487 unsigned long event;
3488};
3489
3490static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
3491{
3492 struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
3493 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
3494 struct net_device *dev = inet6addr_work->dev;
3495 unsigned long event = inet6addr_work->event;
3496 struct mlxsw_sp *mlxsw_sp;
3497 struct mlxsw_sp_rif *rif;
3498
3499 rtnl_lock();
3500 mlxsw_sp = mlxsw_sp_lower_get(dev);
3501 if (!mlxsw_sp)
3502 goto out;
3503
3504 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3505 if (!mlxsw_sp_rif_should_config(rif, dev, event))
3506 goto out;
3507
3508 __mlxsw_sp_inetaddr_event(dev, event);
3509out:
3510 rtnl_unlock();
3511 dev_put(dev);
3512 kfree(inet6addr_work);
3513}
3514
3515/* Called with rcu_read_lock() */
3516int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
3517 unsigned long event, void *ptr)
3518{
3519 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
3520 struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
3521 struct net_device *dev = if6->idev->dev;
3522
3523 if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
3524 return NOTIFY_DONE;
3525
3526 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
3527 if (!inet6addr_work)
3528 return NOTIFY_BAD;
3529
3530 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
3531 inet6addr_work->dev = dev;
3532 inet6addr_work->event = event;
3533 dev_hold(dev);
3534 mlxsw_core_schedule_work(&inet6addr_work->work);
3535
3536 return NOTIFY_DONE;
3537}
3538
bf95233e 3539static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
4724ba56
IS		 3540 const char *mac, int mtu)
3541{
3542 char ritr_pl[MLXSW_REG_RITR_LEN];
3543 int err;
3544
bf95233e 3545 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
4724ba56
IS		 3546 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3547 if (err)
3548 return err;
3549
3550 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3551 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3552 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3553 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3554}
3555
3556int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3557{
3558 struct mlxsw_sp *mlxsw_sp;
bf95233e 3559 struct mlxsw_sp_rif *rif;
a1107487 3560 u16 fid_index;
4724ba56
IS		 3561 int err;
3562
3563 mlxsw_sp = mlxsw_sp_lower_get(dev);
3564 if (!mlxsw_sp)
3565 return 0;
3566
bf95233e
AS		 3567 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3568 if (!rif)
4724ba56 3569 return 0;
a1107487 3570 fid_index = mlxsw_sp_fid_index(rif->fid);
4724ba56 3571
a1107487 3572 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
4724ba56
IS		 3573 if (err)
3574 return err;
3575
bf95233e
AS		 3576 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
3577 dev->mtu);
4724ba56
IS		 3578 if (err)
3579 goto err_rif_edit;
3580
a1107487 3581 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
4724ba56
IS		 3582 if (err)
3583 goto err_rif_fdb_op;
3584
bf95233e
AS		 3585 ether_addr_copy(rif->addr, dev->dev_addr);
3586 rif->mtu = dev->mtu;
4724ba56 3587
bf95233e 3588 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
4724ba56
IS		 3589
3590 return 0;
3591
3592err_rif_fdb_op:
bf95233e 3593 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
4724ba56 3594err_rif_edit:
a1107487 3595 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
4724ba56
IS		 3596 return err;
3597}
3598
b1e45526
IS		 3599static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
3600 struct net_device *l3_dev)
7179eb5a 3601{
b1e45526 3602 struct mlxsw_sp_rif *rif;
7179eb5a 3603
b1e45526
IS		 3604 /* If netdev is already associated with a RIF, then we need to
3605 * destroy it and create a new one with the new virtual router ID.
7179eb5a 3606 */
b1e45526
IS		 3607 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3608 if (rif)
3609 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
7179eb5a 3610
b1e45526 3611 return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
7179eb5a
IS		 3612}
3613
b1e45526
IS		 3614static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
3615 struct net_device *l3_dev)
7179eb5a 3616{
b1e45526 3617 struct mlxsw_sp_rif *rif;
7179eb5a 3618
b1e45526
IS		 3619 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3620 if (!rif)
7179eb5a 3621 return;
b1e45526 3622 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
7179eb5a
IS		 3623}
3624
b1e45526
IS		 3625int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
3626 struct netdev_notifier_changeupper_info *info)
3d70e458 3627{
b1e45526
IS		 3628 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3629 int err = 0;
3d70e458 3630
b1e45526
IS		 3631 if (!mlxsw_sp)
3632 return 0;
3d70e458 3633
b1e45526
IS		 3634 switch (event) {
3635 case NETDEV_PRECHANGEUPPER:
3636 return 0;
3637 case NETDEV_CHANGEUPPER:
3638 if (info->linking)
3639 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
3640 else
3641 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
3642 break;
3643 }
3d70e458 3644
b1e45526 3645 return err;
3d70e458
IS		 3646}
3647
e4f3c1c1
IS		 3648static struct mlxsw_sp_rif_subport *
3649mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
a1107487 3650{
e4f3c1c1
IS		 3651 return container_of(rif, struct mlxsw_sp_rif_subport, common);
3652}
3653
3654static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
3655 const struct mlxsw_sp_rif_params *params)
3656{
3657 struct mlxsw_sp_rif_subport *rif_subport;
3658
3659 rif_subport = mlxsw_sp_rif_subport_rif(rif);
3660 rif_subport->vid = params->vid;
3661 rif_subport->lag = params->lag;
3662 if (params->lag)
3663 rif_subport->lag_id = params->lag_id;
a1107487 3664 else
e4f3c1c1
IS		 3665 rif_subport->system_port = params->system_port;
3666}
3667
3668static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
3669{
3670 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3671 struct mlxsw_sp_rif_subport *rif_subport;
3672 char ritr_pl[MLXSW_REG_RITR_LEN];
3673
3674 rif_subport = mlxsw_sp_rif_subport_rif(rif);
3675 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
3676 rif->rif_index, rif->vr_id, rif->dev->mtu,
3677 rif->dev->dev_addr);
3678 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
3679 rif_subport->lag ? rif_subport->lag_id :
3680 rif_subport->system_port,
3681 rif_subport->vid);
3682
3683 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3684}
3685
3686static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
3687{
3688 return mlxsw_sp_rif_subport_op(rif, true);
a1107487
IS		 3689}
3690
e4f3c1c1
IS		 3691static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
3692{
3693 mlxsw_sp_rif_subport_op(rif, false);
3694}
3695
3696static struct mlxsw_sp_fid *
3697mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
3698{
3699 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
3700}
3701
3702static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
3703 .type = MLXSW_SP_RIF_TYPE_SUBPORT,
3704 .rif_size = sizeof(struct mlxsw_sp_rif_subport),
3705 .setup = mlxsw_sp_rif_subport_setup,
3706 .configure = mlxsw_sp_rif_subport_configure,
3707 .deconfigure = mlxsw_sp_rif_subport_deconfigure,
3708 .fid_get = mlxsw_sp_rif_subport_fid_get,
3709};
3710
3711static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
3712 enum mlxsw_reg_ritr_if_type type,
3713 u16 vid_fid, bool enable)
3714{
3715 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3716 char ritr_pl[MLXSW_REG_RITR_LEN];
3717
3718 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
3719 rif->dev->mtu, rif->dev->dev_addr);
3720 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
3721
3722 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3723}
3724
3725static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
3726{
3727 return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
3728}
3729
3730static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
3731{
3732 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3733 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3734 int err;
3735
3736 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
3737 if (err)
3738 return err;
3739
0d284818
IS		 3740 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
3741 mlxsw_sp_router_port(mlxsw_sp), true);
3742 if (err)
3743 goto err_fid_mc_flood_set;
3744
e4f3c1c1
IS		 3745 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3746 mlxsw_sp_router_port(mlxsw_sp), true);
3747 if (err)
3748 goto err_fid_bc_flood_set;
3749
3750 return 0;
3751
3752err_fid_bc_flood_set:
0d284818
IS		 3753 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
3754 mlxsw_sp_router_port(mlxsw_sp), false);
3755err_fid_mc_flood_set:
e4f3c1c1
IS		 3756 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3757 return err;
3758}
3759
3760static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
3761{
3762 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3763 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3764
3765 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3766 mlxsw_sp_router_port(mlxsw_sp), false);
0d284818
IS		 3767 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
3768 mlxsw_sp_router_port(mlxsw_sp), false);
e4f3c1c1
IS		 3769 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3770}
3771
3772static struct mlxsw_sp_fid *
3773mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
3774{
3775 u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
3776
3777 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
3778}
3779
3780static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
3781 .type = MLXSW_SP_RIF_TYPE_VLAN,
3782 .rif_size = sizeof(struct mlxsw_sp_rif),
3783 .configure = mlxsw_sp_rif_vlan_configure,
3784 .deconfigure = mlxsw_sp_rif_vlan_deconfigure,
3785 .fid_get = mlxsw_sp_rif_vlan_fid_get,
3786};
3787
3788static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
3789{
3790 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3791 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3792 int err;
3793
3794 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
3795 true);
3796 if (err)
3797 return err;
3798
0d284818
IS		 3799 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
3800 mlxsw_sp_router_port(mlxsw_sp), true);
3801 if (err)
3802 goto err_fid_mc_flood_set;
3803
e4f3c1c1
IS		 3804 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3805 mlxsw_sp_router_port(mlxsw_sp), true);
3806 if (err)
3807 goto err_fid_bc_flood_set;
3808
3809 return 0;
3810
3811err_fid_bc_flood_set:
0d284818
IS		 3812 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
3813 mlxsw_sp_router_port(mlxsw_sp), false);
3814err_fid_mc_flood_set:
e4f3c1c1
IS		 3815 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3816 return err;
3817}
3818
3819static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
3820{
3821 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3822 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3823
3824 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3825 mlxsw_sp_router_port(mlxsw_sp), false);
0d284818
IS		 3826 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
3827 mlxsw_sp_router_port(mlxsw_sp), false);
e4f3c1c1
IS		 3828 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3829}
3830
3831static struct mlxsw_sp_fid *
3832mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
3833{
3834 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
3835}
3836
3837static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
3838 .type = MLXSW_SP_RIF_TYPE_FID,
3839 .rif_size = sizeof(struct mlxsw_sp_rif),
3840 .configure = mlxsw_sp_rif_fid_configure,
3841 .deconfigure = mlxsw_sp_rif_fid_deconfigure,
3842 .fid_get = mlxsw_sp_rif_fid_fid_get,
3843};
3844
3845static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
3846 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops,
3847 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_ops,
3848 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops,
3849};
3850
348b8fc3
IS		 3851static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
3852{
3853 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3854
3855 mlxsw_sp->router->rifs = kcalloc(max_rifs,
3856 sizeof(struct mlxsw_sp_rif *),
3857 GFP_KERNEL);
3858 if (!mlxsw_sp->router->rifs)
3859 return -ENOMEM;
e4f3c1c1
IS		 3860
3861 mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
3862
348b8fc3
IS		 3863 return 0;
3864}
3865
3866static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
3867{
3868 int i;
3869
3870 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3871 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
3872
3873 kfree(mlxsw_sp->router->rifs);
3874}
3875
c3852ef7
IS		 3876static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3877{
7e39d115 3878 struct mlxsw_sp_router *router;
c3852ef7
IS		 3879
3880 /* Flush pending FIB notifications and then flush the device's
3881 * table before requesting another dump. The FIB notification
3882 * block is unregistered, so no need to take RTNL.
3883 */
3884 mlxsw_core_flush_owq();
7e39d115
IS		 3885 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
3886 mlxsw_sp_router_fib_flush(router->mlxsw_sp);
c3852ef7
IS		 3887}
3888
4724ba56
IS		 3889static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3890{
3891 char rgcr_pl[MLXSW_REG_RGCR_LEN];
3892 u64 max_rifs;
3893 int err;
3894
3895 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3896 return -EIO;
4724ba56 3897 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
4724ba56 3898
e29237e7 3899 mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
4724ba56
IS		 3900 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3901 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3902 if (err)
348b8fc3 3903 return err;
4724ba56 3904 return 0;
4724ba56
IS		 3905}
3906
3907static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3908{
3909 char rgcr_pl[MLXSW_REG_RGCR_LEN];
4724ba56 3910
e29237e7 3911 mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
4724ba56 3912 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
4724ba56
IS		 3913}
3914
b45f64d1
JP		 3915int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3916{
9011b677 3917 struct mlxsw_sp_router *router;
b45f64d1
JP		 3918 int err;
3919
9011b677
IS		 3920 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
3921 if (!router)
3922 return -ENOMEM;
3923 mlxsw_sp->router = router;
3924 router->mlxsw_sp = mlxsw_sp;
3925
3926 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
b45f64d1
JP		 3927 err = __mlxsw_sp_router_init(mlxsw_sp);
3928 if (err)
9011b677 3929 goto err_router_init;
b45f64d1 3930
348b8fc3
IS		 3931 err = mlxsw_sp_rifs_init(mlxsw_sp);
3932 if (err)
3933 goto err_rifs_init;
3934
9011b677 3935 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
c53b8e1b
IS		 3936 &mlxsw_sp_nexthop_ht_params);
3937 if (err)
3938 goto err_nexthop_ht_init;
3939
9011b677 3940 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
e9ad5e7d
IS		 3941 &mlxsw_sp_nexthop_group_ht_params);
3942 if (err)
3943 goto err_nexthop_group_ht_init;
3944
8494ab06
IS		 3945 err = mlxsw_sp_lpm_init(mlxsw_sp);
3946 if (err)
3947 goto err_lpm_init;
3948
b45f64d1
JP		 3949 err = mlxsw_sp_vrs_init(mlxsw_sp);
3950 if (err)
3951 goto err_vrs_init;
3952
8c9583a8 3953 err = mlxsw_sp_neigh_init(mlxsw_sp);
b45f64d1
JP		 3954 if (err)
3955 goto err_neigh_init;
3956
7e39d115
IS		 3957 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3958 err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
c3852ef7
IS		 3959 mlxsw_sp_router_fib_dump_flush);
3960 if (err)
3961 goto err_register_fib_notifier;
3962
b45f64d1
JP		 3963 return 0;
3964
c3852ef7
IS		 3965err_register_fib_notifier:
3966 mlxsw_sp_neigh_fini(mlxsw_sp);
b45f64d1
JP		 3967err_neigh_init:
3968 mlxsw_sp_vrs_fini(mlxsw_sp);
3969err_vrs_init:
8494ab06
IS		 3970 mlxsw_sp_lpm_fini(mlxsw_sp);
3971err_lpm_init:
9011b677 3972 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
e9ad5e7d 3973err_nexthop_group_ht_init:
9011b677 3974 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
c53b8e1b 3975err_nexthop_ht_init:
348b8fc3
IS		 3976 mlxsw_sp_rifs_fini(mlxsw_sp);
3977err_rifs_init:
b45f64d1 3978 __mlxsw_sp_router_fini(mlxsw_sp);
9011b677
IS		 3979err_router_init:
3980 kfree(mlxsw_sp->router);
b45f64d1
JP		 3981 return err;
3982}
3983
3984void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3985{
7e39d115 3986 unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
b45f64d1
JP		 3987 mlxsw_sp_neigh_fini(mlxsw_sp);
3988 mlxsw_sp_vrs_fini(mlxsw_sp);
8494ab06 3989 mlxsw_sp_lpm_fini(mlxsw_sp);
9011b677
IS		 3990 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
3991 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
348b8fc3 3992 mlxsw_sp_rifs_fini(mlxsw_sp);
b45f64d1 3993 __mlxsw_sp_router_fini(mlxsw_sp);
9011b677 3994 kfree(mlxsw_sp->router);
b45f64d1 3995}