projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
bonding:record primary when modify it via sysfs
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / ethernet / sfc / filter.c
1 /****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2010 Solarflare Communications Inc.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
8 */
9
10 #include <linux/in.h>
11 #include <net/ip.h>
12 #include "efx.h"
13 #include "filter.h"
14 #include "io.h"
15 #include "nic.h"
16 #include "regs.h"
17
18 /* "Fudge factors" - difference between programmed value and actual depth.
19 * Due to pipelined implementation we need to program H/W with a value that
20 * is larger than the hop limit we want.
21 */
22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
24
25 /* Hard maximum hop limit. Hardware will time-out beyond 200-something.
26 * We also need to avoid infinite loops in efx_filter_search() when the
27 * table is full.
28 */
29 #define FILTER_CTL_SRCH_MAX 200
30
31 /* Don't try very hard to find space for performance hints, as this is
32 * counter-productive. */
33 #define FILTER_CTL_SRCH_HINT_MAX 5
34
35 enum efx_filter_table_id {
36 EFX_FILTER_TABLE_RX_IP = 0,
37 EFX_FILTER_TABLE_RX_MAC,
38 EFX_FILTER_TABLE_RX_DEF,
39 EFX_FILTER_TABLE_TX_MAC,
40 EFX_FILTER_TABLE_COUNT,
41 };
42
43 enum efx_filter_index {
44 EFX_FILTER_INDEX_UC_DEF,
45 EFX_FILTER_INDEX_MC_DEF,
46 EFX_FILTER_SIZE_RX_DEF,
47 };
48
49 struct efx_filter_table {
50 enum efx_filter_table_id id;
51 u32 offset; /* address of table relative to BAR */
52 unsigned size; /* number of entries */
53 unsigned step; /* step between entries */
54 unsigned used; /* number currently used */
55 unsigned long *used_bitmap;
56 struct efx_filter_spec *spec;
57 unsigned search_depth[EFX_FILTER_TYPE_COUNT];
58 };
59
60 struct efx_filter_state {
61 spinlock_t lock;
62 struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
63 #ifdef CONFIG_RFS_ACCEL
64 u32 *rps_flow_id;
65 unsigned rps_expire_index;
66 #endif
67 };
68
69 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
70 * key derived from the n-tuple. The initial LFSR state is 0xffff. */
71 static u16 efx_filter_hash(u32 key)
72 {
73 u16 tmp;
74
75 /* First 16 rounds */
76 tmp = 0x1fff ^ key >> 16;
77 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
78 tmp = tmp ^ tmp >> 9;
79 /* Last 16 rounds */
80 tmp = tmp ^ tmp << 13 ^ key;
81 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
82 return tmp ^ tmp >> 9;
83 }
84
85 /* To allow for hash collisions, filter search continues at these
86 * increments from the first possible entry selected by the hash. */
87 static u16 efx_filter_increment(u32 key)
88 {
89 return key * 2 - 1;
90 }
91
92 static enum efx_filter_table_id
93 efx_filter_spec_table_id(const struct efx_filter_spec *spec)
94 {
95 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
96 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
97 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
98 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
99 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
100 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
101 BUILD_BUG_ON(EFX_FILTER_TABLE_TX_MAC != EFX_FILTER_TABLE_RX_MAC + 2);
102 EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
103 return (spec->type >> 2) + ((spec->flags & EFX_FILTER_FLAG_TX) ? 2 : 0);
104 }
105
106 static struct efx_filter_table *
107 efx_filter_spec_table(struct efx_filter_state *state,
108 const struct efx_filter_spec *spec)
109 {
110 if (spec->type == EFX_FILTER_UNSPEC)
111 return NULL;
112 else
113 return &state->table[efx_filter_spec_table_id(spec)];
114 }
115
116 static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
117 {
118 memset(table->search_depth, 0, sizeof(table->search_depth));
119 }
120
121 static void efx_filter_push_rx_config(struct efx_nic *efx)
122 {
123 struct efx_filter_state *state = efx->filter_state;
124 struct efx_filter_table *table;
125 efx_oword_t filter_ctl;
126
127 efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
128
129 table = &state->table[EFX_FILTER_TABLE_RX_IP];
130 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
131 table->search_depth[EFX_FILTER_TCP_FULL] +
132 FILTER_CTL_SRCH_FUDGE_FULL);
133 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
134 table->search_depth[EFX_FILTER_TCP_WILD] +
135 FILTER_CTL_SRCH_FUDGE_WILD);
136 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
137 table->search_depth[EFX_FILTER_UDP_FULL] +
138 FILTER_CTL_SRCH_FUDGE_FULL);
139 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
140 table->search_depth[EFX_FILTER_UDP_WILD] +
141 FILTER_CTL_SRCH_FUDGE_WILD);
142
143 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
144 if (table->size) {
145 EFX_SET_OWORD_FIELD(
146 filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
147 table->search_depth[EFX_FILTER_MAC_FULL] +
148 FILTER_CTL_SRCH_FUDGE_FULL);
149 EFX_SET_OWORD_FIELD(
150 filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
151 table->search_depth[EFX_FILTER_MAC_WILD] +
152 FILTER_CTL_SRCH_FUDGE_WILD);
153 }
154
155 table = &state->table[EFX_FILTER_TABLE_RX_DEF];
156 if (table->size) {
157 EFX_SET_OWORD_FIELD(
158 filter_ctl, FRF_CZ_UNICAST_NOMATCH_Q_ID,
159 table->spec[EFX_FILTER_INDEX_UC_DEF].dmaq_id);
160 EFX_SET_OWORD_FIELD(
161 filter_ctl, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED,
162 !!(table->spec[EFX_FILTER_INDEX_UC_DEF].flags &
163 EFX_FILTER_FLAG_RX_RSS));
164 EFX_SET_OWORD_FIELD(
165 filter_ctl, FRF_CZ_UNICAST_NOMATCH_IP_OVERRIDE,
166 !!(table->spec[EFX_FILTER_INDEX_UC_DEF].flags &
167 EFX_FILTER_FLAG_RX_OVERRIDE_IP));
168 EFX_SET_OWORD_FIELD(
169 filter_ctl, FRF_CZ_MULTICAST_NOMATCH_Q_ID,
170 table->spec[EFX_FILTER_INDEX_MC_DEF].dmaq_id);
171 EFX_SET_OWORD_FIELD(
172 filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED,
173 !!(table->spec[EFX_FILTER_INDEX_MC_DEF].flags &
174 EFX_FILTER_FLAG_RX_RSS));
175 EFX_SET_OWORD_FIELD(
176 filter_ctl, FRF_CZ_MULTICAST_NOMATCH_IP_OVERRIDE,
177 !!(table->spec[EFX_FILTER_INDEX_MC_DEF].flags &
178 EFX_FILTER_FLAG_RX_OVERRIDE_IP));
179 }
180
181 efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
182 }
183
184 static void efx_filter_push_tx_limits(struct efx_nic *efx)
185 {
186 struct efx_filter_state *state = efx->filter_state;
187 struct efx_filter_table *table;
188 efx_oword_t tx_cfg;
189
190 efx_reado(efx, &tx_cfg, FR_AZ_TX_CFG);
191
192 table = &state->table[EFX_FILTER_TABLE_TX_MAC];
193 if (table->size) {
194 EFX_SET_OWORD_FIELD(
195 tx_cfg, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
196 table->search_depth[EFX_FILTER_MAC_FULL] +
197 FILTER_CTL_SRCH_FUDGE_FULL);
198 EFX_SET_OWORD_FIELD(
199 tx_cfg, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
200 table->search_depth[EFX_FILTER_MAC_WILD] +
201 FILTER_CTL_SRCH_FUDGE_WILD);
202 }
203
204 efx_writeo(efx, &tx_cfg, FR_AZ_TX_CFG);
205 }
206
207 static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
208 __be32 host1, __be16 port1,
209 __be32 host2, __be16 port2)
210 {
211 spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
212 spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
213 spec->data[2] = ntohl(host2);
214 }
215
216 static inline void __efx_filter_get_ipv4(const struct efx_filter_spec *spec,
217 __be32 *host1, __be16 *port1,
218 __be32 *host2, __be16 *port2)
219 {
220 *host1 = htonl(spec->data[0] >> 16 | spec->data[1] << 16);
221 *port1 = htons(spec->data[0]);
222 *host2 = htonl(spec->data[2]);
223 *port2 = htons(spec->data[1] >> 16);
224 }
225
226 /**
227 * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
228 * @spec: Specification to initialise
229 * @proto: Transport layer protocol number
230 * @host: Local host address (network byte order)
231 * @port: Local port (network byte order)
232 */
233 int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
234 __be32 host, __be16 port)
235 {
236 __be32 host1;
237 __be16 port1;
238
239 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
240
241 /* This cannot currently be combined with other filtering */
242 if (spec->type != EFX_FILTER_UNSPEC)
243 return -EPROTONOSUPPORT;
244
245 if (port == 0)
246 return -EINVAL;
247
248 switch (proto) {
249 case IPPROTO_TCP:
250 spec->type = EFX_FILTER_TCP_WILD;
251 break;
252 case IPPROTO_UDP:
253 spec->type = EFX_FILTER_UDP_WILD;
254 break;
255 default:
256 return -EPROTONOSUPPORT;
257 }
258
259 /* Filter is constructed in terms of source and destination,
260 * with the odd wrinkle that the ports are swapped in a UDP
261 * wildcard filter. We need to convert from local and remote
262 * (= zero for wildcard) addresses.
263 */
264 host1 = 0;
265 if (proto != IPPROTO_UDP) {
266 port1 = 0;
267 } else {
268 port1 = port;
269 port = 0;
270 }
271
272 __efx_filter_set_ipv4(spec, host1, port1, host, port);
273 return 0;
274 }
275
276 int efx_filter_get_ipv4_local(const struct efx_filter_spec *spec,
277 u8 *proto, __be32 *host, __be16 *port)
278 {
279 __be32 host1;
280 __be16 port1;
281
282 switch (spec->type) {
283 case EFX_FILTER_TCP_WILD:
284 *proto = IPPROTO_TCP;
285 __efx_filter_get_ipv4(spec, &host1, &port1, host, port);
286 return 0;
287 case EFX_FILTER_UDP_WILD:
288 *proto = IPPROTO_UDP;
289 __efx_filter_get_ipv4(spec, &host1, port, host, &port1);
290 return 0;
291 default:
292 return -EINVAL;
293 }
294 }
295
296 /**
297 * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
298 * @spec: Specification to initialise
299 * @proto: Transport layer protocol number
300 * @host: Local host address (network byte order)
301 * @port: Local port (network byte order)
302 * @rhost: Remote host address (network byte order)
303 * @rport: Remote port (network byte order)
304 */
305 int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
306 __be32 host, __be16 port,
307 __be32 rhost, __be16 rport)
308 {
309 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
310
311 /* This cannot currently be combined with other filtering */
312 if (spec->type != EFX_FILTER_UNSPEC)
313 return -EPROTONOSUPPORT;
314
315 if (port == 0 || rport == 0)
316 return -EINVAL;
317
318 switch (proto) {
319 case IPPROTO_TCP:
320 spec->type = EFX_FILTER_TCP_FULL;
321 break;
322 case IPPROTO_UDP:
323 spec->type = EFX_FILTER_UDP_FULL;
324 break;
325 default:
326 return -EPROTONOSUPPORT;
327 }
328
329 __efx_filter_set_ipv4(spec, rhost, rport, host, port);
330 return 0;
331 }
332
333 int efx_filter_get_ipv4_full(const struct efx_filter_spec *spec,
334 u8 *proto, __be32 *host, __be16 *port,
335 __be32 *rhost, __be16 *rport)
336 {
337 switch (spec->type) {
338 case EFX_FILTER_TCP_FULL:
339 *proto = IPPROTO_TCP;
340 break;
341 case EFX_FILTER_UDP_FULL:
342 *proto = IPPROTO_UDP;
343 break;
344 default:
345 return -EINVAL;
346 }
347
348 __efx_filter_get_ipv4(spec, rhost, rport, host, port);
349 return 0;
350 }
351
352 /**
353 * efx_filter_set_eth_local - specify local Ethernet address and optional VID
354 * @spec: Specification to initialise
355 * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
356 * @addr: Local Ethernet MAC address
357 */
358 int efx_filter_set_eth_local(struct efx_filter_spec *spec,
359 u16 vid, const u8 *addr)
360 {
361 EFX_BUG_ON_PARANOID(!(spec->flags &
362 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
363
364 /* This cannot currently be combined with other filtering */
365 if (spec->type != EFX_FILTER_UNSPEC)
366 return -EPROTONOSUPPORT;
367
368 if (vid == EFX_FILTER_VID_UNSPEC) {
369 spec->type = EFX_FILTER_MAC_WILD;
370 spec->data[0] = 0;
371 } else {
372 spec->type = EFX_FILTER_MAC_FULL;
373 spec->data[0] = vid;
374 }
375
376 spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
377 spec->data[2] = addr[0] << 8 | addr[1];
378 return 0;
379 }
380
381 /**
382 * efx_filter_set_uc_def - specify matching otherwise-unmatched unicast
383 * @spec: Specification to initialise
384 */
385 int efx_filter_set_uc_def(struct efx_filter_spec *spec)
386 {
387 EFX_BUG_ON_PARANOID(!(spec->flags &
388 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
389
390 if (spec->type != EFX_FILTER_UNSPEC)
391 return -EINVAL;
392
393 spec->type = EFX_FILTER_UC_DEF;
394 memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */
395 return 0;
396 }
397
398 /**
399 * efx_filter_set_mc_def - specify matching otherwise-unmatched multicast
400 * @spec: Specification to initialise
401 */
402 int efx_filter_set_mc_def(struct efx_filter_spec *spec)
403 {
404 EFX_BUG_ON_PARANOID(!(spec->flags &
405 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
406
407 if (spec->type != EFX_FILTER_UNSPEC)
408 return -EINVAL;
409
410 spec->type = EFX_FILTER_MC_DEF;
411 memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */
412 return 0;
413 }
414
415 static void efx_filter_reset_rx_def(struct efx_nic *efx, unsigned filter_idx)
416 {
417 struct efx_filter_state *state = efx->filter_state;
418 struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_DEF];
419 struct efx_filter_spec *spec = &table->spec[filter_idx];
420
421 efx_filter_init_rx(spec, EFX_FILTER_PRI_MANUAL,
422 EFX_FILTER_FLAG_RX_RSS, 0);
423 spec->type = EFX_FILTER_UC_DEF + filter_idx;
424 table->used_bitmap[0] |= 1 << filter_idx;
425 }
426
427 int efx_filter_get_eth_local(const struct efx_filter_spec *spec,
428 u16 *vid, u8 *addr)
429 {
430 switch (spec->type) {
431 case EFX_FILTER_MAC_WILD:
432 *vid = EFX_FILTER_VID_UNSPEC;
433 break;
434 case EFX_FILTER_MAC_FULL:
435 *vid = spec->data[0];
436 break;
437 default:
438 return -EINVAL;
439 }
440
441 addr[0] = spec->data[2] >> 8;
442 addr[1] = spec->data[2];
443 addr[2] = spec->data[1] >> 24;
444 addr[3] = spec->data[1] >> 16;
445 addr[4] = spec->data[1] >> 8;
446 addr[5] = spec->data[1];
447 return 0;
448 }
449
450 /* Build a filter entry and return its n-tuple key. */
451 static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
452 {
453 u32 data3;
454
455 switch (efx_filter_spec_table_id(spec)) {
456 case EFX_FILTER_TABLE_RX_IP: {
457 bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
458 spec->type == EFX_FILTER_UDP_WILD);
459 EFX_POPULATE_OWORD_7(
460 *filter,
461 FRF_BZ_RSS_EN,
462 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
463 FRF_BZ_SCATTER_EN,
464 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
465 FRF_BZ_TCP_UDP, is_udp,
466 FRF_BZ_RXQ_ID, spec->dmaq_id,
467 EFX_DWORD_2, spec->data[2],
468 EFX_DWORD_1, spec->data[1],
469 EFX_DWORD_0, spec->data[0]);
470 data3 = is_udp;
471 break;
472 }
473
474 case EFX_FILTER_TABLE_RX_DEF:
475 /* One filter spec per type */
476 BUILD_BUG_ON(EFX_FILTER_INDEX_UC_DEF != 0);
477 BUILD_BUG_ON(EFX_FILTER_INDEX_MC_DEF !=
478 EFX_FILTER_MC_DEF - EFX_FILTER_UC_DEF);
479 return spec->type - EFX_FILTER_UC_DEF;
480
481 case EFX_FILTER_TABLE_RX_MAC: {
482 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
483 EFX_POPULATE_OWORD_8(
484 *filter,
485 FRF_CZ_RMFT_RSS_EN,
486 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
487 FRF_CZ_RMFT_SCATTER_EN,
488 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
489 FRF_CZ_RMFT_IP_OVERRIDE,
490 !!(spec->flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP),
491 FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
492 FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
493 FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
494 FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
495 FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
496 data3 = is_wild;
497 break;
498 }
499
500 case EFX_FILTER_TABLE_TX_MAC: {
501 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
502 EFX_POPULATE_OWORD_5(*filter,
503 FRF_CZ_TMFT_TXQ_ID, spec->dmaq_id,
504 FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
505 FRF_CZ_TMFT_SRC_MAC_HI, spec->data[2],
506 FRF_CZ_TMFT_SRC_MAC_LO, spec->data[1],
507 FRF_CZ_TMFT_VLAN_ID, spec->data[0]);
508 data3 = is_wild | spec->dmaq_id << 1;
509 break;
510 }
511
512 default:
513 BUG();
514 }
515
516 return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
517 }
518
519 static bool efx_filter_equal(const struct efx_filter_spec *left,
520 const struct efx_filter_spec *right)
521 {
522 if (left->type != right->type ||
523 memcmp(left->data, right->data, sizeof(left->data)))
524 return false;
525
526 if (left->flags & EFX_FILTER_FLAG_TX &&
527 left->dmaq_id != right->dmaq_id)
528 return false;
529
530 return true;
531 }
532
533 static int efx_filter_search(struct efx_filter_table *table,
534 struct efx_filter_spec *spec, u32 key,
535 bool for_insert, unsigned int *depth_required)
536 {
537 unsigned hash, incr, filter_idx, depth, depth_max;
538
539 hash = efx_filter_hash(key);
540 incr = efx_filter_increment(key);
541
542 filter_idx = hash & (table->size - 1);
543 depth = 1;
544 depth_max = (for_insert ?
545 (spec->priority <= EFX_FILTER_PRI_HINT ?
546 FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX) :
547 table->search_depth[spec->type]);
548
549 for (;;) {
550 /* Return success if entry is used and matches this spec
551 * or entry is unused and we are trying to insert.
552 */
553 if (test_bit(filter_idx, table->used_bitmap) ?
554 efx_filter_equal(spec, &table->spec[filter_idx]) :
555 for_insert) {
556 *depth_required = depth;
557 return filter_idx;
558 }
559
560 /* Return failure if we reached the maximum search depth */
561 if (depth == depth_max)
562 return for_insert ? -EBUSY : -ENOENT;
563
564 filter_idx = (filter_idx + incr) & (table->size - 1);
565 ++depth;
566 }
567 }
568
569 /*
570 * Construct/deconstruct external filter IDs. These must be ordered
571 * by matching priority, for RX NFC semantics.
572 *
573 * Each RX MAC filter entry has a flag for whether it can override an
574 * RX IP filter that also matches. So we assign locations for MAC
575 * filters with overriding behaviour, then for IP filters, then for
576 * MAC filters without overriding behaviour.
577 */
578
579 #define EFX_FILTER_MATCH_PRI_RX_MAC_OVERRIDE_IP 0
580 #define EFX_FILTER_MATCH_PRI_RX_DEF_OVERRIDE_IP 1
581 #define EFX_FILTER_MATCH_PRI_NORMAL_BASE 2
582
583 #define EFX_FILTER_INDEX_WIDTH 13
584 #define EFX_FILTER_INDEX_MASK ((1 << EFX_FILTER_INDEX_WIDTH) - 1)
585
586 static inline u32 efx_filter_make_id(enum efx_filter_table_id table_id,
587 unsigned int index, u8 flags)
588 {
589 unsigned int match_pri = EFX_FILTER_MATCH_PRI_NORMAL_BASE + table_id;
590
591 if (flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP) {
592 if (table_id == EFX_FILTER_TABLE_RX_MAC)
593 match_pri = EFX_FILTER_MATCH_PRI_RX_MAC_OVERRIDE_IP;
594 else if (table_id == EFX_FILTER_TABLE_RX_DEF)
595 match_pri = EFX_FILTER_MATCH_PRI_RX_DEF_OVERRIDE_IP;
596 }
597
598 return match_pri << EFX_FILTER_INDEX_WIDTH | index;
599 }
600
601 static inline enum efx_filter_table_id efx_filter_id_table_id(u32 id)
602 {
603 unsigned int match_pri = id >> EFX_FILTER_INDEX_WIDTH;
604
605 switch (match_pri) {
606 case EFX_FILTER_MATCH_PRI_RX_MAC_OVERRIDE_IP:
607 return EFX_FILTER_TABLE_RX_MAC;
608 case EFX_FILTER_MATCH_PRI_RX_DEF_OVERRIDE_IP:
609 return EFX_FILTER_TABLE_RX_DEF;
610 default:
611 return match_pri - EFX_FILTER_MATCH_PRI_NORMAL_BASE;
612 }
613 }
614
615 static inline unsigned int efx_filter_id_index(u32 id)
616 {
617 return id & EFX_FILTER_INDEX_MASK;
618 }
619
620 static inline u8 efx_filter_id_flags(u32 id)
621 {
622 unsigned int match_pri = id >> EFX_FILTER_INDEX_WIDTH;
623
624 if (match_pri < EFX_FILTER_MATCH_PRI_NORMAL_BASE)
625 return EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_RX_OVERRIDE_IP;
626 else if (match_pri <=
627 EFX_FILTER_MATCH_PRI_NORMAL_BASE + EFX_FILTER_TABLE_RX_DEF)
628 return EFX_FILTER_FLAG_RX;
629 else
630 return EFX_FILTER_FLAG_TX;
631 }
632
633 u32 efx_filter_get_rx_id_limit(struct efx_nic *efx)
634 {
635 struct efx_filter_state *state = efx->filter_state;
636 unsigned int table_id = EFX_FILTER_TABLE_RX_DEF;
637
638 do {
639 if (state->table[table_id].size != 0)
640 return ((EFX_FILTER_MATCH_PRI_NORMAL_BASE + table_id)
641 << EFX_FILTER_INDEX_WIDTH) +
642 state->table[table_id].size;
643 } while (table_id--);
644
645 return 0;
646 }
647
648 /**
649 * efx_filter_insert_filter - add or replace a filter
650 * @efx: NIC in which to insert the filter
651 * @spec: Specification for the filter
652 * @replace: Flag for whether the specified filter may replace a filter
653 * with an identical match expression and equal or lower priority
654 *
655 * On success, return the filter ID.
656 * On failure, return a negative error code.
657 */
658 s32 efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
659 bool replace)
660 {
661 struct efx_filter_state *state = efx->filter_state;
662 struct efx_filter_table *table = efx_filter_spec_table(state, spec);
663 struct efx_filter_spec *saved_spec;
664 efx_oword_t filter;
665 unsigned int filter_idx, depth;
666 u32 key;
667 int rc;
668
669 if (!table || table->size == 0)
670 return -EINVAL;
671
672 key = efx_filter_build(&filter, spec);
673
674 netif_vdbg(efx, hw, efx->net_dev,
675 "%s: type %d search_depth=%d", __func__, spec->type,
676 table->search_depth[spec->type]);
677
678 spin_lock_bh(&state->lock);
679
680 rc = efx_filter_search(table, spec, key, true, &depth);
681 if (rc < 0)
682 goto out;
683 filter_idx = rc;
684 BUG_ON(filter_idx >= table->size);
685 saved_spec = &table->spec[filter_idx];
686
687 if (test_bit(filter_idx, table->used_bitmap)) {
688 /* Should we replace the existing filter? */
689 if (!replace) {
690 rc = -EEXIST;
691 goto out;
692 }
693 if (spec->priority < saved_spec->priority) {
694 rc = -EPERM;
695 goto out;
696 }
697 } else {
698 __set_bit(filter_idx, table->used_bitmap);
699 ++table->used;
700 }
701 *saved_spec = *spec;
702
703 if (table->id == EFX_FILTER_TABLE_RX_DEF) {
704 efx_filter_push_rx_config(efx);
705 } else {
706 if (table->search_depth[spec->type] < depth) {
707 table->search_depth[spec->type] = depth;
708 if (spec->flags & EFX_FILTER_FLAG_TX)
709 efx_filter_push_tx_limits(efx);
710 else
711 efx_filter_push_rx_config(efx);
712 }
713
714 efx_writeo(efx, &filter,
715 table->offset + table->step * filter_idx);
716 }
717
718 netif_vdbg(efx, hw, efx->net_dev,
719 "%s: filter type %d index %d rxq %u set",
720 __func__, spec->type, filter_idx, spec->dmaq_id);
721 rc = efx_filter_make_id(table->id, filter_idx, spec->flags);
722
723 out:
724 spin_unlock_bh(&state->lock);
725 return rc;
726 }
727
728 static void efx_filter_table_clear_entry(struct efx_nic *efx,
729 struct efx_filter_table *table,
730 unsigned int filter_idx)
731 {
732 static efx_oword_t filter;
733
734 if (table->id == EFX_FILTER_TABLE_RX_DEF) {
735 /* RX default filters must always exist */
736 efx_filter_reset_rx_def(efx, filter_idx);
737 efx_filter_push_rx_config(efx);
738 } else if (test_bit(filter_idx, table->used_bitmap)) {
739 __clear_bit(filter_idx, table->used_bitmap);
740 --table->used;
741 memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
742
743 efx_writeo(efx, &filter,
744 table->offset + table->step * filter_idx);
745 }
746 }
747
748 /**
749 * efx_filter_remove_id_safe - remove a filter by ID, carefully
750 * @efx: NIC from which to remove the filter
751 * @priority: Priority of filter, as passed to @efx_filter_insert_filter
752 * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
753 *
754 * This function will range-check @filter_id, so it is safe to call
755 * with a value passed from userland.
756 */
757 int efx_filter_remove_id_safe(struct efx_nic *efx,
758 enum efx_filter_priority priority,
759 u32 filter_id)
760 {
761 struct efx_filter_state *state = efx->filter_state;
762 enum efx_filter_table_id table_id;
763 struct efx_filter_table *table;
764 unsigned int filter_idx;
765 struct efx_filter_spec *spec;
766 u8 filter_flags;
767 int rc;
768
769 table_id = efx_filter_id_table_id(filter_id);
770 if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
771 return -ENOENT;
772 table = &state->table[table_id];
773
774 filter_idx = efx_filter_id_index(filter_id);
775 if (filter_idx >= table->size)
776 return -ENOENT;
777 spec = &table->spec[filter_idx];
778
779 filter_flags = efx_filter_id_flags(filter_id);
780
781 spin_lock_bh(&state->lock);
782
783 if (test_bit(filter_idx, table->used_bitmap) &&
784 spec->priority == priority &&
785 !((spec->flags ^ filter_flags) & EFX_FILTER_FLAG_RX_OVERRIDE_IP)) {
786 efx_filter_table_clear_entry(efx, table, filter_idx);
787 if (table->used == 0)
788 efx_filter_table_reset_search_depth(table);
789 rc = 0;
790 } else {
791 rc = -ENOENT;
792 }
793
794 spin_unlock_bh(&state->lock);
795
796 return rc;
797 }
798
799 /**
800 * efx_filter_get_filter_safe - retrieve a filter by ID, carefully
801 * @efx: NIC from which to remove the filter
802 * @priority: Priority of filter, as passed to @efx_filter_insert_filter
803 * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
804 * @spec: Buffer in which to store filter specification
805 *
806 * This function will range-check @filter_id, so it is safe to call
807 * with a value passed from userland.
808 */
809 int efx_filter_get_filter_safe(struct efx_nic *efx,
810 enum efx_filter_priority priority,
811 u32 filter_id, struct efx_filter_spec *spec_buf)
812 {
813 struct efx_filter_state *state = efx->filter_state;
814 enum efx_filter_table_id table_id;
815 struct efx_filter_table *table;
816 struct efx_filter_spec *spec;
817 unsigned int filter_idx;
818 u8 filter_flags;
819 int rc;
820
821 table_id = efx_filter_id_table_id(filter_id);
822 if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
823 return -ENOENT;
824 table = &state->table[table_id];
825
826 filter_idx = efx_filter_id_index(filter_id);
827 if (filter_idx >= table->size)
828 return -ENOENT;
829 spec = &table->spec[filter_idx];
830
831 filter_flags = efx_filter_id_flags(filter_id);
832
833 spin_lock_bh(&state->lock);
834
835 if (test_bit(filter_idx, table->used_bitmap) &&
836 spec->priority == priority &&
837 !((spec->flags ^ filter_flags) & EFX_FILTER_FLAG_RX_OVERRIDE_IP)) {
838 *spec_buf = *spec;
839 rc = 0;
840 } else {
841 rc = -ENOENT;
842 }
843
844 spin_unlock_bh(&state->lock);
845
846 return rc;
847 }
848
849 static void efx_filter_table_clear(struct efx_nic *efx,
850 enum efx_filter_table_id table_id,
851 enum efx_filter_priority priority)
852 {
853 struct efx_filter_state *state = efx->filter_state;
854 struct efx_filter_table *table = &state->table[table_id];
855 unsigned int filter_idx;
856
857 spin_lock_bh(&state->lock);
858
859 for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
860 if (table->spec[filter_idx].priority <= priority)
861 efx_filter_table_clear_entry(efx, table, filter_idx);
862 if (table->used == 0)
863 efx_filter_table_reset_search_depth(table);
864
865 spin_unlock_bh(&state->lock);
866 }
867
868 /**
869 * efx_filter_clear_rx - remove RX filters by priority
870 * @efx: NIC from which to remove the filters
871 * @priority: Maximum priority to remove
872 */
873 void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
874 {
875 efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
876 efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
877 }
878
879 u32 efx_filter_count_rx_used(struct efx_nic *efx,
880 enum efx_filter_priority priority)
881 {
882 struct efx_filter_state *state = efx->filter_state;
883 enum efx_filter_table_id table_id;
884 struct efx_filter_table *table;
885 unsigned int filter_idx;
886 u32 count = 0;
887
888 spin_lock_bh(&state->lock);
889
890 for (table_id = EFX_FILTER_TABLE_RX_IP;
891 table_id <= EFX_FILTER_TABLE_RX_DEF;
892 table_id++) {
893 table = &state->table[table_id];
894 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
895 if (test_bit(filter_idx, table->used_bitmap) &&
896 table->spec[filter_idx].priority == priority)
897 ++count;
898 }
899 }
900
901 spin_unlock_bh(&state->lock);
902
903 return count;
904 }
905
906 s32 efx_filter_get_rx_ids(struct efx_nic *efx,
907 enum efx_filter_priority priority,
908 u32 *buf, u32 size)
909 {
910 struct efx_filter_state *state = efx->filter_state;
911 enum efx_filter_table_id table_id;
912 struct efx_filter_table *table;
913 unsigned int filter_idx;
914 s32 count = 0;
915
916 spin_lock_bh(&state->lock);
917
918 for (table_id = EFX_FILTER_TABLE_RX_IP;
919 table_id <= EFX_FILTER_TABLE_RX_DEF;
920 table_id++) {
921 table = &state->table[table_id];
922 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
923 if (test_bit(filter_idx, table->used_bitmap) &&
924 table->spec[filter_idx].priority == priority) {
925 if (count == size) {
926 count = -EMSGSIZE;
927 goto out;
928 }
929 buf[count++] = efx_filter_make_id(
930 table_id, filter_idx,
931 table->spec[filter_idx].flags);
932 }
933 }
934 }
935 out:
936 spin_unlock_bh(&state->lock);
937
938 return count;
939 }
940
941 /* Restore filter stater after reset */
942 void efx_restore_filters(struct efx_nic *efx)
943 {
944 struct efx_filter_state *state = efx->filter_state;
945 enum efx_filter_table_id table_id;
946 struct efx_filter_table *table;
947 efx_oword_t filter;
948 unsigned int filter_idx;
949
950 spin_lock_bh(&state->lock);
951
952 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
953 table = &state->table[table_id];
954
955 /* Check whether this is a regular register table */
956 if (table->step == 0)
957 continue;
958
959 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
960 if (!test_bit(filter_idx, table->used_bitmap))
961 continue;
962 efx_filter_build(&filter, &table->spec[filter_idx]);
963 efx_writeo(efx, &filter,
964 table->offset + table->step * filter_idx);
965 }
966 }
967
968 efx_filter_push_rx_config(efx);
969 efx_filter_push_tx_limits(efx);
970
971 spin_unlock_bh(&state->lock);
972 }
973
974 int efx_probe_filters(struct efx_nic *efx)
975 {
976 struct efx_filter_state *state;
977 struct efx_filter_table *table;
978 unsigned table_id;
979
980 state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
981 if (!state)
982 return -ENOMEM;
983 efx->filter_state = state;
984
985 spin_lock_init(&state->lock);
986
987 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
988 #ifdef CONFIG_RFS_ACCEL
989 state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
990 sizeof(*state->rps_flow_id),
991 GFP_KERNEL);
992 if (!state->rps_flow_id)
993 goto fail;
994 #endif
995 table = &state->table[EFX_FILTER_TABLE_RX_IP];
996 table->id = EFX_FILTER_TABLE_RX_IP;
997 table->offset = FR_BZ_RX_FILTER_TBL0;
998 table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
999 table->step = FR_BZ_RX_FILTER_TBL0_STEP;
1000 }
1001
1002 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
1003 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
1004 table->id = EFX_FILTER_TABLE_RX_MAC;
1005 table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
1006 table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1007 table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
1008
1009 table = &state->table[EFX_FILTER_TABLE_RX_DEF];
1010 table->id = EFX_FILTER_TABLE_RX_DEF;
1011 table->size = EFX_FILTER_SIZE_RX_DEF;
1012
1013 table = &state->table[EFX_FILTER_TABLE_TX_MAC];
1014 table->id = EFX_FILTER_TABLE_TX_MAC;
1015 table->offset = FR_CZ_TX_MAC_FILTER_TBL0;
1016 table->size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1017 table->step = FR_CZ_TX_MAC_FILTER_TBL0_STEP;
1018 }
1019
1020 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
1021 table = &state->table[table_id];
1022 if (table->size == 0)
1023 continue;
1024 table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
1025 sizeof(unsigned long),
1026 GFP_KERNEL);
1027 if (!table->used_bitmap)
1028 goto fail;
1029 table->spec = vzalloc(table->size * sizeof(*table->spec));
1030 if (!table->spec)
1031 goto fail;
1032 }
1033
1034 if (state->table[EFX_FILTER_TABLE_RX_DEF].size) {
1035 /* RX default filters must always exist */
1036 unsigned i;
1037 for (i = 0; i < EFX_FILTER_SIZE_RX_DEF; i++)
1038 efx_filter_reset_rx_def(efx, i);
1039 }
1040
1041 efx_filter_push_rx_config(efx);
1042
1043 return 0;
1044
1045 fail:
1046 efx_remove_filters(efx);
1047 return -ENOMEM;
1048 }
1049
1050 void efx_remove_filters(struct efx_nic *efx)
1051 {
1052 struct efx_filter_state *state = efx->filter_state;
1053 enum efx_filter_table_id table_id;
1054
1055 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
1056 kfree(state->table[table_id].used_bitmap);
1057 vfree(state->table[table_id].spec);
1058 }
1059 #ifdef CONFIG_RFS_ACCEL
1060 kfree(state->rps_flow_id);
1061 #endif
1062 kfree(state);
1063 }
1064
1065 #ifdef CONFIG_RFS_ACCEL
1066
1067 int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
1068 u16 rxq_index, u32 flow_id)
1069 {
1070 struct efx_nic *efx = netdev_priv(net_dev);
1071 struct efx_channel *channel;
1072 struct efx_filter_state *state = efx->filter_state;
1073 struct efx_filter_spec spec;
1074 const struct iphdr *ip;
1075 const __be16 *ports;
1076 int nhoff;
1077 int rc;
1078
1079 nhoff = skb_network_offset(skb);
1080
1081 if (skb->protocol != htons(ETH_P_IP))
1082 return -EPROTONOSUPPORT;
1083
1084 /* RFS must validate the IP header length before calling us */
1085 EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip));
1086 ip = (const struct iphdr *)(skb->data + nhoff);
1087 if (ip_is_fragment(ip))
1088 return -EPROTONOSUPPORT;
1089 EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4);
1090 ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
1091
1092 efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
1093 rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
1094 ip->daddr, ports[1], ip->saddr, ports[0]);
1095 if (rc)
1096 return rc;
1097
1098 rc = efx_filter_insert_filter(efx, &spec, true);
1099 if (rc < 0)
1100 return rc;
1101
1102 /* Remember this so we can check whether to expire the filter later */
1103 state->rps_flow_id[rc] = flow_id;
1104 channel = efx_get_channel(efx, skb_get_rx_queue(skb));
1105 ++channel->rfs_filters_added;
1106
1107 netif_info(efx, rx_status, efx->net_dev,
1108 "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
1109 (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
1110 &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
1111 rxq_index, flow_id, rc);
1112
1113 return rc;
1114 }
1115
1116 bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
1117 {
1118 struct efx_filter_state *state = efx->filter_state;
1119 struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
1120 unsigned mask = table->size - 1;
1121 unsigned index;
1122 unsigned stop;
1123
1124 if (!spin_trylock_bh(&state->lock))
1125 return false;
1126
1127 index = state->rps_expire_index;
1128 stop = (index + quota) & mask;
1129
1130 while (index != stop) {
1131 if (test_bit(index, table->used_bitmap) &&
1132 table->spec[index].priority == EFX_FILTER_PRI_HINT &&
1133 rps_may_expire_flow(efx->net_dev,
1134 table->spec[index].dmaq_id,
1135 state->rps_flow_id[index], index)) {
1136 netif_info(efx, rx_status, efx->net_dev,
1137 "expiring filter %d [flow %u]\n",
1138 index, state->rps_flow_id[index]);
1139 efx_filter_table_clear_entry(efx, table, index);
1140 }
1141 index = (index + 1) & mask;
1142 }
1143
1144 state->rps_expire_index = stop;
1145 if (table->used == 0)
1146 efx_filter_table_reset_search_depth(table);
1147
1148 spin_unlock_bh(&state->lock);
1149 return true;
1150 }
1151
1152 #endif /* CONFIG_RFS_ACCEL */
kernel source for telekom puls (alcatel/mediatek)