1 /****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2010 Solarflare Communications Inc.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
18 /* "Fudge factors" - difference between programmed value and actual depth.
19 * Due to pipelined implementation we need to program H/W with a value that
20 * is larger than the hop limit we want.
22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
25 /* Hard maximum hop limit. Hardware will time-out beyond 200-something.
26 * We also need to avoid infinite loops in efx_filter_search() when the
29 #define FILTER_CTL_SRCH_MAX 200
31 /* Don't try very hard to find space for performance hints, as this is
32 * counter-productive. */
33 #define FILTER_CTL_SRCH_HINT_MAX 5
35 enum efx_filter_table_id
{
36 EFX_FILTER_TABLE_RX_IP
= 0,
37 EFX_FILTER_TABLE_RX_MAC
,
38 EFX_FILTER_TABLE_RX_DEF
,
39 EFX_FILTER_TABLE_TX_MAC
,
40 EFX_FILTER_TABLE_COUNT
,
43 enum efx_filter_index
{
44 EFX_FILTER_INDEX_UC_DEF
,
45 EFX_FILTER_INDEX_MC_DEF
,
46 EFX_FILTER_SIZE_RX_DEF
,
49 struct efx_filter_table
{
50 enum efx_filter_table_id id
;
51 u32 offset
; /* address of table relative to BAR */
52 unsigned size
; /* number of entries */
53 unsigned step
; /* step between entries */
54 unsigned used
; /* number currently used */
55 unsigned long *used_bitmap
;
56 struct efx_filter_spec
*spec
;
57 unsigned search_depth
[EFX_FILTER_TYPE_COUNT
];
60 struct efx_filter_state
{
62 struct efx_filter_table table
[EFX_FILTER_TABLE_COUNT
];
63 #ifdef CONFIG_RFS_ACCEL
65 unsigned rps_expire_index
;
69 static void efx_filter_table_clear_entry(struct efx_nic
*efx
,
70 struct efx_filter_table
*table
,
71 unsigned int filter_idx
);
73 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
74 * key derived from the n-tuple. The initial LFSR state is 0xffff. */
75 static u16
efx_filter_hash(u32 key
)
80 tmp
= 0x1fff ^ key
>> 16;
81 tmp
= tmp
^ tmp
>> 3 ^ tmp
>> 6;
84 tmp
= tmp
^ tmp
<< 13 ^ key
;
85 tmp
= tmp
^ tmp
>> 3 ^ tmp
>> 6;
86 return tmp
^ tmp
>> 9;
89 /* To allow for hash collisions, filter search continues at these
90 * increments from the first possible entry selected by the hash. */
91 static u16
efx_filter_increment(u32 key
)
96 static enum efx_filter_table_id
97 efx_filter_spec_table_id(const struct efx_filter_spec
*spec
)
99 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP
!= (EFX_FILTER_TCP_FULL
>> 2));
100 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP
!= (EFX_FILTER_TCP_WILD
>> 2));
101 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP
!= (EFX_FILTER_UDP_FULL
>> 2));
102 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP
!= (EFX_FILTER_UDP_WILD
>> 2));
103 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC
!= (EFX_FILTER_MAC_FULL
>> 2));
104 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC
!= (EFX_FILTER_MAC_WILD
>> 2));
105 BUILD_BUG_ON(EFX_FILTER_TABLE_TX_MAC
!= EFX_FILTER_TABLE_RX_MAC
+ 2);
106 EFX_BUG_ON_PARANOID(spec
->type
== EFX_FILTER_UNSPEC
);
107 return (spec
->type
>> 2) + ((spec
->flags
& EFX_FILTER_FLAG_TX
) ? 2 : 0);
110 static struct efx_filter_table
*
111 efx_filter_spec_table(struct efx_filter_state
*state
,
112 const struct efx_filter_spec
*spec
)
114 if (spec
->type
== EFX_FILTER_UNSPEC
)
117 return &state
->table
[efx_filter_spec_table_id(spec
)];
120 static void efx_filter_table_reset_search_depth(struct efx_filter_table
*table
)
122 memset(table
->search_depth
, 0, sizeof(table
->search_depth
));
125 static void efx_filter_push_rx_config(struct efx_nic
*efx
)
127 struct efx_filter_state
*state
= efx
->filter_state
;
128 struct efx_filter_table
*table
;
129 efx_oword_t filter_ctl
;
131 efx_reado(efx
, &filter_ctl
, FR_BZ_RX_FILTER_CTL
);
133 table
= &state
->table
[EFX_FILTER_TABLE_RX_IP
];
134 EFX_SET_OWORD_FIELD(filter_ctl
, FRF_BZ_TCP_FULL_SRCH_LIMIT
,
135 table
->search_depth
[EFX_FILTER_TCP_FULL
] +
136 FILTER_CTL_SRCH_FUDGE_FULL
);
137 EFX_SET_OWORD_FIELD(filter_ctl
, FRF_BZ_TCP_WILD_SRCH_LIMIT
,
138 table
->search_depth
[EFX_FILTER_TCP_WILD
] +
139 FILTER_CTL_SRCH_FUDGE_WILD
);
140 EFX_SET_OWORD_FIELD(filter_ctl
, FRF_BZ_UDP_FULL_SRCH_LIMIT
,
141 table
->search_depth
[EFX_FILTER_UDP_FULL
] +
142 FILTER_CTL_SRCH_FUDGE_FULL
);
143 EFX_SET_OWORD_FIELD(filter_ctl
, FRF_BZ_UDP_WILD_SRCH_LIMIT
,
144 table
->search_depth
[EFX_FILTER_UDP_WILD
] +
145 FILTER_CTL_SRCH_FUDGE_WILD
);
147 table
= &state
->table
[EFX_FILTER_TABLE_RX_MAC
];
150 filter_ctl
, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT
,
151 table
->search_depth
[EFX_FILTER_MAC_FULL
] +
152 FILTER_CTL_SRCH_FUDGE_FULL
);
154 filter_ctl
, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT
,
155 table
->search_depth
[EFX_FILTER_MAC_WILD
] +
156 FILTER_CTL_SRCH_FUDGE_WILD
);
159 table
= &state
->table
[EFX_FILTER_TABLE_RX_DEF
];
162 filter_ctl
, FRF_CZ_UNICAST_NOMATCH_Q_ID
,
163 table
->spec
[EFX_FILTER_INDEX_UC_DEF
].dmaq_id
);
165 filter_ctl
, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED
,
166 !!(table
->spec
[EFX_FILTER_INDEX_UC_DEF
].flags
&
167 EFX_FILTER_FLAG_RX_RSS
));
169 filter_ctl
, FRF_CZ_MULTICAST_NOMATCH_Q_ID
,
170 table
->spec
[EFX_FILTER_INDEX_MC_DEF
].dmaq_id
);
172 filter_ctl
, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED
,
173 !!(table
->spec
[EFX_FILTER_INDEX_MC_DEF
].flags
&
174 EFX_FILTER_FLAG_RX_RSS
));
176 /* There is a single bit to enable RX scatter for all
177 * unmatched packets. Only set it if scatter is
178 * enabled in both filter specs.
181 filter_ctl
, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q
,
182 !!(table
->spec
[EFX_FILTER_INDEX_UC_DEF
].flags
&
183 table
->spec
[EFX_FILTER_INDEX_MC_DEF
].flags
&
184 EFX_FILTER_FLAG_RX_SCATTER
));
185 } else if (efx_nic_rev(efx
) >= EFX_REV_FALCON_B0
) {
186 /* We don't expose 'default' filters because unmatched
187 * packets always go to the queue number found in the
188 * RSS table. But we still need to set the RX scatter
192 filter_ctl
, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q
,
196 efx_writeo(efx
, &filter_ctl
, FR_BZ_RX_FILTER_CTL
);
199 static void efx_filter_push_tx_limits(struct efx_nic
*efx
)
201 struct efx_filter_state
*state
= efx
->filter_state
;
202 struct efx_filter_table
*table
;
205 efx_reado(efx
, &tx_cfg
, FR_AZ_TX_CFG
);
207 table
= &state
->table
[EFX_FILTER_TABLE_TX_MAC
];
210 tx_cfg
, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE
,
211 table
->search_depth
[EFX_FILTER_MAC_FULL
] +
212 FILTER_CTL_SRCH_FUDGE_FULL
);
214 tx_cfg
, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE
,
215 table
->search_depth
[EFX_FILTER_MAC_WILD
] +
216 FILTER_CTL_SRCH_FUDGE_WILD
);
219 efx_writeo(efx
, &tx_cfg
, FR_AZ_TX_CFG
);
222 static inline void __efx_filter_set_ipv4(struct efx_filter_spec
*spec
,
223 __be32 host1
, __be16 port1
,
224 __be32 host2
, __be16 port2
)
226 spec
->data
[0] = ntohl(host1
) << 16 | ntohs(port1
);
227 spec
->data
[1] = ntohs(port2
) << 16 | ntohl(host1
) >> 16;
228 spec
->data
[2] = ntohl(host2
);
231 static inline void __efx_filter_get_ipv4(const struct efx_filter_spec
*spec
,
232 __be32
*host1
, __be16
*port1
,
233 __be32
*host2
, __be16
*port2
)
235 *host1
= htonl(spec
->data
[0] >> 16 | spec
->data
[1] << 16);
236 *port1
= htons(spec
->data
[0]);
237 *host2
= htonl(spec
->data
[2]);
238 *port2
= htons(spec
->data
[1] >> 16);
242 * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
243 * @spec: Specification to initialise
244 * @proto: Transport layer protocol number
245 * @host: Local host address (network byte order)
246 * @port: Local port (network byte order)
248 int efx_filter_set_ipv4_local(struct efx_filter_spec
*spec
, u8 proto
,
249 __be32 host
, __be16 port
)
254 EFX_BUG_ON_PARANOID(!(spec
->flags
& EFX_FILTER_FLAG_RX
));
256 /* This cannot currently be combined with other filtering */
257 if (spec
->type
!= EFX_FILTER_UNSPEC
)
258 return -EPROTONOSUPPORT
;
265 spec
->type
= EFX_FILTER_TCP_WILD
;
268 spec
->type
= EFX_FILTER_UDP_WILD
;
271 return -EPROTONOSUPPORT
;
274 /* Filter is constructed in terms of source and destination,
275 * with the odd wrinkle that the ports are swapped in a UDP
276 * wildcard filter. We need to convert from local and remote
277 * (= zero for wildcard) addresses.
280 if (proto
!= IPPROTO_UDP
) {
287 __efx_filter_set_ipv4(spec
, host1
, port1
, host
, port
);
291 int efx_filter_get_ipv4_local(const struct efx_filter_spec
*spec
,
292 u8
*proto
, __be32
*host
, __be16
*port
)
297 switch (spec
->type
) {
298 case EFX_FILTER_TCP_WILD
:
299 *proto
= IPPROTO_TCP
;
300 __efx_filter_get_ipv4(spec
, &host1
, &port1
, host
, port
);
302 case EFX_FILTER_UDP_WILD
:
303 *proto
= IPPROTO_UDP
;
304 __efx_filter_get_ipv4(spec
, &host1
, port
, host
, &port1
);
312 * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
313 * @spec: Specification to initialise
314 * @proto: Transport layer protocol number
315 * @host: Local host address (network byte order)
316 * @port: Local port (network byte order)
317 * @rhost: Remote host address (network byte order)
318 * @rport: Remote port (network byte order)
320 int efx_filter_set_ipv4_full(struct efx_filter_spec
*spec
, u8 proto
,
321 __be32 host
, __be16 port
,
322 __be32 rhost
, __be16 rport
)
324 EFX_BUG_ON_PARANOID(!(spec
->flags
& EFX_FILTER_FLAG_RX
));
326 /* This cannot currently be combined with other filtering */
327 if (spec
->type
!= EFX_FILTER_UNSPEC
)
328 return -EPROTONOSUPPORT
;
330 if (port
== 0 || rport
== 0)
335 spec
->type
= EFX_FILTER_TCP_FULL
;
338 spec
->type
= EFX_FILTER_UDP_FULL
;
341 return -EPROTONOSUPPORT
;
344 __efx_filter_set_ipv4(spec
, rhost
, rport
, host
, port
);
348 int efx_filter_get_ipv4_full(const struct efx_filter_spec
*spec
,
349 u8
*proto
, __be32
*host
, __be16
*port
,
350 __be32
*rhost
, __be16
*rport
)
352 switch (spec
->type
) {
353 case EFX_FILTER_TCP_FULL
:
354 *proto
= IPPROTO_TCP
;
356 case EFX_FILTER_UDP_FULL
:
357 *proto
= IPPROTO_UDP
;
363 __efx_filter_get_ipv4(spec
, rhost
, rport
, host
, port
);
368 * efx_filter_set_eth_local - specify local Ethernet address and optional VID
369 * @spec: Specification to initialise
370 * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
371 * @addr: Local Ethernet MAC address
373 int efx_filter_set_eth_local(struct efx_filter_spec
*spec
,
374 u16 vid
, const u8
*addr
)
376 EFX_BUG_ON_PARANOID(!(spec
->flags
&
377 (EFX_FILTER_FLAG_RX
| EFX_FILTER_FLAG_TX
)));
379 /* This cannot currently be combined with other filtering */
380 if (spec
->type
!= EFX_FILTER_UNSPEC
)
381 return -EPROTONOSUPPORT
;
383 if (vid
== EFX_FILTER_VID_UNSPEC
) {
384 spec
->type
= EFX_FILTER_MAC_WILD
;
387 spec
->type
= EFX_FILTER_MAC_FULL
;
391 spec
->data
[1] = addr
[2] << 24 | addr
[3] << 16 | addr
[4] << 8 | addr
[5];
392 spec
->data
[2] = addr
[0] << 8 | addr
[1];
397 * efx_filter_set_uc_def - specify matching otherwise-unmatched unicast
398 * @spec: Specification to initialise
400 int efx_filter_set_uc_def(struct efx_filter_spec
*spec
)
402 EFX_BUG_ON_PARANOID(!(spec
->flags
&
403 (EFX_FILTER_FLAG_RX
| EFX_FILTER_FLAG_TX
)));
405 if (spec
->type
!= EFX_FILTER_UNSPEC
)
408 spec
->type
= EFX_FILTER_UC_DEF
;
409 memset(spec
->data
, 0, sizeof(spec
->data
)); /* ensure equality */
414 * efx_filter_set_mc_def - specify matching otherwise-unmatched multicast
415 * @spec: Specification to initialise
417 int efx_filter_set_mc_def(struct efx_filter_spec
*spec
)
419 EFX_BUG_ON_PARANOID(!(spec
->flags
&
420 (EFX_FILTER_FLAG_RX
| EFX_FILTER_FLAG_TX
)));
422 if (spec
->type
!= EFX_FILTER_UNSPEC
)
425 spec
->type
= EFX_FILTER_MC_DEF
;
426 memset(spec
->data
, 0, sizeof(spec
->data
)); /* ensure equality */
430 static void efx_filter_reset_rx_def(struct efx_nic
*efx
, unsigned filter_idx
)
432 struct efx_filter_state
*state
= efx
->filter_state
;
433 struct efx_filter_table
*table
= &state
->table
[EFX_FILTER_TABLE_RX_DEF
];
434 struct efx_filter_spec
*spec
= &table
->spec
[filter_idx
];
435 enum efx_filter_flags flags
= 0;
437 /* If there's only one channel then disable RSS for non VF
438 * traffic, thereby allowing VFs to use RSS when the PF can't.
440 if (efx
->n_rx_channels
> 1)
441 flags
|= EFX_FILTER_FLAG_RX_RSS
;
444 flags
|= EFX_FILTER_FLAG_RX_SCATTER
;
446 efx_filter_init_rx(spec
, EFX_FILTER_PRI_MANUAL
, flags
, 0);
447 spec
->type
= EFX_FILTER_UC_DEF
+ filter_idx
;
448 table
->used_bitmap
[0] |= 1 << filter_idx
;
451 int efx_filter_get_eth_local(const struct efx_filter_spec
*spec
,
454 switch (spec
->type
) {
455 case EFX_FILTER_MAC_WILD
:
456 *vid
= EFX_FILTER_VID_UNSPEC
;
458 case EFX_FILTER_MAC_FULL
:
459 *vid
= spec
->data
[0];
465 addr
[0] = spec
->data
[2] >> 8;
466 addr
[1] = spec
->data
[2];
467 addr
[2] = spec
->data
[1] >> 24;
468 addr
[3] = spec
->data
[1] >> 16;
469 addr
[4] = spec
->data
[1] >> 8;
470 addr
[5] = spec
->data
[1];
474 /* Build a filter entry and return its n-tuple key. */
475 static u32
efx_filter_build(efx_oword_t
*filter
, struct efx_filter_spec
*spec
)
479 switch (efx_filter_spec_table_id(spec
)) {
480 case EFX_FILTER_TABLE_RX_IP
: {
481 bool is_udp
= (spec
->type
== EFX_FILTER_UDP_FULL
||
482 spec
->type
== EFX_FILTER_UDP_WILD
);
483 EFX_POPULATE_OWORD_7(
486 !!(spec
->flags
& EFX_FILTER_FLAG_RX_RSS
),
488 !!(spec
->flags
& EFX_FILTER_FLAG_RX_SCATTER
),
489 FRF_BZ_TCP_UDP
, is_udp
,
490 FRF_BZ_RXQ_ID
, spec
->dmaq_id
,
491 EFX_DWORD_2
, spec
->data
[2],
492 EFX_DWORD_1
, spec
->data
[1],
493 EFX_DWORD_0
, spec
->data
[0]);
498 case EFX_FILTER_TABLE_RX_MAC
: {
499 bool is_wild
= spec
->type
== EFX_FILTER_MAC_WILD
;
500 EFX_POPULATE_OWORD_7(
503 !!(spec
->flags
& EFX_FILTER_FLAG_RX_RSS
),
504 FRF_CZ_RMFT_SCATTER_EN
,
505 !!(spec
->flags
& EFX_FILTER_FLAG_RX_SCATTER
),
506 FRF_CZ_RMFT_RXQ_ID
, spec
->dmaq_id
,
507 FRF_CZ_RMFT_WILDCARD_MATCH
, is_wild
,
508 FRF_CZ_RMFT_DEST_MAC_HI
, spec
->data
[2],
509 FRF_CZ_RMFT_DEST_MAC_LO
, spec
->data
[1],
510 FRF_CZ_RMFT_VLAN_ID
, spec
->data
[0]);
515 case EFX_FILTER_TABLE_TX_MAC
: {
516 bool is_wild
= spec
->type
== EFX_FILTER_MAC_WILD
;
517 EFX_POPULATE_OWORD_5(*filter
,
518 FRF_CZ_TMFT_TXQ_ID
, spec
->dmaq_id
,
519 FRF_CZ_TMFT_WILDCARD_MATCH
, is_wild
,
520 FRF_CZ_TMFT_SRC_MAC_HI
, spec
->data
[2],
521 FRF_CZ_TMFT_SRC_MAC_LO
, spec
->data
[1],
522 FRF_CZ_TMFT_VLAN_ID
, spec
->data
[0]);
523 data3
= is_wild
| spec
->dmaq_id
<< 1;
531 return spec
->data
[0] ^ spec
->data
[1] ^ spec
->data
[2] ^ data3
;
534 static bool efx_filter_equal(const struct efx_filter_spec
*left
,
535 const struct efx_filter_spec
*right
)
537 if (left
->type
!= right
->type
||
538 memcmp(left
->data
, right
->data
, sizeof(left
->data
)))
541 if (left
->flags
& EFX_FILTER_FLAG_TX
&&
542 left
->dmaq_id
!= right
->dmaq_id
)
549 * Construct/deconstruct external filter IDs. At least the RX filter
550 * IDs must be ordered by matching priority, for RX NFC semantics.
552 * Deconstruction needs to be robust against invalid IDs so that
553 * efx_filter_remove_id_safe() and efx_filter_get_filter_safe() can
554 * accept user-provided IDs.
557 #define EFX_FILTER_MATCH_PRI_COUNT 5
559 static const u8 efx_filter_type_match_pri
[EFX_FILTER_TYPE_COUNT
] = {
560 [EFX_FILTER_TCP_FULL
] = 0,
561 [EFX_FILTER_UDP_FULL
] = 0,
562 [EFX_FILTER_TCP_WILD
] = 1,
563 [EFX_FILTER_UDP_WILD
] = 1,
564 [EFX_FILTER_MAC_FULL
] = 2,
565 [EFX_FILTER_MAC_WILD
] = 3,
566 [EFX_FILTER_UC_DEF
] = 4,
567 [EFX_FILTER_MC_DEF
] = 4,
570 static const enum efx_filter_table_id efx_filter_range_table
[] = {
571 EFX_FILTER_TABLE_RX_IP
, /* RX match pri 0 */
572 EFX_FILTER_TABLE_RX_IP
,
573 EFX_FILTER_TABLE_RX_MAC
,
574 EFX_FILTER_TABLE_RX_MAC
,
575 EFX_FILTER_TABLE_RX_DEF
, /* RX match pri 4 */
576 EFX_FILTER_TABLE_COUNT
, /* TX match pri 0; invalid */
577 EFX_FILTER_TABLE_COUNT
, /* invalid */
578 EFX_FILTER_TABLE_TX_MAC
,
579 EFX_FILTER_TABLE_TX_MAC
, /* TX match pri 3 */
582 #define EFX_FILTER_INDEX_WIDTH 13
583 #define EFX_FILTER_INDEX_MASK ((1 << EFX_FILTER_INDEX_WIDTH) - 1)
586 efx_filter_make_id(const struct efx_filter_spec
*spec
, unsigned int index
)
590 range
= efx_filter_type_match_pri
[spec
->type
];
591 if (!(spec
->flags
& EFX_FILTER_FLAG_RX
))
592 range
+= EFX_FILTER_MATCH_PRI_COUNT
;
594 return range
<< EFX_FILTER_INDEX_WIDTH
| index
;
597 static inline enum efx_filter_table_id
efx_filter_id_table_id(u32 id
)
599 unsigned int range
= id
>> EFX_FILTER_INDEX_WIDTH
;
601 if (range
< ARRAY_SIZE(efx_filter_range_table
))
602 return efx_filter_range_table
[range
];
604 return EFX_FILTER_TABLE_COUNT
; /* invalid */
607 static inline unsigned int efx_filter_id_index(u32 id
)
609 return id
& EFX_FILTER_INDEX_MASK
;
612 static inline u8
efx_filter_id_flags(u32 id
)
614 unsigned int range
= id
>> EFX_FILTER_INDEX_WIDTH
;
616 if (range
< EFX_FILTER_MATCH_PRI_COUNT
)
617 return EFX_FILTER_FLAG_RX
;
619 return EFX_FILTER_FLAG_TX
;
622 u32
efx_filter_get_rx_id_limit(struct efx_nic
*efx
)
624 struct efx_filter_state
*state
= efx
->filter_state
;
625 unsigned int range
= EFX_FILTER_MATCH_PRI_COUNT
- 1;
626 enum efx_filter_table_id table_id
;
629 table_id
= efx_filter_range_table
[range
];
630 if (state
->table
[table_id
].size
!= 0)
631 return range
<< EFX_FILTER_INDEX_WIDTH
|
632 state
->table
[table_id
].size
;
639 * efx_filter_insert_filter - add or replace a filter
640 * @efx: NIC in which to insert the filter
641 * @spec: Specification for the filter
642 * @replace_equal: Flag for whether the specified filter may replace an
643 * existing filter with equal priority
645 * On success, return the filter ID.
646 * On failure, return a negative error code.
648 * If an existing filter has equal match values to the new filter
649 * spec, then the new filter might replace it, depending on the
650 * relative priorities. If the existing filter has lower priority, or
651 * if @replace_equal is set and it has equal priority, then it is
652 * replaced. Otherwise the function fails, returning -%EPERM if
653 * the existing filter has higher priority or -%EEXIST if it has
656 s32
efx_filter_insert_filter(struct efx_nic
*efx
, struct efx_filter_spec
*spec
,
659 struct efx_filter_state
*state
= efx
->filter_state
;
660 struct efx_filter_table
*table
= efx_filter_spec_table(state
, spec
);
662 int rep_index
, ins_index
;
663 unsigned int depth
= 0;
666 if (!table
|| table
->size
== 0)
669 netif_vdbg(efx
, hw
, efx
->net_dev
,
670 "%s: type %d search_depth=%d", __func__
, spec
->type
,
671 table
->search_depth
[spec
->type
]);
673 if (table
->id
== EFX_FILTER_TABLE_RX_DEF
) {
674 /* One filter spec per type */
675 BUILD_BUG_ON(EFX_FILTER_INDEX_UC_DEF
!= 0);
676 BUILD_BUG_ON(EFX_FILTER_INDEX_MC_DEF
!=
677 EFX_FILTER_MC_DEF
- EFX_FILTER_UC_DEF
);
678 rep_index
= spec
->type
- EFX_FILTER_UC_DEF
;
679 ins_index
= rep_index
;
681 spin_lock_bh(&state
->lock
);
683 /* Search concurrently for
684 * (1) a filter to be replaced (rep_index): any filter
685 * with the same match values, up to the current
686 * search depth for this type, and
687 * (2) the insertion point (ins_index): (1) or any
688 * free slot before it or up to the maximum search
689 * depth for this priority
690 * We fail if we cannot find (2).
692 * We can stop once either
693 * (a) we find (1), in which case we have definitely
694 * found (2) as well; or
695 * (b) we have searched exhaustively for (1), and have
696 * either found (2) or searched exhaustively for it
698 u32 key
= efx_filter_build(&filter
, spec
);
699 unsigned int hash
= efx_filter_hash(key
);
700 unsigned int incr
= efx_filter_increment(key
);
701 unsigned int max_rep_depth
= table
->search_depth
[spec
->type
];
702 unsigned int max_ins_depth
=
703 spec
->priority
<= EFX_FILTER_PRI_HINT
?
704 FILTER_CTL_SRCH_HINT_MAX
: FILTER_CTL_SRCH_MAX
;
705 unsigned int i
= hash
& (table
->size
- 1);
710 spin_lock_bh(&state
->lock
);
713 if (!test_bit(i
, table
->used_bitmap
)) {
716 } else if (efx_filter_equal(spec
, &table
->spec
[i
])) {
724 if (depth
>= max_rep_depth
&&
725 (ins_index
>= 0 || depth
>= max_ins_depth
)) {
735 i
= (i
+ incr
) & (table
->size
- 1);
740 /* If we found a filter to be replaced, check whether we
743 if (rep_index
>= 0) {
744 struct efx_filter_spec
*saved_spec
= &table
->spec
[rep_index
];
746 if (spec
->priority
== saved_spec
->priority
&& !replace_equal
) {
750 if (spec
->priority
< saved_spec
->priority
) {
756 /* Insert the filter */
757 if (ins_index
!= rep_index
) {
758 __set_bit(ins_index
, table
->used_bitmap
);
761 table
->spec
[ins_index
] = *spec
;
763 if (table
->id
== EFX_FILTER_TABLE_RX_DEF
) {
764 efx_filter_push_rx_config(efx
);
766 if (table
->search_depth
[spec
->type
] < depth
) {
767 table
->search_depth
[spec
->type
] = depth
;
768 if (spec
->flags
& EFX_FILTER_FLAG_TX
)
769 efx_filter_push_tx_limits(efx
);
771 efx_filter_push_rx_config(efx
);
774 efx_writeo(efx
, &filter
,
775 table
->offset
+ table
->step
* ins_index
);
777 /* If we were able to replace a filter by inserting
778 * at a lower depth, clear the replaced filter
780 if (ins_index
!= rep_index
&& rep_index
>= 0)
781 efx_filter_table_clear_entry(efx
, table
, rep_index
);
784 netif_vdbg(efx
, hw
, efx
->net_dev
,
785 "%s: filter type %d index %d rxq %u set",
786 __func__
, spec
->type
, ins_index
, spec
->dmaq_id
);
787 rc
= efx_filter_make_id(spec
, ins_index
);
790 spin_unlock_bh(&state
->lock
);
794 static void efx_filter_table_clear_entry(struct efx_nic
*efx
,
795 struct efx_filter_table
*table
,
796 unsigned int filter_idx
)
798 static efx_oword_t filter
;
800 if (table
->id
== EFX_FILTER_TABLE_RX_DEF
) {
801 /* RX default filters must always exist */
802 efx_filter_reset_rx_def(efx
, filter_idx
);
803 efx_filter_push_rx_config(efx
);
804 } else if (test_bit(filter_idx
, table
->used_bitmap
)) {
805 __clear_bit(filter_idx
, table
->used_bitmap
);
807 memset(&table
->spec
[filter_idx
], 0, sizeof(table
->spec
[0]));
809 efx_writeo(efx
, &filter
,
810 table
->offset
+ table
->step
* filter_idx
);
815 * efx_filter_remove_id_safe - remove a filter by ID, carefully
816 * @efx: NIC from which to remove the filter
817 * @priority: Priority of filter, as passed to @efx_filter_insert_filter
818 * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
820 * This function will range-check @filter_id, so it is safe to call
821 * with a value passed from userland.
823 int efx_filter_remove_id_safe(struct efx_nic
*efx
,
824 enum efx_filter_priority priority
,
827 struct efx_filter_state
*state
= efx
->filter_state
;
828 enum efx_filter_table_id table_id
;
829 struct efx_filter_table
*table
;
830 unsigned int filter_idx
;
831 struct efx_filter_spec
*spec
;
835 table_id
= efx_filter_id_table_id(filter_id
);
836 if ((unsigned int)table_id
>= EFX_FILTER_TABLE_COUNT
)
838 table
= &state
->table
[table_id
];
840 filter_idx
= efx_filter_id_index(filter_id
);
841 if (filter_idx
>= table
->size
)
843 spec
= &table
->spec
[filter_idx
];
845 filter_flags
= efx_filter_id_flags(filter_id
);
847 spin_lock_bh(&state
->lock
);
849 if (test_bit(filter_idx
, table
->used_bitmap
) &&
850 spec
->priority
== priority
) {
851 efx_filter_table_clear_entry(efx
, table
, filter_idx
);
852 if (table
->used
== 0)
853 efx_filter_table_reset_search_depth(table
);
859 spin_unlock_bh(&state
->lock
);
865 * efx_filter_get_filter_safe - retrieve a filter by ID, carefully
866 * @efx: NIC from which to remove the filter
867 * @priority: Priority of filter, as passed to @efx_filter_insert_filter
868 * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
869 * @spec: Buffer in which to store filter specification
871 * This function will range-check @filter_id, so it is safe to call
872 * with a value passed from userland.
874 int efx_filter_get_filter_safe(struct efx_nic
*efx
,
875 enum efx_filter_priority priority
,
876 u32 filter_id
, struct efx_filter_spec
*spec_buf
)
878 struct efx_filter_state
*state
= efx
->filter_state
;
879 enum efx_filter_table_id table_id
;
880 struct efx_filter_table
*table
;
881 struct efx_filter_spec
*spec
;
882 unsigned int filter_idx
;
886 table_id
= efx_filter_id_table_id(filter_id
);
887 if ((unsigned int)table_id
>= EFX_FILTER_TABLE_COUNT
)
889 table
= &state
->table
[table_id
];
891 filter_idx
= efx_filter_id_index(filter_id
);
892 if (filter_idx
>= table
->size
)
894 spec
= &table
->spec
[filter_idx
];
896 filter_flags
= efx_filter_id_flags(filter_id
);
898 spin_lock_bh(&state
->lock
);
900 if (test_bit(filter_idx
, table
->used_bitmap
) &&
901 spec
->priority
== priority
) {
908 spin_unlock_bh(&state
->lock
);
913 static void efx_filter_table_clear(struct efx_nic
*efx
,
914 enum efx_filter_table_id table_id
,
915 enum efx_filter_priority priority
)
917 struct efx_filter_state
*state
= efx
->filter_state
;
918 struct efx_filter_table
*table
= &state
->table
[table_id
];
919 unsigned int filter_idx
;
921 spin_lock_bh(&state
->lock
);
923 for (filter_idx
= 0; filter_idx
< table
->size
; ++filter_idx
)
924 if (table
->spec
[filter_idx
].priority
<= priority
)
925 efx_filter_table_clear_entry(efx
, table
, filter_idx
);
926 if (table
->used
== 0)
927 efx_filter_table_reset_search_depth(table
);
929 spin_unlock_bh(&state
->lock
);
933 * efx_filter_clear_rx - remove RX filters by priority
934 * @efx: NIC from which to remove the filters
935 * @priority: Maximum priority to remove
937 void efx_filter_clear_rx(struct efx_nic
*efx
, enum efx_filter_priority priority
)
939 efx_filter_table_clear(efx
, EFX_FILTER_TABLE_RX_IP
, priority
);
940 efx_filter_table_clear(efx
, EFX_FILTER_TABLE_RX_MAC
, priority
);
943 u32
efx_filter_count_rx_used(struct efx_nic
*efx
,
944 enum efx_filter_priority priority
)
946 struct efx_filter_state
*state
= efx
->filter_state
;
947 enum efx_filter_table_id table_id
;
948 struct efx_filter_table
*table
;
949 unsigned int filter_idx
;
952 spin_lock_bh(&state
->lock
);
954 for (table_id
= EFX_FILTER_TABLE_RX_IP
;
955 table_id
<= EFX_FILTER_TABLE_RX_DEF
;
957 table
= &state
->table
[table_id
];
958 for (filter_idx
= 0; filter_idx
< table
->size
; filter_idx
++) {
959 if (test_bit(filter_idx
, table
->used_bitmap
) &&
960 table
->spec
[filter_idx
].priority
== priority
)
965 spin_unlock_bh(&state
->lock
);
970 s32
efx_filter_get_rx_ids(struct efx_nic
*efx
,
971 enum efx_filter_priority priority
,
974 struct efx_filter_state
*state
= efx
->filter_state
;
975 enum efx_filter_table_id table_id
;
976 struct efx_filter_table
*table
;
977 unsigned int filter_idx
;
980 spin_lock_bh(&state
->lock
);
982 for (table_id
= EFX_FILTER_TABLE_RX_IP
;
983 table_id
<= EFX_FILTER_TABLE_RX_DEF
;
985 table
= &state
->table
[table_id
];
986 for (filter_idx
= 0; filter_idx
< table
->size
; filter_idx
++) {
987 if (test_bit(filter_idx
, table
->used_bitmap
) &&
988 table
->spec
[filter_idx
].priority
== priority
) {
993 buf
[count
++] = efx_filter_make_id(
994 &table
->spec
[filter_idx
], filter_idx
);
999 spin_unlock_bh(&state
->lock
);
1004 /* Restore filter stater after reset */
1005 void efx_restore_filters(struct efx_nic
*efx
)
1007 struct efx_filter_state
*state
= efx
->filter_state
;
1008 enum efx_filter_table_id table_id
;
1009 struct efx_filter_table
*table
;
1011 unsigned int filter_idx
;
1013 spin_lock_bh(&state
->lock
);
1015 for (table_id
= 0; table_id
< EFX_FILTER_TABLE_COUNT
; table_id
++) {
1016 table
= &state
->table
[table_id
];
1018 /* Check whether this is a regular register table */
1019 if (table
->step
== 0)
1022 for (filter_idx
= 0; filter_idx
< table
->size
; filter_idx
++) {
1023 if (!test_bit(filter_idx
, table
->used_bitmap
))
1025 efx_filter_build(&filter
, &table
->spec
[filter_idx
]);
1026 efx_writeo(efx
, &filter
,
1027 table
->offset
+ table
->step
* filter_idx
);
1031 efx_filter_push_rx_config(efx
);
1032 efx_filter_push_tx_limits(efx
);
1034 spin_unlock_bh(&state
->lock
);
1037 int efx_probe_filters(struct efx_nic
*efx
)
1039 struct efx_filter_state
*state
;
1040 struct efx_filter_table
*table
;
1043 state
= kzalloc(sizeof(*efx
->filter_state
), GFP_KERNEL
);
1046 efx
->filter_state
= state
;
1048 spin_lock_init(&state
->lock
);
1050 if (efx_nic_rev(efx
) >= EFX_REV_FALCON_B0
) {
1051 #ifdef CONFIG_RFS_ACCEL
1052 state
->rps_flow_id
= kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS
,
1053 sizeof(*state
->rps_flow_id
),
1055 if (!state
->rps_flow_id
)
1058 table
= &state
->table
[EFX_FILTER_TABLE_RX_IP
];
1059 table
->id
= EFX_FILTER_TABLE_RX_IP
;
1060 table
->offset
= FR_BZ_RX_FILTER_TBL0
;
1061 table
->size
= FR_BZ_RX_FILTER_TBL0_ROWS
;
1062 table
->step
= FR_BZ_RX_FILTER_TBL0_STEP
;
1065 if (efx_nic_rev(efx
) >= EFX_REV_SIENA_A0
) {
1066 table
= &state
->table
[EFX_FILTER_TABLE_RX_MAC
];
1067 table
->id
= EFX_FILTER_TABLE_RX_MAC
;
1068 table
->offset
= FR_CZ_RX_MAC_FILTER_TBL0
;
1069 table
->size
= FR_CZ_RX_MAC_FILTER_TBL0_ROWS
;
1070 table
->step
= FR_CZ_RX_MAC_FILTER_TBL0_STEP
;
1072 table
= &state
->table
[EFX_FILTER_TABLE_RX_DEF
];
1073 table
->id
= EFX_FILTER_TABLE_RX_DEF
;
1074 table
->size
= EFX_FILTER_SIZE_RX_DEF
;
1076 table
= &state
->table
[EFX_FILTER_TABLE_TX_MAC
];
1077 table
->id
= EFX_FILTER_TABLE_TX_MAC
;
1078 table
->offset
= FR_CZ_TX_MAC_FILTER_TBL0
;
1079 table
->size
= FR_CZ_TX_MAC_FILTER_TBL0_ROWS
;
1080 table
->step
= FR_CZ_TX_MAC_FILTER_TBL0_STEP
;
1083 for (table_id
= 0; table_id
< EFX_FILTER_TABLE_COUNT
; table_id
++) {
1084 table
= &state
->table
[table_id
];
1085 if (table
->size
== 0)
1087 table
->used_bitmap
= kcalloc(BITS_TO_LONGS(table
->size
),
1088 sizeof(unsigned long),
1090 if (!table
->used_bitmap
)
1092 table
->spec
= vzalloc(table
->size
* sizeof(*table
->spec
));
1097 if (state
->table
[EFX_FILTER_TABLE_RX_DEF
].size
) {
1098 /* RX default filters must always exist */
1100 for (i
= 0; i
< EFX_FILTER_SIZE_RX_DEF
; i
++)
1101 efx_filter_reset_rx_def(efx
, i
);
1104 efx_filter_push_rx_config(efx
);
1109 efx_remove_filters(efx
);
1113 void efx_remove_filters(struct efx_nic
*efx
)
1115 struct efx_filter_state
*state
= efx
->filter_state
;
1116 enum efx_filter_table_id table_id
;
1118 for (table_id
= 0; table_id
< EFX_FILTER_TABLE_COUNT
; table_id
++) {
1119 kfree(state
->table
[table_id
].used_bitmap
);
1120 vfree(state
->table
[table_id
].spec
);
1122 #ifdef CONFIG_RFS_ACCEL
1123 kfree(state
->rps_flow_id
);
1128 /* Update scatter enable flags for filters pointing to our own RX queues */
1129 void efx_filter_update_rx_scatter(struct efx_nic
*efx
)
1131 struct efx_filter_state
*state
= efx
->filter_state
;
1132 enum efx_filter_table_id table_id
;
1133 struct efx_filter_table
*table
;
1135 unsigned int filter_idx
;
1137 spin_lock_bh(&state
->lock
);
1139 for (table_id
= EFX_FILTER_TABLE_RX_IP
;
1140 table_id
<= EFX_FILTER_TABLE_RX_DEF
;
1142 table
= &state
->table
[table_id
];
1144 for (filter_idx
= 0; filter_idx
< table
->size
; filter_idx
++) {
1145 if (!test_bit(filter_idx
, table
->used_bitmap
) ||
1146 table
->spec
[filter_idx
].dmaq_id
>=
1150 if (efx
->rx_scatter
)
1151 table
->spec
[filter_idx
].flags
|=
1152 EFX_FILTER_FLAG_RX_SCATTER
;
1154 table
->spec
[filter_idx
].flags
&=
1155 ~EFX_FILTER_FLAG_RX_SCATTER
;
1157 if (table_id
== EFX_FILTER_TABLE_RX_DEF
)
1158 /* Pushed by efx_filter_push_rx_config() */
1161 efx_filter_build(&filter
, &table
->spec
[filter_idx
]);
1162 efx_writeo(efx
, &filter
,
1163 table
->offset
+ table
->step
* filter_idx
);
1167 efx_filter_push_rx_config(efx
);
1169 spin_unlock_bh(&state
->lock
);
1172 #ifdef CONFIG_RFS_ACCEL
1174 int efx_filter_rfs(struct net_device
*net_dev
, const struct sk_buff
*skb
,
1175 u16 rxq_index
, u32 flow_id
)
1177 struct efx_nic
*efx
= netdev_priv(net_dev
);
1178 struct efx_channel
*channel
;
1179 struct efx_filter_state
*state
= efx
->filter_state
;
1180 struct efx_filter_spec spec
;
1181 const struct iphdr
*ip
;
1182 const __be16
*ports
;
1186 nhoff
= skb_network_offset(skb
);
1188 if (skb
->protocol
!= htons(ETH_P_IP
))
1189 return -EPROTONOSUPPORT
;
1191 /* RFS must validate the IP header length before calling us */
1192 EFX_BUG_ON_PARANOID(skb_headlen(skb
) < nhoff
+ sizeof(*ip
));
1193 ip
= (const struct iphdr
*)(skb
->data
+ nhoff
);
1194 if (ip_is_fragment(ip
))
1195 return -EPROTONOSUPPORT
;
1196 EFX_BUG_ON_PARANOID(skb_headlen(skb
) < nhoff
+ 4 * ip
->ihl
+ 4);
1197 ports
= (const __be16
*)(skb
->data
+ nhoff
+ 4 * ip
->ihl
);
1199 efx_filter_init_rx(&spec
, EFX_FILTER_PRI_HINT
,
1200 efx
->rx_scatter
? EFX_FILTER_FLAG_RX_SCATTER
: 0,
1202 rc
= efx_filter_set_ipv4_full(&spec
, ip
->protocol
,
1203 ip
->daddr
, ports
[1], ip
->saddr
, ports
[0]);
1207 rc
= efx_filter_insert_filter(efx
, &spec
, true);
1211 /* Remember this so we can check whether to expire the filter later */
1212 state
->rps_flow_id
[rc
] = flow_id
;
1213 channel
= efx_get_channel(efx
, skb_get_rx_queue(skb
));
1214 ++channel
->rfs_filters_added
;
1216 netif_info(efx
, rx_status
, efx
->net_dev
,
1217 "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
1218 (ip
->protocol
== IPPROTO_TCP
) ? "TCP" : "UDP",
1219 &ip
->saddr
, ntohs(ports
[0]), &ip
->daddr
, ntohs(ports
[1]),
1220 rxq_index
, flow_id
, rc
);
1225 bool __efx_filter_rfs_expire(struct efx_nic
*efx
, unsigned quota
)
1227 struct efx_filter_state
*state
= efx
->filter_state
;
1228 struct efx_filter_table
*table
= &state
->table
[EFX_FILTER_TABLE_RX_IP
];
1229 unsigned mask
= table
->size
- 1;
1233 if (!spin_trylock_bh(&state
->lock
))
1236 index
= state
->rps_expire_index
;
1237 stop
= (index
+ quota
) & mask
;
1239 while (index
!= stop
) {
1240 if (test_bit(index
, table
->used_bitmap
) &&
1241 table
->spec
[index
].priority
== EFX_FILTER_PRI_HINT
&&
1242 rps_may_expire_flow(efx
->net_dev
,
1243 table
->spec
[index
].dmaq_id
,
1244 state
->rps_flow_id
[index
], index
)) {
1245 netif_info(efx
, rx_status
, efx
->net_dev
,
1246 "expiring filter %d [flow %u]\n",
1247 index
, state
->rps_flow_id
[index
]);
1248 efx_filter_table_clear_entry(efx
, table
, index
);
1250 index
= (index
+ 1) & mask
;
1253 state
->rps_expire_index
= stop
;
1254 if (table
->used
== 0)
1255 efx_filter_table_reset_search_depth(table
);
1257 spin_unlock_bh(&state
->lock
);
1261 #endif /* CONFIG_RFS_ACCEL */