if (unlikely(!pskb_may_pull(skb, noff + sizeof(*iph))))
return false;
iph = ip_hdr(skb);
- fk->addrs.src = iph->saddr;
- fk->addrs.dst = iph->daddr;
+ iph_to_flow_copy_v4addrs(fk, iph);
noff += iph->ihl << 2;
if (!ip_is_fragment(iph))
proto = iph->protocol;
if (unlikely(!pskb_may_pull(skb, noff + sizeof(*iph6))))
return false;
iph6 = ipv6_hdr(skb);
- fk->addrs.src = (__force __be32)ipv6_addr_hash(&iph6->saddr);
- fk->addrs.dst = (__force __be32)ipv6_addr_hash(&iph6->daddr);
+ iph_to_flow_copy_v6addrs(fk, iph6);
noff += sizeof(*iph6);
proto = iph6->nexthdr;
} else {
hash = bond_eth_hash(skb);
else
hash = (__force u32)flow.ports.ports;
- hash ^= (__force u32)flow.addrs.dst ^ (__force u32)flow.addrs.src;
+ hash ^= (__force u32)flow_get_u32_dst(&flow) ^
+ (__force u32)flow_get_u32_src(&flow);
hash ^= (hash >> 16);
hash ^= (hash >> 8);
return -EPROTONOSUPPORT;
};
data.type = FILTER_IPV4_5TUPLE;
- data.u.ipv4.src_addr = ntohl(keys->addrs.src);
- data.u.ipv4.dst_addr = ntohl(keys->addrs.dst);
+ data.u.ipv4.src_addr = ntohl(keys->addrs.v4addrs.src);
+ data.u.ipv4.dst_addr = ntohl(keys->addrs.v4addrs.dst);
data.u.ipv4.src_port = ntohs(keys->ports.src);
data.u.ipv4.dst_port = ntohs(keys->ports.dst);
data.u.ipv4.flags = FILTER_FIELDS_IPV4_5TUPLE;
struct enic_rfs_fltr_node *tpos;
hlist_for_each_entry(tpos, h, node)
- if (tpos->keys.addrs.src == k->addrs.src &&
- tpos->keys.addrs.dst == k->addrs.dst &&
+ if (tpos->keys.addrs.v4addrs.src == k->addrs.v4addrs.src &&
+ tpos->keys.addrs.v4addrs.dst == k->addrs.v4addrs.dst &&
tpos->keys.ports.ports == k->ports.ports &&
tpos->keys.basic.ip_proto == k->basic.ip_proto &&
tpos->keys.basic.n_proto == k->basic.n_proto)
break;
}
- fsp->h_u.tcp_ip4_spec.ip4src = n->keys.addrs.src;
+ fsp->h_u.tcp_ip4_spec.ip4src = flow_get_u32_src(&n->keys);
fsp->m_u.tcp_ip4_spec.ip4src = (__u32)~0;
- fsp->h_u.tcp_ip4_spec.ip4dst = n->keys.addrs.dst;
+ fsp->h_u.tcp_ip4_spec.ip4dst = flow_get_u32_dst(&n->keys);
fsp->m_u.tcp_ip4_spec.ip4dst = (__u32)~0;
fsp->h_u.tcp_ip4_spec.psrc = n->keys.ports.src;
*/
struct flow_dissector_key_control {
u16 thoff;
- u16 padding;
+ u16 addr_type;
};
/**
};
/**
- * struct flow_dissector_key_addrs:
- * @src: source ip address in case of IPv4
- * For IPv6 it contains 32bit hash of src address
- * @dst: destination ip address in case of IPv4
- * For IPv6 it contains 32bit hash of dst address
+ * struct flow_dissector_key_ipv4_addrs:
+ * @src: source ip address
+ * @dst: destination ip address
*/
-struct flow_dissector_key_addrs {
+struct flow_dissector_key_ipv4_addrs {
/* (src,dst) must be grouped, in the same way than in IP header */
__be32 src;
__be32 dst;
};
+/**
+ * struct flow_dissector_key_ipv6_addrs:
+ * @src: source ip address
+ * @dst: destination ip address
+ */
+struct flow_dissector_key_ipv6_addrs {
+ /* (src,dst) must be grouped, in the same way than in IP header */
+ struct in6_addr src;
+ struct in6_addr dst;
+};
+
+/**
+ * struct flow_dissector_key_addrs:
+ * @v4addrs: IPv4 addresses
+ * @v6addrs: IPv6 addresses
+ */
+struct flow_dissector_key_addrs {
+ union {
+ struct flow_dissector_key_ipv4_addrs v4addrs;
+ struct flow_dissector_key_ipv6_addrs v6addrs;
+ };
+};
+
/**
* flow_dissector_key_tp_ports:
* @ports: port numbers of Transport header
};
};
-/**
- * struct flow_dissector_key_ipv6_addrs:
- * @src: source ip address
- * @dst: destination ip address
- */
-struct flow_dissector_key_ipv6_addrs {
- /* (src,dst) must be grouped, in the same way than in IP header */
- struct in6_addr src;
- struct in6_addr dst;
-};
/**
* struct flow_dissector_key_eth_addrs:
enum flow_dissector_key_id {
FLOW_DISSECTOR_KEY_CONTROL, /* struct flow_dissector_key_control */
FLOW_DISSECTOR_KEY_BASIC, /* struct flow_dissector_key_basic */
- FLOW_DISSECTOR_KEY_IPV4_ADDRS, /* struct flow_dissector_key_addrs */
+ FLOW_DISSECTOR_KEY_IPV4_ADDRS, /* struct flow_dissector_key_ipv4_addrs */
+ FLOW_DISSECTOR_KEY_IPV6_ADDRS, /* struct flow_dissector_key_ipv6_addrs */
FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS, /* struct flow_dissector_key_addrs */
FLOW_DISSECTOR_KEY_PORTS, /* struct flow_dissector_key_ports */
- FLOW_DISSECTOR_KEY_IPV6_ADDRS, /* struct flow_dissector_key_ipv6_addrs */
FLOW_DISSECTOR_KEY_ETH_ADDRS, /* struct flow_dissector_key_eth_addrs */
FLOW_DISSECTOR_KEY_MAX,
#define FLOW_KEYS_HASH_OFFSET \
offsetof(struct flow_keys, FLOW_KEYS_HASH_START_FIELD)
+__be32 flow_get_u32_src(const struct flow_keys *flow);
+__be32 flow_get_u32_dst(const struct flow_keys *flow);
+
extern struct flow_dissector flow_keys_dissector;
extern struct flow_dissector flow_keys_buf_dissector;
skb->len, proto, 0);
}
+/* copy IPv4 saddr & daddr to flow_keys, possibly using 64bit load/store
+ * Equivalent to : flow->v4addrs.src = iph->saddr;
+ * flow->v4addrs.dst = iph->daddr;
+ */
+static inline void iph_to_flow_copy_v4addrs(struct flow_keys *flow,
+ const struct iphdr *iph)
+{
+ BUILD_BUG_ON(offsetof(typeof(flow->addrs), v4addrs.dst) !=
+ offsetof(typeof(flow->addrs), v4addrs.src) +
+ sizeof(flow->addrs.v4addrs.src));
+ memcpy(&flow->addrs.v4addrs, &iph->saddr, sizeof(flow->addrs.v4addrs));
+ flow->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
+}
+
static inline void inet_set_txhash(struct sock *sk)
{
struct inet_sock *inet = inet_sk(sk);
memset(&keys, 0, sizeof(keys));
- keys.addrs.src = inet->inet_saddr;
- keys.addrs.dst = inet->inet_daddr;
+ keys.addrs.v4addrs.src = inet->inet_saddr;
+ keys.addrs.v4addrs.dst = inet->inet_daddr;
+ keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
keys.ports.src = inet->inet_sport;
keys.ports.dst = inet->inet_dport;
return hlimit;
}
+/* copy IPv6 saddr & daddr to flow_keys, possibly using 64bit load/store
+ * Equivalent to : flow->v6addrs.src = iph->saddr;
+ * flow->v6addrs.dst = iph->daddr;
+ */
+static inline void iph_to_flow_copy_v6addrs(struct flow_keys *flow,
+ const struct ipv6hdr *iph)
+{
+ BUILD_BUG_ON(offsetof(typeof(flow->addrs), v6addrs.dst) !=
+ offsetof(typeof(flow->addrs), v6addrs.src) +
+ sizeof(flow->addrs.v6addrs.src));
+ memcpy(&flow->addrs.v6addrs, &iph->saddr, sizeof(flow->addrs.v6addrs));
+ flow->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
+}
+
#if IS_ENABLED(CONFIG_IPV6)
static inline void ip6_set_txhash(struct sock *sk)
{
memset(&keys, 0, sizeof(keys));
- keys.addrs.src = (__force __be32)ipv6_addr_hash(&np->saddr);
- keys.addrs.dst = (__force __be32)ipv6_addr_hash(&sk->sk_v6_daddr);
+ memcpy(&keys.addrs.v6addrs.src, &np->saddr,
+ sizeof(keys.addrs.v6addrs.src));
+ memcpy(&keys.addrs.v6addrs.dst, &sk->sk_v6_daddr,
+ sizeof(keys.addrs.v6addrs.dst));
+ keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
keys.ports.src = inet->inet_sport;
keys.ports.dst = inet->inet_dport;
if (!skb_flow_dissector_uses_key(flow_dissector,
FLOW_DISSECTOR_KEY_IPV4_ADDRS))
break;
+
key_addrs = skb_flow_dissector_target(flow_dissector,
- FLOW_DISSECTOR_KEY_IPV4_ADDRS,
- target_container);
- memcpy(key_addrs, &iph->saddr, sizeof(*key_addrs));
+ FLOW_DISSECTOR_KEY_IPV4_ADDRS, target_container);
+ memcpy(&key_addrs->v4addrs, &iph->saddr,
+ sizeof(key_addrs->v4addrs));
+ key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
break;
}
case htons(ETH_P_IPV6): {
FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
target_container);
- key_addrs->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
- key_addrs->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
+ key_addrs->v4addrs.src =
+ (__force __be32)ipv6_addr_hash(&iph->saddr);
+ key_addrs->v4addrs.dst =
+ (__force __be32)ipv6_addr_hash(&iph->daddr);
+ key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
goto flow_label;
}
if (skb_flow_dissector_uses_key(flow_dissector,
target_container);
memcpy(key_ipv6_addrs, &iph->saddr, sizeof(*key_ipv6_addrs));
+ key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
goto flow_label;
}
break;
key_addrs = skb_flow_dissector_target(flow_dissector,
FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
target_container);
- key_addrs->src = hdr->srcnode;
- key_addrs->dst = 0;
+ key_addrs->v4addrs.src = hdr->srcnode;
+ key_addrs->v4addrs.dst = 0;
+ key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
}
return true;
}
static inline size_t flow_keys_hash_length(struct flow_keys *flow)
{
+ size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs);
BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32));
- return (sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) / sizeof(u32);
+ BUILD_BUG_ON(offsetof(typeof(*flow), addrs) !=
+ sizeof(*flow) - sizeof(flow->addrs));
+
+ switch (flow->control.addr_type) {
+ case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
+ diff -= sizeof(flow->addrs.v4addrs);
+ break;
+ case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
+ diff -= sizeof(flow->addrs.v6addrs);
+ break;
+ }
+ return (sizeof(*flow) - diff) / sizeof(u32);
+}
+
+__be32 flow_get_u32_src(const struct flow_keys *flow)
+{
+ switch (flow->control.addr_type) {
+ case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
+ return flow->addrs.v4addrs.src;
+ case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
+ return (__force __be32)ipv6_addr_hash(
+ &flow->addrs.v6addrs.src);
+ default:
+ return 0;
+ }
+}
+EXPORT_SYMBOL(flow_get_u32_src);
+
+__be32 flow_get_u32_dst(const struct flow_keys *flow)
+{
+ switch (flow->control.addr_type) {
+ case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
+ return flow->addrs.v4addrs.dst;
+ case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
+ return (__force __be32)ipv6_addr_hash(
+ &flow->addrs.v6addrs.dst);
+ default:
+ return 0;
+ }
+}
+EXPORT_SYMBOL(flow_get_u32_dst);
+
+static inline void __flow_hash_consistentify(struct flow_keys *keys)
+{
+ int addr_diff, i;
+
+ switch (keys->control.addr_type) {
+ case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
+ addr_diff = (__force u32)keys->addrs.v4addrs.dst -
+ (__force u32)keys->addrs.v4addrs.src;
+ if ((addr_diff < 0) ||
+ (addr_diff == 0 &&
+ ((__force u16)keys->ports.dst <
+ (__force u16)keys->ports.src))) {
+ swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst);
+ swap(keys->ports.src, keys->ports.dst);
+ }
+ break;
+ case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
+ addr_diff = memcmp(&keys->addrs.v6addrs.dst,
+ &keys->addrs.v6addrs.src,
+ sizeof(keys->addrs.v6addrs.dst));
+ if ((addr_diff < 0) ||
+ (addr_diff == 0 &&
+ ((__force u16)keys->ports.dst <
+ (__force u16)keys->ports.src))) {
+ for (i = 0; i < 4; i++)
+ swap(keys->addrs.v6addrs.src.s6_addr32[i],
+ keys->addrs.v6addrs.dst.s6_addr32[i]);
+ swap(keys->ports.src, keys->ports.dst);
+ }
+ break;
+ }
}
static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
{
u32 hash;
- /* get a consistent hash (same value on both flow directions) */
- if (((__force u32)keys->addrs.dst < (__force u32)keys->addrs.src) ||
- (((__force u32)keys->addrs.dst == (__force u32)keys->addrs.src) &&
- ((__force u16)keys->ports.dst < (__force u16)keys->ports.src))) {
- swap(keys->addrs.dst, keys->addrs.src);
- swap(keys->ports.src, keys->ports.dst);
- }
+ __flow_hash_consistentify(keys);
hash = __flow_hash_words((u32 *)flow_keys_hash_start(keys),
flow_keys_hash_length(keys), keyval);
data->n_proto = flow->basic.n_proto;
data->ip_proto = flow->basic.ip_proto;
data->ports = flow->ports.ports;
- data->src = flow->addrs.src;
- data->dst = flow->addrs.dst;
+ data->src = flow->addrs.v4addrs.src;
+ data->dst = flow->addrs.v4addrs.dst;
}
EXPORT_SYMBOL(make_flow_keys_digest);
},
{
.key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
- .offset = offsetof(struct flow_keys, addrs),
+ .offset = offsetof(struct flow_keys, addrs.v4addrs),
+ },
+ {
+ .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
+ .offset = offsetof(struct flow_keys, addrs.v6addrs),
},
{
.key_id = FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
- .offset = offsetof(struct flow_keys, addrs),
+ .offset = offsetof(struct flow_keys, addrs.v4addrs),
},
{
.key_id = FLOW_DISSECTOR_KEY_PORTS,
/* parse any remaining L2/L3 headers, check for L4 */
if (!skb_flow_dissect_flow_keys_buf(&keys, data, eth->h_proto,
sizeof(*eth), len))
- return max_t(u32, keys.basic.thoff, sizeof(*eth));
+ return max_t(u32, keys.control.thoff, sizeof(*eth));
/* parse for any L4 headers */
return min_t(u32, __skb_get_poff(NULL, data, &keys, len), len);
static u32 flow_get_src(const struct sk_buff *skb, const struct flow_keys *flow)
{
- if (flow->addrs.src)
- return ntohl(flow->addrs.src);
+ __be32 src = flow_get_u32_src(flow);
+
+ if (src)
+ return ntohl(src);
+
return addr_fold(skb->sk);
}
static u32 flow_get_dst(const struct sk_buff *skb, const struct flow_keys *flow)
{
- if (flow->addrs.dst)
- return ntohl(flow->addrs.dst);
+ __be32 dst = flow_get_u32_dst(flow);
+
+ if (dst)
+ return ntohl(dst);
+
return addr_fold(skb_dst(skb)) ^ (__force u16) tc_skb_protocol(skb);
}
struct flow_dissector_key_control control;
struct flow_dissector_key_basic basic;
struct flow_dissector_key_eth_addrs eth;
+ struct flow_dissector_key_addrs ipaddrs;
union {
- struct flow_dissector_key_addrs ipv4;
+ struct flow_dissector_key_ipv4_addrs ipv4;
struct flow_dissector_key_ipv6_addrs ipv6;
};
struct flow_dissector_key_ports tp;
&mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
sizeof(key->basic.ip_proto));
}
- if (key->basic.n_proto == htons(ETH_P_IP)) {
+ if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
&mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
sizeof(key->ipv4.src));
fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
&mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
sizeof(key->ipv4.dst));
- } else if (key->basic.n_proto == htons(ETH_P_IPV6)) {
+ } else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
&mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
sizeof(key->ipv6.src));
sizeof(key->basic.ip_proto)))
goto nla_put_failure;
- if (key->basic.n_proto == htons(ETH_P_IP) &&
+ if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
(fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
&mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
sizeof(key->ipv4.src)) ||
&mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
sizeof(key->ipv4.dst))))
goto nla_put_failure;
- else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
+ else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
(fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
&mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
sizeof(key->ipv6.src)) ||