[GitHub/MotorolaMobilityLLC/kernel-slsi.git] / net / core / flow_dissector.c

#include <linux/kernel.h>
#include <linux/skbuff.h>
#include <linux/export.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/if_vlan.h>
#include <net/ip.h>
#include <net/ipv6.h>
#include <linux/igmp.h>
#include <linux/icmp.h>
#include <linux/sctp.h>
#include <linux/dccp.h>
#include <linux/if_tunnel.h>
#include <linux/if_pppox.h>
#include <linux/ppp_defs.h>
#include <linux/stddef.h>
#include <linux/if_ether.h>
#include <net/flow_dissector.h>
#include <scsi/fc/fc_fcoe.h>

static bool skb_flow_dissector_uses_key(struct flow_dissector *flow_dissector,
					enum flow_dissector_key_id key_id)
{
	return flow_dissector->used_keys & (1 << key_id);
}

static void skb_flow_dissector_set_key(struct flow_dissector *flow_dissector,
				       enum flow_dissector_key_id key_id)
{
	flow_dissector->used_keys |= (1 << key_id);
}

static void *skb_flow_dissector_target(struct flow_dissector *flow_dissector,
				       enum flow_dissector_key_id key_id,
				       void *target_container)
{
	return ((char *) target_container) + flow_dissector->offset[key_id];
}

void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
			     const struct flow_dissector_key *key,
			     unsigned int key_count)
{
	unsigned int i;

	memset(flow_dissector, 0, sizeof(*flow_dissector));

	for (i = 0; i < key_count; i++, key++) {
		/* User should make sure that every key target offset is withing
		 * boundaries of unsigned short.
		 */
		BUG_ON(key->offset > USHRT_MAX);
		BUG_ON(skb_flow_dissector_uses_key(flow_dissector,
						   key->key_id));

		skb_flow_dissector_set_key(flow_dissector, key->key_id);
		flow_dissector->offset[key->key_id] = key->offset;
	}

	/* Ensure that the dissector always includes basic key. That way
	 * we are able to avoid handling lack of it in fast path.
	 */
	BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
					    FLOW_DISSECTOR_KEY_BASIC));
}
EXPORT_SYMBOL(skb_flow_dissector_init);

/**
 * __skb_flow_get_ports - extract the upper layer ports and return them
 * @skb: sk_buff to extract the ports from
 * @thoff: transport header offset
 * @ip_proto: protocol for which to get port offset
 * @data: raw buffer pointer to the packet, if NULL use skb->data
 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 *
 * The function will try to retrieve the ports at offset thoff + poff where poff
 * is the protocol port offset returned from proto_ports_offset
 */
__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
			    void *data, int hlen)
{
	int poff = proto_ports_offset(ip_proto);

	if (!data) {
		data = skb->data;
		hlen = skb_headlen(skb);
	}

	if (poff >= 0) {
		__be32 *ports, _ports;

		ports = __skb_header_pointer(skb, thoff + poff,
					     sizeof(_ports), data, hlen, &_ports);
		if (ports)
			return *ports;
	}

	return 0;
}
EXPORT_SYMBOL(__skb_flow_get_ports);

/**
 * __skb_flow_dissect - extract the flow_keys struct and return it
 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
 * @flow_dissector: list of keys to dissect
 * @target_container: target structure to put dissected values into
 * @data: raw buffer pointer to the packet, if NULL use skb->data
 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 *
 * The function will try to retrieve individual keys into target specified
 * by flow_dissector from either the skbuff or a raw buffer specified by the
 * rest parameters.
 *
 * Caller must take care of zeroing target container memory.
 */
bool __skb_flow_dissect(const struct sk_buff *skb,
			struct flow_dissector *flow_dissector,
			void *target_container,
			void *data, __be16 proto, int nhoff, int hlen)
{
	struct flow_dissector_key_basic *key_basic;
	struct flow_dissector_key_addrs *key_addrs;
	struct flow_dissector_key_ports *key_ports;
	u8 ip_proto;

	if (!data) {
		data = skb->data;
		proto = skb->protocol;
		nhoff = skb_network_offset(skb);
		hlen = skb_headlen(skb);
	}

	/* It is ensured by skb_flow_dissector_init() that basic key will
	 * be always present.
	 */
	key_basic = skb_flow_dissector_target(flow_dissector,
					      FLOW_DISSECTOR_KEY_BASIC,
					      target_container);

	if (skb_flow_dissector_uses_key(flow_dissector,
					FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
		struct ethhdr *eth = eth_hdr(skb);
		struct flow_dissector_key_eth_addrs *key_eth_addrs;

		key_eth_addrs = skb_flow_dissector_target(flow_dissector,
							  FLOW_DISSECTOR_KEY_ETH_ADDRS,
							  target_container);
		memcpy(key_eth_addrs, &eth->h_dest, sizeof(*key_eth_addrs));
	}

again:
	switch (proto) {
	case htons(ETH_P_IP): {
		const struct iphdr *iph;
		struct iphdr _iph;
ip:
		iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
		if (!iph || iph->ihl < 5)
			return false;
		nhoff += iph->ihl * 4;

		ip_proto = iph->protocol;
		if (ip_is_fragment(iph))
			ip_proto = 0;

		if (!skb_flow_dissector_uses_key(flow_dissector,
						 FLOW_DISSECTOR_KEY_IPV4_ADDRS))
			break;
		key_addrs = skb_flow_dissector_target(flow_dissector,
						      FLOW_DISSECTOR_KEY_IPV4_ADDRS,
						      target_container);
		memcpy(key_addrs, &iph->saddr, sizeof(*key_addrs));
		break;
	}
	case htons(ETH_P_IPV6): {
		const struct ipv6hdr *iph;
		struct ipv6hdr _iph;
		__be32 flow_label;

ipv6:
		iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
		if (!iph)
			return false;

		ip_proto = iph->nexthdr;
		nhoff += sizeof(struct ipv6hdr);

		if (skb_flow_dissector_uses_key(flow_dissector,
						FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
			key_addrs = skb_flow_dissector_target(flow_dissector,
							      FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
							      target_container);

			key_addrs->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
			key_addrs->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
			goto flow_label;
		}
		if (skb_flow_dissector_uses_key(flow_dissector,
						FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
			struct flow_dissector_key_ipv6_addrs *key_ipv6_addrs;

			key_ipv6_addrs = skb_flow_dissector_target(flow_dissector,
								   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
								   target_container);

			memcpy(key_ipv6_addrs, &iph->saddr, sizeof(*key_ipv6_addrs));
			goto flow_label;
		}
		break;
flow_label:
		flow_label = ip6_flowlabel(iph);
		if (flow_label) {
			/* Awesome, IPv6 packet has a flow label so we can
			 * use that to represent the ports without any
			 * further dissection.
			 */

			key_basic->n_proto = proto;
			key_basic->ip_proto = ip_proto;
			key_basic->thoff = (u16)nhoff;

			if (!skb_flow_dissector_uses_key(flow_dissector,
							 FLOW_DISSECTOR_KEY_PORTS))
				break;
			key_ports = skb_flow_dissector_target(flow_dissector,
							      FLOW_DISSECTOR_KEY_PORTS,
							      target_container);
			key_ports->ports = flow_label;

			return true;
		}

		break;
	}
	case htons(ETH_P_8021AD):
	case htons(ETH_P_8021Q): {
		const struct vlan_hdr *vlan;
		struct vlan_hdr _vlan;

		vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan);
		if (!vlan)
			return false;

		proto = vlan->h_vlan_encapsulated_proto;
		nhoff += sizeof(*vlan);
		goto again;
	}
	case htons(ETH_P_PPP_SES): {
		struct {
			struct pppoe_hdr hdr;
			__be16 proto;
		} *hdr, _hdr;
		hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
		if (!hdr)
			return false;
		proto = hdr->proto;
		nhoff += PPPOE_SES_HLEN;
		switch (proto) {
		case htons(PPP_IP):
			goto ip;
		case htons(PPP_IPV6):
			goto ipv6;
		default:
			return false;
		}
	}
	case htons(ETH_P_TIPC): {
		struct {
			__be32 pre[3];
			__be32 srcnode;
		} *hdr, _hdr;
		hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
		if (!hdr)
			return false;
		key_basic->n_proto = proto;
		key_basic->thoff = (u16)nhoff;

		if (skb_flow_dissector_uses_key(flow_dissector,
						FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
			key_addrs = skb_flow_dissector_target(flow_dissector,
							      FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
							      target_container);
			key_addrs->src = hdr->srcnode;
			key_addrs->dst = 0;
		}
		return true;
	}
	case htons(ETH_P_FCOE):
		key_basic->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
		/* fall through */
	default:
		return false;
	}

	switch (ip_proto) {
	case IPPROTO_GRE: {
		struct gre_hdr {
			__be16 flags;
			__be16 proto;
		} *hdr, _hdr;

		hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
		if (!hdr)
			return false;
		/*
		 * Only look inside GRE if version zero and no
		 * routing
		 */
		if (!(hdr->flags & (GRE_VERSION|GRE_ROUTING))) {
			proto = hdr->proto;
			nhoff += 4;
			if (hdr->flags & GRE_CSUM)
				nhoff += 4;
			if (hdr->flags & GRE_KEY)
				nhoff += 4;
			if (hdr->flags & GRE_SEQ)
				nhoff += 4;
			if (proto == htons(ETH_P_TEB)) {
				const struct ethhdr *eth;
				struct ethhdr _eth;

				eth = __skb_header_pointer(skb, nhoff,
							   sizeof(_eth),
							   data, hlen, &_eth);
				if (!eth)
					return false;
				proto = eth->h_proto;
				nhoff += sizeof(*eth);
			}
			goto again;
		}
		break;
	}
	case IPPROTO_IPIP:
		proto = htons(ETH_P_IP);
		goto ip;
	case IPPROTO_IPV6:
		proto = htons(ETH_P_IPV6);
		goto ipv6;
	default:
		break;
	}

	/* It is ensured by skb_flow_dissector_init() that basic key will
	 * be always present.
	 */
	key_basic = skb_flow_dissector_target(flow_dissector,
					      FLOW_DISSECTOR_KEY_BASIC,
					      target_container);
	key_basic->n_proto = proto;
	key_basic->ip_proto = ip_proto;
	key_basic->thoff = (u16) nhoff;

	if (skb_flow_dissector_uses_key(flow_dissector,
					FLOW_DISSECTOR_KEY_PORTS)) {
		key_ports = skb_flow_dissector_target(flow_dissector,
						      FLOW_DISSECTOR_KEY_PORTS,
						      target_container);
		key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
							data, hlen);
	}

	return true;
}
EXPORT_SYMBOL(__skb_flow_dissect);

static u32 hashrnd __read_mostly;
static __always_inline void __flow_hash_secret_init(void)
{
	net_get_random_once(&hashrnd, sizeof(hashrnd));
}

static __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c, u32 keyval)
{
	return jhash_3words(a, b, c, keyval);
}

static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
{
	u32 hash;

	/* get a consistent hash (same value on both flow directions) */
	if (((__force u32)keys->addrs.dst < (__force u32)keys->addrs.src) ||
	    (((__force u32)keys->addrs.dst == (__force u32)keys->addrs.src) &&
	     ((__force u16)keys->ports.dst < (__force u16)keys->ports.src))) {
		swap(keys->addrs.dst, keys->addrs.src);
		swap(keys->ports.src, keys->ports.dst);
	}

	hash = __flow_hash_3words((__force u32)keys->addrs.dst,
				  (__force u32)keys->addrs.src,
				  (__force u32)keys->ports.ports,
				  keyval);
	if (!hash)
		hash = 1;

	return hash;
}

u32 flow_hash_from_keys(struct flow_keys *keys)
{
	__flow_hash_secret_init();
	return __flow_hash_from_keys(keys, hashrnd);
}
EXPORT_SYMBOL(flow_hash_from_keys);

static inline u32 ___skb_get_hash(const struct sk_buff *skb,
				  struct flow_keys *keys, u32 keyval)
{
	if (!skb_flow_dissect_flow_keys(skb, keys))
		return 0;

	return __flow_hash_from_keys(keys, keyval);
}

struct _flow_keys_digest_data {
	__be16	n_proto;
	u8	ip_proto;
	u8	padding;
	__be32	ports;
	__be32	src;
	__be32	dst;
};

void make_flow_keys_digest(struct flow_keys_digest *digest,
			   const struct flow_keys *flow)
{
	struct _flow_keys_digest_data *data =
	    (struct _flow_keys_digest_data *)digest;

	BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));

	memset(digest, 0, sizeof(*digest));

	data->n_proto = flow->basic.n_proto;
	data->ip_proto = flow->basic.ip_proto;
	data->ports = flow->ports.ports;
	data->src = flow->addrs.src;
	data->dst = flow->addrs.dst;
}
EXPORT_SYMBOL(make_flow_keys_digest);

/**
 * __skb_get_hash: calculate a flow hash
 * @skb: sk_buff to calculate flow hash from
 *
 * This function calculates a flow hash based on src/dst addresses
 * and src/dst port numbers.  Sets hash in skb to non-zero hash value
 * on success, zero indicates no valid hash.  Also, sets l4_hash in skb
 * if hash is a canonical 4-tuple hash over transport ports.
 */
void __skb_get_hash(struct sk_buff *skb)
{
	struct flow_keys keys;
	u32 hash;

	__flow_hash_secret_init();

	hash = ___skb_get_hash(skb, &keys, hashrnd);
	if (!hash)
		return;
	if (keys.ports.ports)
		skb->l4_hash = 1;
	skb->sw_hash = 1;
	skb->hash = hash;
}
EXPORT_SYMBOL(__skb_get_hash);

__u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb)
{
	struct flow_keys keys;

	return ___skb_get_hash(skb, &keys, perturb);
}
EXPORT_SYMBOL(skb_get_hash_perturb);

u32 __skb_get_poff(const struct sk_buff *skb, void *data,
		   const struct flow_keys *keys, int hlen)
{
	u32 poff = keys->basic.thoff;

	switch (keys->basic.ip_proto) {
	case IPPROTO_TCP: {
		/* access doff as u8 to avoid unaligned access */
		const u8 *doff;
		u8 _doff;

		doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
					    data, hlen, &_doff);
		if (!doff)
			return poff;

		poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
		break;
	}
	case IPPROTO_UDP:
	case IPPROTO_UDPLITE:
		poff += sizeof(struct udphdr);
		break;
	/* For the rest, we do not really care about header
	 * extensions at this point for now.
	 */
	case IPPROTO_ICMP:
		poff += sizeof(struct icmphdr);
		break;
	case IPPROTO_ICMPV6:
		poff += sizeof(struct icmp6hdr);
		break;
	case IPPROTO_IGMP:
		poff += sizeof(struct igmphdr);
		break;
	case IPPROTO_DCCP:
		poff += sizeof(struct dccp_hdr);
		break;
	case IPPROTO_SCTP:
		poff += sizeof(struct sctphdr);
		break;
	}

	return poff;
}

/**
 * skb_get_poff - get the offset to the payload
 * @skb: sk_buff to get the payload offset from
 *
 * The function will get the offset to the payload as far as it could
 * be dissected.  The main user is currently BPF, so that we can dynamically
 * truncate packets without needing to push actual payload to the user
 * space and can analyze headers only, instead.
 */
u32 skb_get_poff(const struct sk_buff *skb)
{
	struct flow_keys keys;

	if (!skb_flow_dissect_flow_keys(skb, &keys))
		return 0;

	return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
}

static const struct flow_dissector_key flow_keys_dissector_keys[] = {
	{
		.key_id = FLOW_DISSECTOR_KEY_BASIC,
		.offset = offsetof(struct flow_keys, basic),
	},
	{
		.key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
		.offset = offsetof(struct flow_keys, addrs),
	},
	{
		.key_id = FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
		.offset = offsetof(struct flow_keys, addrs),
	},
	{
		.key_id = FLOW_DISSECTOR_KEY_PORTS,
		.offset = offsetof(struct flow_keys, ports),
	},
};

static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = {
	{
		.key_id = FLOW_DISSECTOR_KEY_BASIC,
		.offset = offsetof(struct flow_keys, basic),
	},
};

struct flow_dissector flow_keys_dissector __read_mostly;
EXPORT_SYMBOL(flow_keys_dissector);

struct flow_dissector flow_keys_buf_dissector __read_mostly;

static int __init init_default_flow_dissectors(void)
{
	skb_flow_dissector_init(&flow_keys_dissector,
				flow_keys_dissector_keys,
				ARRAY_SIZE(flow_keys_dissector_keys));
	skb_flow_dissector_init(&flow_keys_buf_dissector,
				flow_keys_buf_dissector_keys,
				ARRAY_SIZE(flow_keys_buf_dissector_keys));
	return 0;
}

late_initcall_sync(init_default_flow_dissectors);
Commit	Line	Data
fbff949e	1	#include <linux/kernel.h>
0744dd00	2	#include <linux/skbuff.h>
c452ed70	3	#include <linux/export.h>
0744dd00 ED	4	#include <linux/ip.h>
	5	#include <linux/ipv6.h>
	6	#include <linux/if_vlan.h>
	7	#include <net/ip.h>
ddbe5032	8	#include <net/ipv6.h>
f77668dc DB	9	#include <linux/igmp.h>
	10	#include <linux/icmp.h>
	11	#include <linux/sctp.h>
	12	#include <linux/dccp.h>
0744dd00 ED	13	#include <linux/if_tunnel.h>
	14	#include <linux/if_pppox.h>
	15	#include <linux/ppp_defs.h>
06635a35	16	#include <linux/stddef.h>
67a900cc	17	#include <linux/if_ether.h>
1bd758eb	18	#include <net/flow_dissector.h>
56193d1b	19	#include <scsi/fc/fc_fcoe.h>
0744dd00	20
fbff949e JP	21	static bool skb_flow_dissector_uses_key(struct flow_dissector *flow_dissector,
	22	enum flow_dissector_key_id key_id)
	23	{
	24	return flow_dissector->used_keys & (1 << key_id);
	25	}
	26
	27	static void skb_flow_dissector_set_key(struct flow_dissector *flow_dissector,
	28	enum flow_dissector_key_id key_id)
	29	{
	30	flow_dissector->used_keys \|= (1 << key_id);
	31	}
	32
	33	static void skb_flow_dissector_target(struct flow_dissector flow_dissector,
	34	enum flow_dissector_key_id key_id,
	35	void *target_container)
	36	{
	37	return ((char *) target_container) + flow_dissector->offset[key_id];
	38	}
	39
	40	void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
	41	const struct flow_dissector_key *key,
	42	unsigned int key_count)
	43	{
	44	unsigned int i;
	45
	46	memset(flow_dissector, 0, sizeof(*flow_dissector));
	47
	48	for (i = 0; i < key_count; i++, key++) {
	49	/* User should make sure that every key target offset is withing
	50	* boundaries of unsigned short.
	51	*/
	52	BUG_ON(key->offset > USHRT_MAX);
	53	BUG_ON(skb_flow_dissector_uses_key(flow_dissector,
	54	key->key_id));
	55
	56	skb_flow_dissector_set_key(flow_dissector, key->key_id);
	57	flow_dissector->offset[key->key_id] = key->offset;
	58	}
	59
	60	/* Ensure that the dissector always includes basic key. That way
	61	* we are able to avoid handling lack of it in fast path.
	62	*/
	63	BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
	64	FLOW_DISSECTOR_KEY_BASIC));
	65	}
	66	EXPORT_SYMBOL(skb_flow_dissector_init);
	67
357afe9c	68	/**
6451b3f5 WC	69	* __skb_flow_get_ports - extract the upper layer ports and return them
6451b3f5 WC	70	* @skb: sk_buff to extract the ports from
357afe9c NA	71	* @thoff: transport header offset
357afe9c NA	72	* @ip_proto: protocol for which to get port offset
6451b3f5 WC	73	* @data: raw buffer pointer to the packet, if NULL use skb->data
6451b3f5 WC	74	* @hlen: packet header length, if @data is NULL use skb_headlen(skb)
357afe9c NA	75	*
	76	* The function will try to retrieve the ports at offset thoff + poff where poff
	77	* is the protocol port offset returned from proto_ports_offset
	78	*/
690e36e7 DM	79	__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
690e36e7 DM	80	void *data, int hlen)
357afe9c NA	81	{
	82	int poff = proto_ports_offset(ip_proto);
	83
690e36e7 DM	84	if (!data) {
	85	data = skb->data;
	86	hlen = skb_headlen(skb);
	87	}
	88
357afe9c NA	89	if (poff >= 0) {
	90	__be32 *ports, _ports;
	91
690e36e7 DM	92	ports = __skb_header_pointer(skb, thoff + poff,
690e36e7 DM	93	sizeof(_ports), data, hlen, &_ports);
357afe9c NA	94	if (ports)
	95	return *ports;
	96	}
	97
	98	return 0;
	99	}
690e36e7	100	EXPORT_SYMBOL(__skb_flow_get_ports);
357afe9c	101
453a940e WC	102	/**
	103	* __skb_flow_dissect - extract the flow_keys struct and return it
	104	* @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
06635a35 JP	105	* @flow_dissector: list of keys to dissect
06635a35 JP	106	* @target_container: target structure to put dissected values into
453a940e WC	107	* @data: raw buffer pointer to the packet, if NULL use skb->data
	108	* @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
	109	* @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
	110	* @hlen: packet header length, if @data is NULL use skb_headlen(skb)
	111	*
06635a35 JP	112	* The function will try to retrieve individual keys into target specified
	113	* by flow_dissector from either the skbuff or a raw buffer specified by the
	114	* rest parameters.
	115	*
	116	* Caller must take care of zeroing target container memory.
453a940e	117	*/
06635a35 JP	118	bool __skb_flow_dissect(const struct sk_buff *skb,
	119	struct flow_dissector *flow_dissector,
	120	void *target_container,
453a940e	121	void *data, __be16 proto, int nhoff, int hlen)
0744dd00	122	{
06635a35 JP	123	struct flow_dissector_key_basic *key_basic;
	124	struct flow_dissector_key_addrs *key_addrs;
	125	struct flow_dissector_key_ports *key_ports;
0744dd00	126	u8 ip_proto;
0744dd00	127
690e36e7 DM	128	if (!data) {
690e36e7 DM	129	data = skb->data;
453a940e WC	130	proto = skb->protocol;
453a940e WC	131	nhoff = skb_network_offset(skb);
690e36e7 DM	132	hlen = skb_headlen(skb);
	133	}
	134
06635a35 JP	135	/* It is ensured by skb_flow_dissector_init() that basic key will
	136	* be always present.
	137	*/
	138	key_basic = skb_flow_dissector_target(flow_dissector,
	139	FLOW_DISSECTOR_KEY_BASIC,
	140	target_container);
0744dd00	141
67a900cc JP	142	if (skb_flow_dissector_uses_key(flow_dissector,
	143	FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
	144	struct ethhdr *eth = eth_hdr(skb);
	145	struct flow_dissector_key_eth_addrs *key_eth_addrs;
	146
	147	key_eth_addrs = skb_flow_dissector_target(flow_dissector,
	148	FLOW_DISSECTOR_KEY_ETH_ADDRS,
	149	target_container);
	150	memcpy(key_eth_addrs, &eth->h_dest, sizeof(*key_eth_addrs));
	151	}
	152
0744dd00 ED	153	again:
0744dd00 ED	154	switch (proto) {
2b8837ae	155	case htons(ETH_P_IP): {
0744dd00 ED	156	const struct iphdr *iph;
	157	struct iphdr _iph;
	158	ip:
690e36e7	159	iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
6f092343	160	if (!iph \|\| iph->ihl < 5)
0744dd00	161	return false;
3797d3e8	162	nhoff += iph->ihl * 4;
0744dd00	163
3797d3e8	164	ip_proto = iph->protocol;
0744dd00 ED	165	if (ip_is_fragment(iph))
0744dd00 ED	166	ip_proto = 0;
3797d3e8	167
06635a35 JP	168	if (!skb_flow_dissector_uses_key(flow_dissector,
06635a35 JP	169	FLOW_DISSECTOR_KEY_IPV4_ADDRS))
5af7fb6e	170	break;
06635a35 JP	171	key_addrs = skb_flow_dissector_target(flow_dissector,
	172	FLOW_DISSECTOR_KEY_IPV4_ADDRS,
	173	target_container);
	174	memcpy(key_addrs, &iph->saddr, sizeof(*key_addrs));
0744dd00 ED	175	break;
0744dd00 ED	176	}
2b8837ae	177	case htons(ETH_P_IPV6): {
0744dd00 ED	178	const struct ipv6hdr *iph;
0744dd00 ED	179	struct ipv6hdr _iph;
19469a87 TH	180	__be32 flow_label;
19469a87 TH	181
0744dd00	182	ipv6:
690e36e7	183	iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
0744dd00 ED	184	if (!iph)
	185	return false;
	186
	187	ip_proto = iph->nexthdr;
0744dd00	188	nhoff += sizeof(struct ipv6hdr);
19469a87	189
b924933c JP	190	if (skb_flow_dissector_uses_key(flow_dissector,
	191	FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
	192	key_addrs = skb_flow_dissector_target(flow_dissector,
	193	FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
	194	target_container);
56193d1b	195
b924933c JP	196	key_addrs->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
	197	key_addrs->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
	198	goto flow_label;
	199	}
	200	if (skb_flow_dissector_uses_key(flow_dissector,
	201	FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
	202	struct flow_dissector_key_ipv6_addrs *key_ipv6_addrs;
	203
	204	key_ipv6_addrs = skb_flow_dissector_target(flow_dissector,
	205	FLOW_DISSECTOR_KEY_IPV6_ADDRS,
	206	target_container);
5af7fb6e	207
b924933c JP	208	memcpy(key_ipv6_addrs, &iph->saddr, sizeof(*key_ipv6_addrs));
	209	goto flow_label;
	210	}
	211	break;
	212	flow_label:
19469a87 TH	213	flow_label = ip6_flowlabel(iph);
	214	if (flow_label) {
	215	/* Awesome, IPv6 packet has a flow label so we can
	216	* use that to represent the ports without any
	217	* further dissection.
	218	*/
06635a35 JP	219
	220	key_basic->n_proto = proto;
	221	key_basic->ip_proto = ip_proto;
	222	key_basic->thoff = (u16)nhoff;
	223
	224	if (!skb_flow_dissector_uses_key(flow_dissector,
	225	FLOW_DISSECTOR_KEY_PORTS))
	226	break;
	227	key_ports = skb_flow_dissector_target(flow_dissector,
	228	FLOW_DISSECTOR_KEY_PORTS,
	229	target_container);
	230	key_ports->ports = flow_label;
19469a87 TH	231
	232	return true;
	233	}
	234
0744dd00 ED	235	break;
0744dd00 ED	236	}
2b8837ae JP	237	case htons(ETH_P_8021AD):
2b8837ae JP	238	case htons(ETH_P_8021Q): {
0744dd00 ED	239	const struct vlan_hdr *vlan;
	240	struct vlan_hdr _vlan;
	241
690e36e7	242	vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan);
0744dd00 ED	243	if (!vlan)
	244	return false;
	245
	246	proto = vlan->h_vlan_encapsulated_proto;
	247	nhoff += sizeof(*vlan);
	248	goto again;
	249	}
2b8837ae	250	case htons(ETH_P_PPP_SES): {
0744dd00 ED	251	struct {
	252	struct pppoe_hdr hdr;
	253	__be16 proto;
	254	} *hdr, _hdr;
690e36e7	255	hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
0744dd00 ED	256	if (!hdr)
	257	return false;
	258	proto = hdr->proto;
	259	nhoff += PPPOE_SES_HLEN;
	260	switch (proto) {
2b8837ae	261	case htons(PPP_IP):
0744dd00	262	goto ip;
2b8837ae	263	case htons(PPP_IPV6):
0744dd00 ED	264	goto ipv6;
	265	default:
	266	return false;
	267	}
	268	}
08bfc9cb EH	269	case htons(ETH_P_TIPC): {
	270	struct {
	271	__be32 pre[3];
	272	__be32 srcnode;
	273	} *hdr, _hdr;
	274	hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
	275	if (!hdr)
	276	return false;
06635a35 JP	277	key_basic->n_proto = proto;
	278	key_basic->thoff = (u16)nhoff;
	279
	280	if (skb_flow_dissector_uses_key(flow_dissector,
	281	FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
06635a35 JP	282	key_addrs = skb_flow_dissector_target(flow_dissector,
	283	FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
	284	target_container);
	285	key_addrs->src = hdr->srcnode;
	286	key_addrs->dst = 0;
	287	}
08bfc9cb EH	288	return true;
08bfc9cb EH	289	}
56193d1b	290	case htons(ETH_P_FCOE):
06635a35	291	key_basic->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
56193d1b	292	/* fall through */
0744dd00 ED	293	default:
	294	return false;
	295	}
	296
	297	switch (ip_proto) {
	298	case IPPROTO_GRE: {
	299	struct gre_hdr {
	300	__be16 flags;
	301	__be16 proto;
	302	} *hdr, _hdr;
	303
690e36e7	304	hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
0744dd00 ED	305	if (!hdr)
	306	return false;
	307	/*
	308	* Only look inside GRE if version zero and no
	309	* routing
	310	*/
	311	if (!(hdr->flags & (GRE_VERSION\|GRE_ROUTING))) {
	312	proto = hdr->proto;
	313	nhoff += 4;
	314	if (hdr->flags & GRE_CSUM)
	315	nhoff += 4;
	316	if (hdr->flags & GRE_KEY)
	317	nhoff += 4;
	318	if (hdr->flags & GRE_SEQ)
	319	nhoff += 4;
e1733de2 MD	320	if (proto == htons(ETH_P_TEB)) {
	321	const struct ethhdr *eth;
	322	struct ethhdr _eth;
	323
690e36e7 DM	324	eth = __skb_header_pointer(skb, nhoff,
	325	sizeof(_eth),
	326	data, hlen, &_eth);
e1733de2 MD	327	if (!eth)
	328	return false;
	329	proto = eth->h_proto;
	330	nhoff += sizeof(*eth);
	331	}
0744dd00 ED	332	goto again;
	333	}
	334	break;
	335	}
	336	case IPPROTO_IPIP:
fca41895 TH	337	proto = htons(ETH_P_IP);
fca41895 TH	338	goto ip;
b438f940 TH	339	case IPPROTO_IPV6:
	340	proto = htons(ETH_P_IPV6);
	341	goto ipv6;
0744dd00 ED	342	default:
	343	break;
	344	}
	345
06635a35 JP	346	/* It is ensured by skb_flow_dissector_init() that basic key will
	347	* be always present.
	348	*/
	349	key_basic = skb_flow_dissector_target(flow_dissector,
	350	FLOW_DISSECTOR_KEY_BASIC,
	351	target_container);
	352	key_basic->n_proto = proto;
	353	key_basic->ip_proto = ip_proto;
	354	key_basic->thoff = (u16) nhoff;
	355
	356	if (skb_flow_dissector_uses_key(flow_dissector,
	357	FLOW_DISSECTOR_KEY_PORTS)) {
	358	key_ports = skb_flow_dissector_target(flow_dissector,
	359	FLOW_DISSECTOR_KEY_PORTS,
	360	target_container);
	361	key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
	362	data, hlen);
	363	}
5af7fb6e	364
0744dd00 ED	365	return true;
0744dd00 ED	366	}
690e36e7	367	EXPORT_SYMBOL(__skb_flow_dissect);
441d9d32 CW	368
441d9d32 CW	369	static u32 hashrnd __read_mostly;
66415cf8 HFS	370	static __always_inline void __flow_hash_secret_init(void)
	371	{
	372	net_get_random_once(&hashrnd, sizeof(hashrnd));
	373	}
	374
50fb7992	375	static __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c, u32 keyval)
66415cf8	376	{
50fb7992	377	return jhash_3words(a, b, c, keyval);
66415cf8 HFS	378	}
66415cf8 HFS	379
50fb7992	380	static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
5ed20a68 TH	381	{
	382	u32 hash;
	383
	384	/* get a consistent hash (same value on both flow directions) */
06635a35 JP	385	if (((__force u32)keys->addrs.dst < (__force u32)keys->addrs.src) \|\|
06635a35 JP	386	(((__force u32)keys->addrs.dst == (__force u32)keys->addrs.src) &&
59346afe	387	((__force u16)keys->ports.dst < (__force u16)keys->ports.src))) {
06635a35	388	swap(keys->addrs.dst, keys->addrs.src);
59346afe	389	swap(keys->ports.src, keys->ports.dst);
5ed20a68 TH	390	}
5ed20a68 TH	391
06635a35 JP	392	hash = __flow_hash_3words((__force u32)keys->addrs.dst,
	393	(__force u32)keys->addrs.src,
	394	(__force u32)keys->ports.ports,
50fb7992	395	keyval);
5ed20a68 TH	396	if (!hash)
	397	hash = 1;
	398
	399	return hash;
	400	}
	401
	402	u32 flow_hash_from_keys(struct flow_keys *keys)
	403	{
50fb7992 TH	404	__flow_hash_secret_init();
50fb7992 TH	405	return __flow_hash_from_keys(keys, hashrnd);
5ed20a68 TH	406	}
	407	EXPORT_SYMBOL(flow_hash_from_keys);
	408
50fb7992 TH	409	static inline u32 ___skb_get_hash(const struct sk_buff *skb,
	410	struct flow_keys *keys, u32 keyval)
	411	{
06635a35	412	if (!skb_flow_dissect_flow_keys(skb, keys))
50fb7992 TH	413	return 0;
	414
	415	return __flow_hash_from_keys(keys, keyval);
	416	}
	417
2f59e1eb TH	418	struct _flow_keys_digest_data {
	419	__be16 n_proto;
	420	u8 ip_proto;
	421	u8 padding;
	422	__be32 ports;
	423	__be32 src;
	424	__be32 dst;
	425	};
	426
	427	void make_flow_keys_digest(struct flow_keys_digest *digest,
	428	const struct flow_keys *flow)
	429	{
	430	struct _flow_keys_digest_data *data =
	431	(struct _flow_keys_digest_data *)digest;
	432
	433	BUILD_BUG_ON(sizeof(data) > sizeof(digest));
	434
	435	memset(digest, 0, sizeof(*digest));
	436
06635a35 JP	437	data->n_proto = flow->basic.n_proto;
	438	data->ip_proto = flow->basic.ip_proto;
	439	data->ports = flow->ports.ports;
	440	data->src = flow->addrs.src;
	441	data->dst = flow->addrs.dst;
2f59e1eb TH	442	}
	443	EXPORT_SYMBOL(make_flow_keys_digest);
	444
d4fd3275 JP	445	/**
	446	* __skb_get_hash: calculate a flow hash
	447	* @skb: sk_buff to calculate flow hash from
	448	*
	449	* This function calculates a flow hash based on src/dst addresses
61b905da TH	450	* and src/dst port numbers. Sets hash in skb to non-zero hash value
61b905da TH	451	* on success, zero indicates no valid hash. Also, sets l4_hash in skb
441d9d32 CW	452	* if hash is a canonical 4-tuple hash over transport ports.
441d9d32 CW	453	*/
3958afa1	454	void __skb_get_hash(struct sk_buff *skb)
441d9d32 CW	455	{
441d9d32 CW	456	struct flow_keys keys;
50fb7992	457	u32 hash;
441d9d32	458
50fb7992 TH	459	__flow_hash_secret_init();
	460
	461	hash = ___skb_get_hash(skb, &keys, hashrnd);
	462	if (!hash)
441d9d32	463	return;
06635a35	464	if (keys.ports.ports)
61b905da	465	skb->l4_hash = 1;
a3b18ddb	466	skb->sw_hash = 1;
50fb7992	467	skb->hash = hash;
441d9d32	468	}
3958afa1	469	EXPORT_SYMBOL(__skb_get_hash);
441d9d32	470
50fb7992 TH	471	__u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb)
	472	{
	473	struct flow_keys keys;
	474
	475	return ___skb_get_hash(skb, &keys, perturb);
	476	}
	477	EXPORT_SYMBOL(skb_get_hash_perturb);
	478
56193d1b AD	479	u32 __skb_get_poff(const struct sk_buff skb, void data,
56193d1b AD	480	const struct flow_keys *keys, int hlen)
f77668dc	481	{
06635a35	482	u32 poff = keys->basic.thoff;
f77668dc	483
06635a35	484	switch (keys->basic.ip_proto) {
f77668dc	485	case IPPROTO_TCP: {
5af7fb6e AD	486	/* access doff as u8 to avoid unaligned access */
	487	const u8 *doff;
	488	u8 _doff;
f77668dc	489
5af7fb6e AD	490	doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
	491	data, hlen, &_doff);
	492	if (!doff)
f77668dc DB	493	return poff;
f77668dc DB	494
5af7fb6e	495	poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
f77668dc DB	496	break;
	497	}
	498	case IPPROTO_UDP:
	499	case IPPROTO_UDPLITE:
	500	poff += sizeof(struct udphdr);
	501	break;
	502	/* For the rest, we do not really care about header
	503	* extensions at this point for now.
	504	*/
	505	case IPPROTO_ICMP:
	506	poff += sizeof(struct icmphdr);
	507	break;
	508	case IPPROTO_ICMPV6:
	509	poff += sizeof(struct icmp6hdr);
	510	break;
	511	case IPPROTO_IGMP:
	512	poff += sizeof(struct igmphdr);
	513	break;
	514	case IPPROTO_DCCP:
	515	poff += sizeof(struct dccp_hdr);
	516	break;
	517	case IPPROTO_SCTP:
	518	poff += sizeof(struct sctphdr);
	519	break;
	520	}
	521
	522	return poff;
	523	}
	524
0db89b8b JP	525	/**
	526	* skb_get_poff - get the offset to the payload
	527	* @skb: sk_buff to get the payload offset from
	528	*
	529	* The function will get the offset to the payload as far as it could
	530	* be dissected. The main user is currently BPF, so that we can dynamically
56193d1b AD	531	* truncate packets without needing to push actual payload to the user
	532	* space and can analyze headers only, instead.
	533	*/
	534	u32 skb_get_poff(const struct sk_buff *skb)
	535	{
	536	struct flow_keys keys;
	537
06635a35	538	if (!skb_flow_dissect_flow_keys(skb, &keys))
56193d1b AD	539	return 0;
	540
	541	return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
	542	}
06635a35 JP	543
	544	static const struct flow_dissector_key flow_keys_dissector_keys[] = {
	545	{
	546	.key_id = FLOW_DISSECTOR_KEY_BASIC,
	547	.offset = offsetof(struct flow_keys, basic),
	548	},
	549	{
	550	.key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
	551	.offset = offsetof(struct flow_keys, addrs),
	552	},
	553	{
	554	.key_id = FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
	555	.offset = offsetof(struct flow_keys, addrs),
	556	},
	557	{
	558	.key_id = FLOW_DISSECTOR_KEY_PORTS,
	559	.offset = offsetof(struct flow_keys, ports),
	560	},
	561	};
	562
	563	static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = {
	564	{
	565	.key_id = FLOW_DISSECTOR_KEY_BASIC,
	566	.offset = offsetof(struct flow_keys, basic),
	567	},
	568	};
	569
	570	struct flow_dissector flow_keys_dissector __read_mostly;
	571	EXPORT_SYMBOL(flow_keys_dissector);
	572
	573	struct flow_dissector flow_keys_buf_dissector __read_mostly;
	574
	575	static int __init init_default_flow_dissectors(void)
	576	{
	577	skb_flow_dissector_init(&flow_keys_dissector,
	578	flow_keys_dissector_keys,
	579	ARRAY_SIZE(flow_keys_dissector_keys));
	580	skb_flow_dissector_init(&flow_keys_buf_dissector,
	581	flow_keys_buf_dissector_keys,
	582	ARRAY_SIZE(flow_keys_buf_dissector_keys));
	583	return 0;
	584	}
	585
	586	late_initcall_sync(init_default_flow_dissectors);