[GitHub/MotorolaMobilityLLC/kernel-slsi.git] / net / core / flow_dissector.c

#include <linux/kernel.h>
#include <linux/skbuff.h>
#include <linux/export.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/if_vlan.h>
#include <net/ip.h>
#include <net/ipv6.h>
#include <linux/igmp.h>
#include <linux/icmp.h>
#include <linux/sctp.h>
#include <linux/dccp.h>
#include <linux/if_tunnel.h>
#include <linux/if_pppox.h>
#include <linux/ppp_defs.h>
#include <linux/stddef.h>
#include <net/flow_dissector.h>
#include <scsi/fc/fc_fcoe.h>

static bool skb_flow_dissector_uses_key(struct flow_dissector *flow_dissector,
					enum flow_dissector_key_id key_id)
{
	return flow_dissector->used_keys & (1 << key_id);
}

static void skb_flow_dissector_set_key(struct flow_dissector *flow_dissector,
				       enum flow_dissector_key_id key_id)
{
	flow_dissector->used_keys |= (1 << key_id);
}

static void *skb_flow_dissector_target(struct flow_dissector *flow_dissector,
				       enum flow_dissector_key_id key_id,
				       void *target_container)
{
	return ((char *) target_container) + flow_dissector->offset[key_id];
}

void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
			     const struct flow_dissector_key *key,
			     unsigned int key_count)
{
	unsigned int i;

	memset(flow_dissector, 0, sizeof(*flow_dissector));

	for (i = 0; i < key_count; i++, key++) {
		/* User should make sure that every key target offset is withing
		 * boundaries of unsigned short.
		 */
		BUG_ON(key->offset > USHRT_MAX);
		BUG_ON(skb_flow_dissector_uses_key(flow_dissector,
						   key->key_id));

		skb_flow_dissector_set_key(flow_dissector, key->key_id);
		flow_dissector->offset[key->key_id] = key->offset;
	}

	/* Ensure that the dissector always includes basic key. That way
	 * we are able to avoid handling lack of it in fast path.
	 */
	BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
					    FLOW_DISSECTOR_KEY_BASIC));
}
EXPORT_SYMBOL(skb_flow_dissector_init);

/**
 * __skb_flow_get_ports - extract the upper layer ports and return them
 * @skb: sk_buff to extract the ports from
 * @thoff: transport header offset
 * @ip_proto: protocol for which to get port offset
 * @data: raw buffer pointer to the packet, if NULL use skb->data
 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 *
 * The function will try to retrieve the ports at offset thoff + poff where poff
 * is the protocol port offset returned from proto_ports_offset
 */
__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
			    void *data, int hlen)
{
	int poff = proto_ports_offset(ip_proto);

	if (!data) {
		data = skb->data;
		hlen = skb_headlen(skb);
	}

	if (poff >= 0) {
		__be32 *ports, _ports;

		ports = __skb_header_pointer(skb, thoff + poff,
					     sizeof(_ports), data, hlen, &_ports);
		if (ports)
			return *ports;
	}

	return 0;
}
EXPORT_SYMBOL(__skb_flow_get_ports);

/**
 * __skb_flow_dissect - extract the flow_keys struct and return it
 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
 * @flow_dissector: list of keys to dissect
 * @target_container: target structure to put dissected values into
 * @data: raw buffer pointer to the packet, if NULL use skb->data
 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 *
 * The function will try to retrieve individual keys into target specified
 * by flow_dissector from either the skbuff or a raw buffer specified by the
 * rest parameters.
 *
 * Caller must take care of zeroing target container memory.
 */
bool __skb_flow_dissect(const struct sk_buff *skb,
			struct flow_dissector *flow_dissector,
			void *target_container,
			void *data, __be16 proto, int nhoff, int hlen)
{
	struct flow_dissector_key_basic *key_basic;
	struct flow_dissector_key_addrs *key_addrs;
	struct flow_dissector_key_ports *key_ports;
	u8 ip_proto;

	if (!data) {
		data = skb->data;
		proto = skb->protocol;
		nhoff = skb_network_offset(skb);
		hlen = skb_headlen(skb);
	}

	/* It is ensured by skb_flow_dissector_init() that basic key will
	 * be always present.
	 */
	key_basic = skb_flow_dissector_target(flow_dissector,
					      FLOW_DISSECTOR_KEY_BASIC,
					      target_container);

again:
	switch (proto) {
	case htons(ETH_P_IP): {
		const struct iphdr *iph;
		struct iphdr _iph;
ip:
		iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
		if (!iph || iph->ihl < 5)
			return false;
		nhoff += iph->ihl * 4;

		ip_proto = iph->protocol;
		if (ip_is_fragment(iph))
			ip_proto = 0;

		if (!skb_flow_dissector_uses_key(flow_dissector,
						 FLOW_DISSECTOR_KEY_IPV4_ADDRS))
			break;
		key_addrs = skb_flow_dissector_target(flow_dissector,
						      FLOW_DISSECTOR_KEY_IPV4_ADDRS,
						      target_container);
		memcpy(key_addrs, &iph->saddr, sizeof(*key_addrs));
		break;
	}
	case htons(ETH_P_IPV6): {
		const struct ipv6hdr *iph;
		struct ipv6hdr _iph;
		__be32 flow_label;

ipv6:
		iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
		if (!iph)
			return false;

		ip_proto = iph->nexthdr;
		nhoff += sizeof(struct ipv6hdr);

		if (skb_flow_dissector_uses_key(flow_dissector,
						FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
			key_addrs = skb_flow_dissector_target(flow_dissector,
							      FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
							      target_container);

			key_addrs->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
			key_addrs->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
			goto flow_label;
		}
		if (skb_flow_dissector_uses_key(flow_dissector,
						FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
			struct flow_dissector_key_ipv6_addrs *key_ipv6_addrs;

			key_ipv6_addrs = skb_flow_dissector_target(flow_dissector,
								   FLOW_DISSECTOR_KEY_IPV6_ADDRS,
								   target_container);

			memcpy(key_ipv6_addrs, &iph->saddr, sizeof(*key_ipv6_addrs));
			goto flow_label;
		}
		break;
flow_label:
		flow_label = ip6_flowlabel(iph);
		if (flow_label) {
			/* Awesome, IPv6 packet has a flow label so we can
			 * use that to represent the ports without any
			 * further dissection.
			 */

			key_basic->n_proto = proto;
			key_basic->ip_proto = ip_proto;
			key_basic->thoff = (u16)nhoff;

			if (!skb_flow_dissector_uses_key(flow_dissector,
							 FLOW_DISSECTOR_KEY_PORTS))
				break;
			key_ports = skb_flow_dissector_target(flow_dissector,
							      FLOW_DISSECTOR_KEY_PORTS,
							      target_container);
			key_ports->ports = flow_label;

			return true;
		}

		break;
	}
	case htons(ETH_P_8021AD):
	case htons(ETH_P_8021Q): {
		const struct vlan_hdr *vlan;
		struct vlan_hdr _vlan;

		vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan);
		if (!vlan)
			return false;

		proto = vlan->h_vlan_encapsulated_proto;
		nhoff += sizeof(*vlan);
		goto again;
	}
	case htons(ETH_P_PPP_SES): {
		struct {
			struct pppoe_hdr hdr;
			__be16 proto;
		} *hdr, _hdr;
		hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
		if (!hdr)
			return false;
		proto = hdr->proto;
		nhoff += PPPOE_SES_HLEN;
		switch (proto) {
		case htons(PPP_IP):
			goto ip;
		case htons(PPP_IPV6):
			goto ipv6;
		default:
			return false;
		}
	}
	case htons(ETH_P_TIPC): {
		struct {
			__be32 pre[3];
			__be32 srcnode;
		} *hdr, _hdr;
		hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
		if (!hdr)
			return false;
		key_basic->n_proto = proto;
		key_basic->thoff = (u16)nhoff;

		if (skb_flow_dissector_uses_key(flow_dissector,
						FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
			return true;
			key_addrs = skb_flow_dissector_target(flow_dissector,
							      FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
							      target_container);
			key_addrs->src = hdr->srcnode;
			key_addrs->dst = 0;
		}
		return true;
	}
	case htons(ETH_P_FCOE):
		key_basic->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
		/* fall through */
	default:
		return false;
	}

	switch (ip_proto) {
	case IPPROTO_GRE: {
		struct gre_hdr {
			__be16 flags;
			__be16 proto;
		} *hdr, _hdr;

		hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
		if (!hdr)
			return false;
		/*
		 * Only look inside GRE if version zero and no
		 * routing
		 */
		if (!(hdr->flags & (GRE_VERSION|GRE_ROUTING))) {
			proto = hdr->proto;
			nhoff += 4;
			if (hdr->flags & GRE_CSUM)
				nhoff += 4;
			if (hdr->flags & GRE_KEY)
				nhoff += 4;
			if (hdr->flags & GRE_SEQ)
				nhoff += 4;
			if (proto == htons(ETH_P_TEB)) {
				const struct ethhdr *eth;
				struct ethhdr _eth;

				eth = __skb_header_pointer(skb, nhoff,
							   sizeof(_eth),
							   data, hlen, &_eth);
				if (!eth)
					return false;
				proto = eth->h_proto;
				nhoff += sizeof(*eth);
			}
			goto again;
		}
		break;
	}
	case IPPROTO_IPIP:
		proto = htons(ETH_P_IP);
		goto ip;
	case IPPROTO_IPV6:
		proto = htons(ETH_P_IPV6);
		goto ipv6;
	default:
		break;
	}

	/* It is ensured by skb_flow_dissector_init() that basic key will
	 * be always present.
	 */
	key_basic = skb_flow_dissector_target(flow_dissector,
					      FLOW_DISSECTOR_KEY_BASIC,
					      target_container);
	key_basic->n_proto = proto;
	key_basic->ip_proto = ip_proto;
	key_basic->thoff = (u16) nhoff;

	if (skb_flow_dissector_uses_key(flow_dissector,
					FLOW_DISSECTOR_KEY_PORTS)) {
		key_ports = skb_flow_dissector_target(flow_dissector,
						      FLOW_DISSECTOR_KEY_PORTS,
						      target_container);
		key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
							data, hlen);
	}

	return true;
}
EXPORT_SYMBOL(__skb_flow_dissect);

static u32 hashrnd __read_mostly;
static __always_inline void __flow_hash_secret_init(void)
{
	net_get_random_once(&hashrnd, sizeof(hashrnd));
}

static __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c, u32 keyval)
{
	return jhash_3words(a, b, c, keyval);
}

static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
{
	u32 hash;

	/* get a consistent hash (same value on both flow directions) */
	if (((__force u32)keys->addrs.dst < (__force u32)keys->addrs.src) ||
	    (((__force u32)keys->addrs.dst == (__force u32)keys->addrs.src) &&
	     ((__force u16)keys->ports.port16[1] < (__force u16)keys->ports.port16[0]))) {
		swap(keys->addrs.dst, keys->addrs.src);
		swap(keys->ports.port16[0], keys->ports.port16[1]);
	}

	hash = __flow_hash_3words((__force u32)keys->addrs.dst,
				  (__force u32)keys->addrs.src,
				  (__force u32)keys->ports.ports,
				  keyval);
	if (!hash)
		hash = 1;

	return hash;
}

u32 flow_hash_from_keys(struct flow_keys *keys)
{
	__flow_hash_secret_init();
	return __flow_hash_from_keys(keys, hashrnd);
}
EXPORT_SYMBOL(flow_hash_from_keys);

static inline u32 ___skb_get_hash(const struct sk_buff *skb,
				  struct flow_keys *keys, u32 keyval)
{
	if (!skb_flow_dissect_flow_keys(skb, keys))
		return 0;

	return __flow_hash_from_keys(keys, keyval);
}

struct _flow_keys_digest_data {
	__be16	n_proto;
	u8	ip_proto;
	u8	padding;
	__be32	ports;
	__be32	src;
	__be32	dst;
};

void make_flow_keys_digest(struct flow_keys_digest *digest,
			   const struct flow_keys *flow)
{
	struct _flow_keys_digest_data *data =
	    (struct _flow_keys_digest_data *)digest;

	BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));

	memset(digest, 0, sizeof(*digest));

	data->n_proto = flow->basic.n_proto;
	data->ip_proto = flow->basic.ip_proto;
	data->ports = flow->ports.ports;
	data->src = flow->addrs.src;
	data->dst = flow->addrs.dst;
}
EXPORT_SYMBOL(make_flow_keys_digest);

/**
 * __skb_get_hash: calculate a flow hash
 * @skb: sk_buff to calculate flow hash from
 *
 * This function calculates a flow hash based on src/dst addresses
 * and src/dst port numbers.  Sets hash in skb to non-zero hash value
 * on success, zero indicates no valid hash.  Also, sets l4_hash in skb
 * if hash is a canonical 4-tuple hash over transport ports.
 */
void __skb_get_hash(struct sk_buff *skb)
{
	struct flow_keys keys;
	u32 hash;

	__flow_hash_secret_init();

	hash = ___skb_get_hash(skb, &keys, hashrnd);
	if (!hash)
		return;
	if (keys.ports.ports)
		skb->l4_hash = 1;
	skb->sw_hash = 1;
	skb->hash = hash;
}
EXPORT_SYMBOL(__skb_get_hash);

__u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb)
{
	struct flow_keys keys;

	return ___skb_get_hash(skb, &keys, perturb);
}
EXPORT_SYMBOL(skb_get_hash_perturb);

u32 __skb_get_poff(const struct sk_buff *skb, void *data,
		   const struct flow_keys *keys, int hlen)
{
	u32 poff = keys->basic.thoff;

	switch (keys->basic.ip_proto) {
	case IPPROTO_TCP: {
		/* access doff as u8 to avoid unaligned access */
		const u8 *doff;
		u8 _doff;

		doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
					    data, hlen, &_doff);
		if (!doff)
			return poff;

		poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
		break;
	}
	case IPPROTO_UDP:
	case IPPROTO_UDPLITE:
		poff += sizeof(struct udphdr);
		break;
	/* For the rest, we do not really care about header
	 * extensions at this point for now.
	 */
	case IPPROTO_ICMP:
		poff += sizeof(struct icmphdr);
		break;
	case IPPROTO_ICMPV6:
		poff += sizeof(struct icmp6hdr);
		break;
	case IPPROTO_IGMP:
		poff += sizeof(struct igmphdr);
		break;
	case IPPROTO_DCCP:
		poff += sizeof(struct dccp_hdr);
		break;
	case IPPROTO_SCTP:
		poff += sizeof(struct sctphdr);
		break;
	}

	return poff;
}

/**
 * skb_get_poff - get the offset to the payload
 * @skb: sk_buff to get the payload offset from
 *
 * The function will get the offset to the payload as far as it could
 * be dissected.  The main user is currently BPF, so that we can dynamically
 * truncate packets without needing to push actual payload to the user
 * space and can analyze headers only, instead.
 */
u32 skb_get_poff(const struct sk_buff *skb)
{
	struct flow_keys keys;

	if (!skb_flow_dissect_flow_keys(skb, &keys))
		return 0;

	return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
}

static const struct flow_dissector_key flow_keys_dissector_keys[] = {
	{
		.key_id = FLOW_DISSECTOR_KEY_BASIC,
		.offset = offsetof(struct flow_keys, basic),
	},
	{
		.key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
		.offset = offsetof(struct flow_keys, addrs),
	},
	{
		.key_id = FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
		.offset = offsetof(struct flow_keys, addrs),
	},
	{
		.key_id = FLOW_DISSECTOR_KEY_PORTS,
		.offset = offsetof(struct flow_keys, ports),
	},
};

static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = {
	{
		.key_id = FLOW_DISSECTOR_KEY_BASIC,
		.offset = offsetof(struct flow_keys, basic),
	},
};

struct flow_dissector flow_keys_dissector __read_mostly;
EXPORT_SYMBOL(flow_keys_dissector);

struct flow_dissector flow_keys_buf_dissector __read_mostly;

static int __init init_default_flow_dissectors(void)
{
	skb_flow_dissector_init(&flow_keys_dissector,
				flow_keys_dissector_keys,
				ARRAY_SIZE(flow_keys_dissector_keys));
	skb_flow_dissector_init(&flow_keys_buf_dissector,
				flow_keys_buf_dissector_keys,
				ARRAY_SIZE(flow_keys_buf_dissector_keys));
	return 0;
}

late_initcall_sync(init_default_flow_dissectors);
Commit	Line	Data
fbff949e	1	#include <linux/kernel.h>
0744dd00	2	#include <linux/skbuff.h>
c452ed70	3	#include <linux/export.h>
0744dd00 ED	4	#include <linux/ip.h>
	5	#include <linux/ipv6.h>
	6	#include <linux/if_vlan.h>
	7	#include <net/ip.h>
ddbe5032	8	#include <net/ipv6.h>
f77668dc DB	9	#include <linux/igmp.h>
	10	#include <linux/icmp.h>
	11	#include <linux/sctp.h>
	12	#include <linux/dccp.h>
0744dd00 ED	13	#include <linux/if_tunnel.h>
	14	#include <linux/if_pppox.h>
	15	#include <linux/ppp_defs.h>
06635a35	16	#include <linux/stddef.h>
1bd758eb	17	#include <net/flow_dissector.h>
56193d1b	18	#include <scsi/fc/fc_fcoe.h>
0744dd00	19
fbff949e JP	20	static bool skb_flow_dissector_uses_key(struct flow_dissector *flow_dissector,
	21	enum flow_dissector_key_id key_id)
	22	{
	23	return flow_dissector->used_keys & (1 << key_id);
	24	}
	25
	26	static void skb_flow_dissector_set_key(struct flow_dissector *flow_dissector,
	27	enum flow_dissector_key_id key_id)
	28	{
	29	flow_dissector->used_keys \|= (1 << key_id);
	30	}
	31
	32	static void skb_flow_dissector_target(struct flow_dissector flow_dissector,
	33	enum flow_dissector_key_id key_id,
	34	void *target_container)
	35	{
	36	return ((char *) target_container) + flow_dissector->offset[key_id];
	37	}
	38
	39	void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
	40	const struct flow_dissector_key *key,
	41	unsigned int key_count)
	42	{
	43	unsigned int i;
	44
	45	memset(flow_dissector, 0, sizeof(*flow_dissector));
	46
	47	for (i = 0; i < key_count; i++, key++) {
	48	/* User should make sure that every key target offset is withing
	49	* boundaries of unsigned short.
	50	*/
	51	BUG_ON(key->offset > USHRT_MAX);
	52	BUG_ON(skb_flow_dissector_uses_key(flow_dissector,
	53	key->key_id));
	54
	55	skb_flow_dissector_set_key(flow_dissector, key->key_id);
	56	flow_dissector->offset[key->key_id] = key->offset;
	57	}
	58
	59	/* Ensure that the dissector always includes basic key. That way
	60	* we are able to avoid handling lack of it in fast path.
	61	*/
	62	BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
	63	FLOW_DISSECTOR_KEY_BASIC));
	64	}
	65	EXPORT_SYMBOL(skb_flow_dissector_init);
	66
357afe9c	67	/**
6451b3f5 WC	68	* __skb_flow_get_ports - extract the upper layer ports and return them
6451b3f5 WC	69	* @skb: sk_buff to extract the ports from
357afe9c NA	70	* @thoff: transport header offset
357afe9c NA	71	* @ip_proto: protocol for which to get port offset
6451b3f5 WC	72	* @data: raw buffer pointer to the packet, if NULL use skb->data
6451b3f5 WC	73	* @hlen: packet header length, if @data is NULL use skb_headlen(skb)
357afe9c NA	74	*
	75	* The function will try to retrieve the ports at offset thoff + poff where poff
	76	* is the protocol port offset returned from proto_ports_offset
	77	*/
690e36e7 DM	78	__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
690e36e7 DM	79	void *data, int hlen)
357afe9c NA	80	{
	81	int poff = proto_ports_offset(ip_proto);
	82
690e36e7 DM	83	if (!data) {
	84	data = skb->data;
	85	hlen = skb_headlen(skb);
	86	}
	87
357afe9c NA	88	if (poff >= 0) {
	89	__be32 *ports, _ports;
	90
690e36e7 DM	91	ports = __skb_header_pointer(skb, thoff + poff,
690e36e7 DM	92	sizeof(_ports), data, hlen, &_ports);
357afe9c NA	93	if (ports)
	94	return *ports;
	95	}
	96
	97	return 0;
	98	}
690e36e7	99	EXPORT_SYMBOL(__skb_flow_get_ports);
357afe9c	100
453a940e WC	101	/**
	102	* __skb_flow_dissect - extract the flow_keys struct and return it
	103	* @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
06635a35 JP	104	* @flow_dissector: list of keys to dissect
06635a35 JP	105	* @target_container: target structure to put dissected values into
453a940e WC	106	* @data: raw buffer pointer to the packet, if NULL use skb->data
	107	* @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
	108	* @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
	109	* @hlen: packet header length, if @data is NULL use skb_headlen(skb)
	110	*
06635a35 JP	111	* The function will try to retrieve individual keys into target specified
	112	* by flow_dissector from either the skbuff or a raw buffer specified by the
	113	* rest parameters.
	114	*
	115	* Caller must take care of zeroing target container memory.
453a940e	116	*/
06635a35 JP	117	bool __skb_flow_dissect(const struct sk_buff *skb,
	118	struct flow_dissector *flow_dissector,
	119	void *target_container,
453a940e	120	void *data, __be16 proto, int nhoff, int hlen)
0744dd00	121	{
06635a35 JP	122	struct flow_dissector_key_basic *key_basic;
	123	struct flow_dissector_key_addrs *key_addrs;
	124	struct flow_dissector_key_ports *key_ports;
0744dd00	125	u8 ip_proto;
0744dd00	126
690e36e7 DM	127	if (!data) {
690e36e7 DM	128	data = skb->data;
453a940e WC	129	proto = skb->protocol;
453a940e WC	130	nhoff = skb_network_offset(skb);
690e36e7 DM	131	hlen = skb_headlen(skb);
	132	}
	133
06635a35 JP	134	/* It is ensured by skb_flow_dissector_init() that basic key will
	135	* be always present.
	136	*/
	137	key_basic = skb_flow_dissector_target(flow_dissector,
	138	FLOW_DISSECTOR_KEY_BASIC,
	139	target_container);
0744dd00 ED	140
	141	again:
	142	switch (proto) {
2b8837ae	143	case htons(ETH_P_IP): {
0744dd00 ED	144	const struct iphdr *iph;
	145	struct iphdr _iph;
	146	ip:
690e36e7	147	iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
6f092343	148	if (!iph \|\| iph->ihl < 5)
0744dd00	149	return false;
3797d3e8	150	nhoff += iph->ihl * 4;
0744dd00	151
3797d3e8	152	ip_proto = iph->protocol;
0744dd00 ED	153	if (ip_is_fragment(iph))
0744dd00 ED	154	ip_proto = 0;
3797d3e8	155
06635a35 JP	156	if (!skb_flow_dissector_uses_key(flow_dissector,
06635a35 JP	157	FLOW_DISSECTOR_KEY_IPV4_ADDRS))
5af7fb6e	158	break;
06635a35 JP	159	key_addrs = skb_flow_dissector_target(flow_dissector,
	160	FLOW_DISSECTOR_KEY_IPV4_ADDRS,
	161	target_container);
	162	memcpy(key_addrs, &iph->saddr, sizeof(*key_addrs));
0744dd00 ED	163	break;
0744dd00 ED	164	}
2b8837ae	165	case htons(ETH_P_IPV6): {
0744dd00 ED	166	const struct ipv6hdr *iph;
0744dd00 ED	167	struct ipv6hdr _iph;
19469a87 TH	168	__be32 flow_label;
19469a87 TH	169
0744dd00	170	ipv6:
690e36e7	171	iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
0744dd00 ED	172	if (!iph)
	173	return false;
	174
	175	ip_proto = iph->nexthdr;
0744dd00	176	nhoff += sizeof(struct ipv6hdr);
19469a87	177
b924933c JP	178	if (skb_flow_dissector_uses_key(flow_dissector,
	179	FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
	180	key_addrs = skb_flow_dissector_target(flow_dissector,
	181	FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
	182	target_container);
56193d1b	183
b924933c JP	184	key_addrs->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
	185	key_addrs->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
	186	goto flow_label;
	187	}
	188	if (skb_flow_dissector_uses_key(flow_dissector,
	189	FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
	190	struct flow_dissector_key_ipv6_addrs *key_ipv6_addrs;
	191
	192	key_ipv6_addrs = skb_flow_dissector_target(flow_dissector,
	193	FLOW_DISSECTOR_KEY_IPV6_ADDRS,
	194	target_container);
5af7fb6e	195
b924933c JP	196	memcpy(key_ipv6_addrs, &iph->saddr, sizeof(*key_ipv6_addrs));
	197	goto flow_label;
	198	}
	199	break;
	200	flow_label:
19469a87 TH	201	flow_label = ip6_flowlabel(iph);
	202	if (flow_label) {
	203	/* Awesome, IPv6 packet has a flow label so we can
	204	* use that to represent the ports without any
	205	* further dissection.
	206	*/
06635a35 JP	207
	208	key_basic->n_proto = proto;
	209	key_basic->ip_proto = ip_proto;
	210	key_basic->thoff = (u16)nhoff;
	211
	212	if (!skb_flow_dissector_uses_key(flow_dissector,
	213	FLOW_DISSECTOR_KEY_PORTS))
	214	break;
	215	key_ports = skb_flow_dissector_target(flow_dissector,
	216	FLOW_DISSECTOR_KEY_PORTS,
	217	target_container);
	218	key_ports->ports = flow_label;
19469a87 TH	219
	220	return true;
	221	}
	222
0744dd00 ED	223	break;
0744dd00 ED	224	}
2b8837ae JP	225	case htons(ETH_P_8021AD):
2b8837ae JP	226	case htons(ETH_P_8021Q): {
0744dd00 ED	227	const struct vlan_hdr *vlan;
	228	struct vlan_hdr _vlan;
	229
690e36e7	230	vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan);
0744dd00 ED	231	if (!vlan)
	232	return false;
	233
	234	proto = vlan->h_vlan_encapsulated_proto;
	235	nhoff += sizeof(*vlan);
	236	goto again;
	237	}
2b8837ae	238	case htons(ETH_P_PPP_SES): {
0744dd00 ED	239	struct {
	240	struct pppoe_hdr hdr;
	241	__be16 proto;
	242	} *hdr, _hdr;
690e36e7	243	hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
0744dd00 ED	244	if (!hdr)
	245	return false;
	246	proto = hdr->proto;
	247	nhoff += PPPOE_SES_HLEN;
	248	switch (proto) {
2b8837ae	249	case htons(PPP_IP):
0744dd00	250	goto ip;
2b8837ae	251	case htons(PPP_IPV6):
0744dd00 ED	252	goto ipv6;
	253	default:
	254	return false;
	255	}
	256	}
08bfc9cb EH	257	case htons(ETH_P_TIPC): {
	258	struct {
	259	__be32 pre[3];
	260	__be32 srcnode;
	261	} *hdr, _hdr;
	262	hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
	263	if (!hdr)
	264	return false;
06635a35 JP	265	key_basic->n_proto = proto;
	266	key_basic->thoff = (u16)nhoff;
	267
	268	if (skb_flow_dissector_uses_key(flow_dissector,
	269	FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
	270	return true;
	271	key_addrs = skb_flow_dissector_target(flow_dissector,
	272	FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
	273	target_container);
	274	key_addrs->src = hdr->srcnode;
	275	key_addrs->dst = 0;
	276	}
08bfc9cb EH	277	return true;
08bfc9cb EH	278	}
56193d1b	279	case htons(ETH_P_FCOE):
06635a35	280	key_basic->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
56193d1b	281	/* fall through */
0744dd00 ED	282	default:
	283	return false;
	284	}
	285
	286	switch (ip_proto) {
	287	case IPPROTO_GRE: {
	288	struct gre_hdr {
	289	__be16 flags;
	290	__be16 proto;
	291	} *hdr, _hdr;
	292
690e36e7	293	hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
0744dd00 ED	294	if (!hdr)
	295	return false;
	296	/*
	297	* Only look inside GRE if version zero and no
	298	* routing
	299	*/
	300	if (!(hdr->flags & (GRE_VERSION\|GRE_ROUTING))) {
	301	proto = hdr->proto;
	302	nhoff += 4;
	303	if (hdr->flags & GRE_CSUM)
	304	nhoff += 4;
	305	if (hdr->flags & GRE_KEY)
	306	nhoff += 4;
	307	if (hdr->flags & GRE_SEQ)
	308	nhoff += 4;
e1733de2 MD	309	if (proto == htons(ETH_P_TEB)) {
	310	const struct ethhdr *eth;
	311	struct ethhdr _eth;
	312
690e36e7 DM	313	eth = __skb_header_pointer(skb, nhoff,
	314	sizeof(_eth),
	315	data, hlen, &_eth);
e1733de2 MD	316	if (!eth)
	317	return false;
	318	proto = eth->h_proto;
	319	nhoff += sizeof(*eth);
	320	}
0744dd00 ED	321	goto again;
	322	}
	323	break;
	324	}
	325	case IPPROTO_IPIP:
fca41895 TH	326	proto = htons(ETH_P_IP);
fca41895 TH	327	goto ip;
b438f940 TH	328	case IPPROTO_IPV6:
	329	proto = htons(ETH_P_IPV6);
	330	goto ipv6;
0744dd00 ED	331	default:
	332	break;
	333	}
	334
06635a35 JP	335	/* It is ensured by skb_flow_dissector_init() that basic key will
	336	* be always present.
	337	*/
	338	key_basic = skb_flow_dissector_target(flow_dissector,
	339	FLOW_DISSECTOR_KEY_BASIC,
	340	target_container);
	341	key_basic->n_proto = proto;
	342	key_basic->ip_proto = ip_proto;
	343	key_basic->thoff = (u16) nhoff;
	344
	345	if (skb_flow_dissector_uses_key(flow_dissector,
	346	FLOW_DISSECTOR_KEY_PORTS)) {
	347	key_ports = skb_flow_dissector_target(flow_dissector,
	348	FLOW_DISSECTOR_KEY_PORTS,
	349	target_container);
	350	key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
	351	data, hlen);
	352	}
5af7fb6e	353
0744dd00 ED	354	return true;
0744dd00 ED	355	}
690e36e7	356	EXPORT_SYMBOL(__skb_flow_dissect);
441d9d32 CW	357
441d9d32 CW	358	static u32 hashrnd __read_mostly;
66415cf8 HFS	359	static __always_inline void __flow_hash_secret_init(void)
	360	{
	361	net_get_random_once(&hashrnd, sizeof(hashrnd));
	362	}
	363
50fb7992	364	static __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c, u32 keyval)
66415cf8	365	{
50fb7992	366	return jhash_3words(a, b, c, keyval);
66415cf8 HFS	367	}
66415cf8 HFS	368
50fb7992	369	static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
5ed20a68 TH	370	{
	371	u32 hash;
	372
	373	/* get a consistent hash (same value on both flow directions) */
06635a35 JP	374	if (((__force u32)keys->addrs.dst < (__force u32)keys->addrs.src) \|\|
	375	(((__force u32)keys->addrs.dst == (__force u32)keys->addrs.src) &&
	376	((__force u16)keys->ports.port16[1] < (__force u16)keys->ports.port16[0]))) {
	377	swap(keys->addrs.dst, keys->addrs.src);
	378	swap(keys->ports.port16[0], keys->ports.port16[1]);
5ed20a68 TH	379	}
5ed20a68 TH	380
06635a35 JP	381	hash = __flow_hash_3words((__force u32)keys->addrs.dst,
	382	(__force u32)keys->addrs.src,
	383	(__force u32)keys->ports.ports,
50fb7992	384	keyval);
5ed20a68 TH	385	if (!hash)
	386	hash = 1;
	387
	388	return hash;
	389	}
	390
	391	u32 flow_hash_from_keys(struct flow_keys *keys)
	392	{
50fb7992 TH	393	__flow_hash_secret_init();
50fb7992 TH	394	return __flow_hash_from_keys(keys, hashrnd);
5ed20a68 TH	395	}
	396	EXPORT_SYMBOL(flow_hash_from_keys);
	397
50fb7992 TH	398	static inline u32 ___skb_get_hash(const struct sk_buff *skb,
	399	struct flow_keys *keys, u32 keyval)
	400	{
06635a35	401	if (!skb_flow_dissect_flow_keys(skb, keys))
50fb7992 TH	402	return 0;
	403
	404	return __flow_hash_from_keys(keys, keyval);
	405	}
	406
2f59e1eb TH	407	struct _flow_keys_digest_data {
	408	__be16 n_proto;
	409	u8 ip_proto;
	410	u8 padding;
	411	__be32 ports;
	412	__be32 src;
	413	__be32 dst;
	414	};
	415
	416	void make_flow_keys_digest(struct flow_keys_digest *digest,
	417	const struct flow_keys *flow)
	418	{
	419	struct _flow_keys_digest_data *data =
	420	(struct _flow_keys_digest_data *)digest;
	421
	422	BUILD_BUG_ON(sizeof(data) > sizeof(digest));
	423
	424	memset(digest, 0, sizeof(*digest));
	425
06635a35 JP	426	data->n_proto = flow->basic.n_proto;
	427	data->ip_proto = flow->basic.ip_proto;
	428	data->ports = flow->ports.ports;
	429	data->src = flow->addrs.src;
	430	data->dst = flow->addrs.dst;
2f59e1eb TH	431	}
	432	EXPORT_SYMBOL(make_flow_keys_digest);
	433
d4fd3275 JP	434	/**
	435	* __skb_get_hash: calculate a flow hash
	436	* @skb: sk_buff to calculate flow hash from
	437	*
	438	* This function calculates a flow hash based on src/dst addresses
61b905da TH	439	* and src/dst port numbers. Sets hash in skb to non-zero hash value
61b905da TH	440	* on success, zero indicates no valid hash. Also, sets l4_hash in skb
441d9d32 CW	441	* if hash is a canonical 4-tuple hash over transport ports.
441d9d32 CW	442	*/
3958afa1	443	void __skb_get_hash(struct sk_buff *skb)
441d9d32 CW	444	{
441d9d32 CW	445	struct flow_keys keys;
50fb7992	446	u32 hash;
441d9d32	447
50fb7992 TH	448	__flow_hash_secret_init();
	449
	450	hash = ___skb_get_hash(skb, &keys, hashrnd);
	451	if (!hash)
441d9d32	452	return;
06635a35	453	if (keys.ports.ports)
61b905da	454	skb->l4_hash = 1;
a3b18ddb	455	skb->sw_hash = 1;
50fb7992	456	skb->hash = hash;
441d9d32	457	}
3958afa1	458	EXPORT_SYMBOL(__skb_get_hash);
441d9d32	459
50fb7992 TH	460	__u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb)
	461	{
	462	struct flow_keys keys;
	463
	464	return ___skb_get_hash(skb, &keys, perturb);
	465	}
	466	EXPORT_SYMBOL(skb_get_hash_perturb);
	467
56193d1b AD	468	u32 __skb_get_poff(const struct sk_buff skb, void data,
56193d1b AD	469	const struct flow_keys *keys, int hlen)
f77668dc	470	{
06635a35	471	u32 poff = keys->basic.thoff;
f77668dc	472
06635a35	473	switch (keys->basic.ip_proto) {
f77668dc	474	case IPPROTO_TCP: {
5af7fb6e AD	475	/* access doff as u8 to avoid unaligned access */
	476	const u8 *doff;
	477	u8 _doff;
f77668dc	478
5af7fb6e AD	479	doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
	480	data, hlen, &_doff);
	481	if (!doff)
f77668dc DB	482	return poff;
f77668dc DB	483
5af7fb6e	484	poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
f77668dc DB	485	break;
	486	}
	487	case IPPROTO_UDP:
	488	case IPPROTO_UDPLITE:
	489	poff += sizeof(struct udphdr);
	490	break;
	491	/* For the rest, we do not really care about header
	492	* extensions at this point for now.
	493	*/
	494	case IPPROTO_ICMP:
	495	poff += sizeof(struct icmphdr);
	496	break;
	497	case IPPROTO_ICMPV6:
	498	poff += sizeof(struct icmp6hdr);
	499	break;
	500	case IPPROTO_IGMP:
	501	poff += sizeof(struct igmphdr);
	502	break;
	503	case IPPROTO_DCCP:
	504	poff += sizeof(struct dccp_hdr);
	505	break;
	506	case IPPROTO_SCTP:
	507	poff += sizeof(struct sctphdr);
	508	break;
	509	}
	510
	511	return poff;
	512	}
	513
0db89b8b JP	514	/**
	515	* skb_get_poff - get the offset to the payload
	516	* @skb: sk_buff to get the payload offset from
	517	*
	518	* The function will get the offset to the payload as far as it could
	519	* be dissected. The main user is currently BPF, so that we can dynamically
56193d1b AD	520	* truncate packets without needing to push actual payload to the user
	521	* space and can analyze headers only, instead.
	522	*/
	523	u32 skb_get_poff(const struct sk_buff *skb)
	524	{
	525	struct flow_keys keys;
	526
06635a35	527	if (!skb_flow_dissect_flow_keys(skb, &keys))
56193d1b AD	528	return 0;
	529
	530	return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
	531	}
06635a35 JP	532
	533	static const struct flow_dissector_key flow_keys_dissector_keys[] = {
	534	{
	535	.key_id = FLOW_DISSECTOR_KEY_BASIC,
	536	.offset = offsetof(struct flow_keys, basic),
	537	},
	538	{
	539	.key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
	540	.offset = offsetof(struct flow_keys, addrs),
	541	},
	542	{
	543	.key_id = FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
	544	.offset = offsetof(struct flow_keys, addrs),
	545	},
	546	{
	547	.key_id = FLOW_DISSECTOR_KEY_PORTS,
	548	.offset = offsetof(struct flow_keys, ports),
	549	},
	550	};
	551
	552	static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = {
	553	{
	554	.key_id = FLOW_DISSECTOR_KEY_BASIC,
	555	.offset = offsetof(struct flow_keys, basic),
	556	},
	557	};
	558
	559	struct flow_dissector flow_keys_dissector __read_mostly;
	560	EXPORT_SYMBOL(flow_keys_dissector);
	561
	562	struct flow_dissector flow_keys_buf_dissector __read_mostly;
	563
	564	static int __init init_default_flow_dissectors(void)
	565	{
	566	skb_flow_dissector_init(&flow_keys_dissector,
	567	flow_keys_dissector_keys,
	568	ARRAY_SIZE(flow_keys_dissector_keys));
	569	skb_flow_dissector_init(&flow_keys_buf_dissector,
	570	flow_keys_buf_dissector_keys,
	571	ARRAY_SIZE(flow_keys_buf_dissector_keys));
	572	return 0;
	573	}
	574
	575	late_initcall_sync(init_default_flow_dissectors);