[GitHub/mt8127/android_kernel_alcatel_ttab.git] / crypto / vmac.c

/*
 * Modified to interface to the Linux kernel
 * Copyright (c) 2009, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
 * Place - Suite 330, Boston, MA 02111-1307 USA.
 */

/* --------------------------------------------------------------------------
 * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai.
 * This implementation is herby placed in the public domain.
 * The authors offers no warranty. Use at your own risk.
 * Please send bug reports to the authors.
 * Last modified: 17 APR 08, 1700 PDT
 * ----------------------------------------------------------------------- */

#include <linux/init.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <asm/byteorder.h>
#include <crypto/scatterwalk.h>
#include <crypto/vmac.h>
#include <crypto/internal/hash.h>

/*
 * Constants and masks
 */
#define UINT64_C(x) x##ULL
static const u64 p64   = UINT64_C(0xfffffffffffffeff);	/* 2^64 - 257 prime  */
static const u64 m62   = UINT64_C(0x3fffffffffffffff);	/* 62-bit mask       */
static const u64 m63   = UINT64_C(0x7fffffffffffffff);	/* 63-bit mask       */
static const u64 m64   = UINT64_C(0xffffffffffffffff);	/* 64-bit mask       */
static const u64 mpoly = UINT64_C(0x1fffffff1fffffff);	/* Poly key mask     */

#define pe64_to_cpup le64_to_cpup		/* Prefer little endian */

#ifdef __LITTLE_ENDIAN
#define INDEX_HIGH 1
#define INDEX_LOW 0
#else
#define INDEX_HIGH 0
#define INDEX_LOW 1
#endif

/*
 * The following routines are used in this implementation. They are
 * written via macros to simulate zero-overhead call-by-reference.
 *
 * MUL64: 64x64->128-bit multiplication
 * PMUL64: assumes top bits cleared on inputs
 * ADD128: 128x128->128-bit addition
 */

#define ADD128(rh, rl, ih, il)						\
	do {								\
		u64 _il = (il);						\
		(rl) += (_il);						\
		if ((rl) < (_il))					\
			(rh)++;						\
		(rh) += (ih);						\
	} while (0)

#define MUL32(i1, i2)	((u64)(u32)(i1)*(u32)(i2))

#define PMUL64(rh, rl, i1, i2)	/* Assumes m doesn't overflow */	\
	do {								\
		u64 _i1 = (i1), _i2 = (i2);				\
		u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2);	\
		rh = MUL32(_i1>>32, _i2>>32);				\
		rl = MUL32(_i1, _i2);					\
		ADD128(rh, rl, (m >> 32), (m << 32));			\
	} while (0)

#define MUL64(rh, rl, i1, i2)						\
	do {								\
		u64 _i1 = (i1), _i2 = (i2);				\
		u64 m1 = MUL32(_i1, _i2>>32);				\
		u64 m2 = MUL32(_i1>>32, _i2);				\
		rh = MUL32(_i1>>32, _i2>>32);				\
		rl = MUL32(_i1, _i2);					\
		ADD128(rh, rl, (m1 >> 32), (m1 << 32));			\
		ADD128(rh, rl, (m2 >> 32), (m2 << 32));			\
	} while (0)

/*
 * For highest performance the L1 NH and L2 polynomial hashes should be
 * carefully implemented to take advantage of one's target architecture.
 * Here these two hash functions are defined multiple time; once for
 * 64-bit architectures, once for 32-bit SSE2 architectures, and once
 * for the rest (32-bit) architectures.
 * For each, nh_16 *must* be defined (works on multiples of 16 bytes).
 * Optionally, nh_vmac_nhbytes can be defined (for multiples of
 * VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two
 * NH computations at once).
 */

#ifdef CONFIG_64BIT

#define nh_16(mp, kp, nw, rh, rl)					\
	do {								\
		int i; u64 th, tl;					\
		rh = rl = 0;						\
		for (i = 0; i < nw; i += 2) {				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
			ADD128(rh, rl, th, tl);				\
		}							\
	} while (0)

#define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1)				\
	do {								\
		int i; u64 th, tl;					\
		rh1 = rl1 = rh = rl = 0;				\
		for (i = 0; i < nw; i += 2) {				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+3]);	\
			ADD128(rh1, rl1, th, tl);			\
		}							\
	} while (0)

#if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */
#define nh_vmac_nhbytes(mp, kp, nw, rh, rl)				\
	do {								\
		int i; u64 th, tl;					\
		rh = rl = 0;						\
		for (i = 0; i < nw; i += 8) {				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2],	\
				pe64_to_cpup((mp)+i+3)+(kp)[i+3]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4],	\
				pe64_to_cpup((mp)+i+5)+(kp)[i+5]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6],	\
				pe64_to_cpup((mp)+i+7)+(kp)[i+7]);	\
			ADD128(rh, rl, th, tl);				\
		}							\
	} while (0)

#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1)			\
	do {								\
		int i; u64 th, tl;					\
		rh1 = rl1 = rh = rl = 0;				\
		for (i = 0; i < nw; i += 8) {				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+3]);	\
			ADD128(rh1, rl1, th, tl);			\
			MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2],	\
				pe64_to_cpup((mp)+i+3)+(kp)[i+3]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4],	\
				pe64_to_cpup((mp)+i+3)+(kp)[i+5]);	\
			ADD128(rh1, rl1, th, tl);			\
			MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4],	\
				pe64_to_cpup((mp)+i+5)+(kp)[i+5]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6],	\
				pe64_to_cpup((mp)+i+5)+(kp)[i+7]);	\
			ADD128(rh1, rl1, th, tl);			\
			MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6],	\
				pe64_to_cpup((mp)+i+7)+(kp)[i+7]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8],	\
				pe64_to_cpup((mp)+i+7)+(kp)[i+9]);	\
			ADD128(rh1, rl1, th, tl);			\
		}							\
	} while (0)
#endif

#define poly_step(ah, al, kh, kl, mh, ml)				\
	do {								\
		u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0;		\
		/* compute ab*cd, put bd into result registers */	\
		PMUL64(t3h, t3l, al, kh);				\
		PMUL64(t2h, t2l, ah, kl);				\
		PMUL64(t1h, t1l, ah, 2*kh);				\
		PMUL64(ah, al, al, kl);					\
		/* add 2 * ac to result */				\
		ADD128(ah, al, t1h, t1l);				\
		/* add together ad + bc */				\
		ADD128(t2h, t2l, t3h, t3l);				\
		/* now (ah,al), (t2l,2*t2h) need summing */		\
		/* first add the high registers, carrying into t2h */	\
		ADD128(t2h, ah, z, t2l);				\
		/* double t2h and add top bit of ah */			\
		t2h = 2 * t2h + (ah >> 63);				\
		ah &= m63;						\
		/* now add the low registers */				\
		ADD128(ah, al, mh, ml);					\
		ADD128(ah, al, z, t2h);					\
	} while (0)

#else /* ! CONFIG_64BIT */

#ifndef nh_16
#define nh_16(mp, kp, nw, rh, rl)					\
	do {								\
		u64 t1, t2, m1, m2, t;					\
		int i;							\
		rh = rl = t = 0;					\
		for (i = 0; i < nw; i += 2)  {				\
			t1 = pe64_to_cpup(mp+i) + kp[i];		\
			t2 = pe64_to_cpup(mp+i+1) + kp[i+1];		\
			m2 = MUL32(t1 >> 32, t2);			\
			m1 = MUL32(t1, t2 >> 32);			\
			ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32),	\
				MUL32(t1, t2));				\
			rh += (u64)(u32)(m1 >> 32)			\
				+ (u32)(m2 >> 32);			\
			t += (u64)(u32)m1 + (u32)m2;			\
		}							\
		ADD128(rh, rl, (t >> 32), (t << 32));			\
	} while (0)
#endif

static void poly_step_func(u64 *ahi, u64 *alo,
			const u64 *kh, const u64 *kl,
			const u64 *mh, const u64 *ml)
{
#define a0 (*(((u32 *)alo)+INDEX_LOW))
#define a1 (*(((u32 *)alo)+INDEX_HIGH))
#define a2 (*(((u32 *)ahi)+INDEX_LOW))
#define a3 (*(((u32 *)ahi)+INDEX_HIGH))
#define k0 (*(((u32 *)kl)+INDEX_LOW))
#define k1 (*(((u32 *)kl)+INDEX_HIGH))
#define k2 (*(((u32 *)kh)+INDEX_LOW))
#define k3 (*(((u32 *)kh)+INDEX_HIGH))

	u64 p, q, t;
	u32 t2;

	p = MUL32(a3, k3);
	p += p;
	p += *(u64 *)mh;
	p += MUL32(a0, k2);
	p += MUL32(a1, k1);
	p += MUL32(a2, k0);
	t = (u32)(p);
	p >>= 32;
	p += MUL32(a0, k3);
	p += MUL32(a1, k2);
	p += MUL32(a2, k1);
	p += MUL32(a3, k0);
	t |= ((u64)((u32)p & 0x7fffffff)) << 32;
	p >>= 31;
	p += (u64)(((u32 *)ml)[INDEX_LOW]);
	p += MUL32(a0, k0);
	q =  MUL32(a1, k3);
	q += MUL32(a2, k2);
	q += MUL32(a3, k1);
	q += q;
	p += q;
	t2 = (u32)(p);
	p >>= 32;
	p += (u64)(((u32 *)ml)[INDEX_HIGH]);
	p += MUL32(a0, k1);
	p += MUL32(a1, k0);
	q =  MUL32(a2, k3);
	q += MUL32(a3, k2);
	q += q;
	p += q;
	*(u64 *)(alo) = (p << 32) | t2;
	p >>= 32;
	*(u64 *)(ahi) = p + t;

#undef a0
#undef a1
#undef a2
#undef a3
#undef k0
#undef k1
#undef k2
#undef k3
}

#define poly_step(ah, al, kh, kl, mh, ml)				\
	poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml))

#endif  /* end of specialized NH and poly definitions */

/* At least nh_16 is defined. Defined others as needed here */
#ifndef nh_16_2
#define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2)				\
	do { 								\
		nh_16(mp, kp, nw, rh, rl);				\
		nh_16(mp, ((kp)+2), nw, rh2, rl2);			\
	} while (0)
#endif
#ifndef nh_vmac_nhbytes
#define nh_vmac_nhbytes(mp, kp, nw, rh, rl)				\
	nh_16(mp, kp, nw, rh, rl)
#endif
#ifndef nh_vmac_nhbytes_2
#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2)			\
	do {								\
		nh_vmac_nhbytes(mp, kp, nw, rh, rl);			\
		nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2);		\
	} while (0)
#endif

static void vhash_abort(struct vmac_ctx *ctx)
{
	ctx->polytmp[0] = ctx->polykey[0] ;
	ctx->polytmp[1] = ctx->polykey[1] ;
	ctx->first_block_processed = 0;
}

static u64 l3hash(u64 p1, u64 p2, u64 k1, u64 k2, u64 len)
{
	u64 rh, rl, t, z = 0;

	/* fully reduce (p1,p2)+(len,0) mod p127 */
	t = p1 >> 63;
	p1 &= m63;
	ADD128(p1, p2, len, t);
	/* At this point, (p1,p2) is at most 2^127+(len<<64) */
	t = (p1 > m63) + ((p1 == m63) && (p2 == m64));
	ADD128(p1, p2, z, t);
	p1 &= m63;

	/* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */
	t = p1 + (p2 >> 32);
	t += (t >> 32);
	t += (u32)t > 0xfffffffeu;
	p1 += (t >> 32);
	p2 += (p1 << 32);

	/* compute (p1+k1)%p64 and (p2+k2)%p64 */
	p1 += k1;
	p1 += (0 - (p1 < k1)) & 257;
	p2 += k2;
	p2 += (0 - (p2 < k2)) & 257;

	/* compute (p1+k1)*(p2+k2)%p64 */
	MUL64(rh, rl, p1, p2);
	t = rh >> 56;
	ADD128(t, rl, z, rh);
	rh <<= 8;
	ADD128(t, rl, z, rh);
	t += t << 8;
	rl += t;
	rl += (0 - (rl < t)) & 257;
	rl += (0 - (rl > p64-1)) & 257;
	return rl;
}

static void vhash_update(const unsigned char *m,
			unsigned int mbytes, /* Pos multiple of VMAC_NHBYTES */
			struct vmac_ctx *ctx)
{
	u64 rh, rl, *mptr;
	const u64 *kptr = (u64 *)ctx->nhkey;
	int i;
	u64 ch, cl;
	u64 pkh = ctx->polykey[0];
	u64 pkl = ctx->polykey[1];

	mptr = (u64 *)m;
	i = mbytes / VMAC_NHBYTES;  /* Must be non-zero */

	ch = ctx->polytmp[0];
	cl = ctx->polytmp[1];

	if (!ctx->first_block_processed) {
		ctx->first_block_processed = 1;
		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
		rh &= m62;
		ADD128(ch, cl, rh, rl);
		mptr += (VMAC_NHBYTES/sizeof(u64));
		i--;
	}

	while (i--) {
		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
		rh &= m62;
		poly_step(ch, cl, pkh, pkl, rh, rl);
		mptr += (VMAC_NHBYTES/sizeof(u64));
	}

	ctx->polytmp[0] = ch;
	ctx->polytmp[1] = cl;
}

static u64 vhash(unsigned char m[], unsigned int mbytes,
			u64 *tagl, struct vmac_ctx *ctx)
{
	u64 rh, rl, *mptr;
	const u64 *kptr = (u64 *)ctx->nhkey;
	int i, remaining;
	u64 ch, cl;
	u64 pkh = ctx->polykey[0];
	u64 pkl = ctx->polykey[1];

	mptr = (u64 *)m;
	i = mbytes / VMAC_NHBYTES;
	remaining = mbytes % VMAC_NHBYTES;

	if (ctx->first_block_processed) {
		ch = ctx->polytmp[0];
		cl = ctx->polytmp[1];
	} else if (i) {
		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, ch, cl);
		ch &= m62;
		ADD128(ch, cl, pkh, pkl);
		mptr += (VMAC_NHBYTES/sizeof(u64));
		i--;
	} else if (remaining) {
		nh_16(mptr, kptr, 2*((remaining+15)/16), ch, cl);
		ch &= m62;
		ADD128(ch, cl, pkh, pkl);
		mptr += (VMAC_NHBYTES/sizeof(u64));
		goto do_l3;
	} else {/* Empty String */
		ch = pkh; cl = pkl;
		goto do_l3;
	}

	while (i--) {
		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
		rh &= m62;
		poly_step(ch, cl, pkh, pkl, rh, rl);
		mptr += (VMAC_NHBYTES/sizeof(u64));
	}
	if (remaining) {
		nh_16(mptr, kptr, 2*((remaining+15)/16), rh, rl);
		rh &= m62;
		poly_step(ch, cl, pkh, pkl, rh, rl);
	}

do_l3:
	vhash_abort(ctx);
	remaining *= 8;
	return l3hash(ch, cl, ctx->l3key[0], ctx->l3key[1], remaining);
}

static u64 vmac(unsigned char m[], unsigned int mbytes,
			unsigned char n[16], u64 *tagl,
			struct vmac_ctx_t *ctx)
{
	u64 *in_n, *out_p;
	u64 p, h;
	int i;

	in_n = ctx->__vmac_ctx.cached_nonce;
	out_p = ctx->__vmac_ctx.cached_aes;

	i = n[15] & 1;
	if ((*(u64 *)(n+8) != in_n[1]) || (*(u64 *)(n) != in_n[0])) {
		in_n[0] = *(u64 *)(n);
		in_n[1] = *(u64 *)(n+8);
		((unsigned char *)in_n)[15] &= 0xFE;
		crypto_cipher_encrypt_one(ctx->child,
			(unsigned char *)out_p, (unsigned char *)in_n);

		((unsigned char *)in_n)[15] |= (unsigned char)(1-i);
	}
	p = be64_to_cpup(out_p + i);
	h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx);
	return le64_to_cpu(p + h);
}

static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx)
{
	u64 in[2] = {0}, out[2];
	unsigned i;
	int err = 0;

	err = crypto_cipher_setkey(ctx->child, user_key, VMAC_KEY_LEN);
	if (err)
		return err;

	/* Fill nh key */
	((unsigned char *)in)[0] = 0x80;
	for (i = 0; i < sizeof(ctx->__vmac_ctx.nhkey)/8; i += 2) {
		crypto_cipher_encrypt_one(ctx->child,
			(unsigned char *)out, (unsigned char *)in);
		ctx->__vmac_ctx.nhkey[i] = be64_to_cpup(out);
		ctx->__vmac_ctx.nhkey[i+1] = be64_to_cpup(out+1);
		((unsigned char *)in)[15] += 1;
	}

	/* Fill poly key */
	((unsigned char *)in)[0] = 0xC0;
	in[1] = 0;
	for (i = 0; i < sizeof(ctx->__vmac_ctx.polykey)/8; i += 2) {
		crypto_cipher_encrypt_one(ctx->child,
			(unsigned char *)out, (unsigned char *)in);
		ctx->__vmac_ctx.polytmp[i] =
			ctx->__vmac_ctx.polykey[i] =
				be64_to_cpup(out) & mpoly;
		ctx->__vmac_ctx.polytmp[i+1] =
			ctx->__vmac_ctx.polykey[i+1] =
				be64_to_cpup(out+1) & mpoly;
		((unsigned char *)in)[15] += 1;
	}

	/* Fill ip key */
	((unsigned char *)in)[0] = 0xE0;
	in[1] = 0;
	for (i = 0; i < sizeof(ctx->__vmac_ctx.l3key)/8; i += 2) {
		do {
			crypto_cipher_encrypt_one(ctx->child,
				(unsigned char *)out, (unsigned char *)in);
			ctx->__vmac_ctx.l3key[i] = be64_to_cpup(out);
			ctx->__vmac_ctx.l3key[i+1] = be64_to_cpup(out+1);
			((unsigned char *)in)[15] += 1;
		} while (ctx->__vmac_ctx.l3key[i] >= p64
			|| ctx->__vmac_ctx.l3key[i+1] >= p64);
	}

	/* Invalidate nonce/aes cache and reset other elements */
	ctx->__vmac_ctx.cached_nonce[0] = (u64)-1; /* Ensure illegal nonce */
	ctx->__vmac_ctx.cached_nonce[1] = (u64)0;  /* Ensure illegal nonce */
	ctx->__vmac_ctx.first_block_processed = 0;

	return err;
}

static int vmac_setkey(struct crypto_shash *parent,
		const u8 *key, unsigned int keylen)
{
	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);

	if (keylen != VMAC_KEY_LEN) {
		crypto_shash_set_flags(parent, CRYPTO_TFM_RES_BAD_KEY_LEN);
		return -EINVAL;
	}

	return vmac_set_key((u8 *)key, ctx);
}

static int vmac_init(struct shash_desc *pdesc)
{
	return 0;
}

static int vmac_update(struct shash_desc *pdesc, const u8 *p,
		unsigned int len)
{
	struct crypto_shash *parent = pdesc->tfm;
	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);

	vhash_update(p, len, &ctx->__vmac_ctx);

	return 0;
}

static int vmac_final(struct shash_desc *pdesc, u8 *out)
{
	struct crypto_shash *parent = pdesc->tfm;
	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
	vmac_t mac;
	u8 nonce[16] = {};

	mac = vmac(NULL, 0, nonce, NULL, ctx);
	memcpy(out, &mac, sizeof(vmac_t));
	memset(&mac, 0, sizeof(vmac_t));
	memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx));
	return 0;
}

static int vmac_init_tfm(struct crypto_tfm *tfm)
{
	struct crypto_cipher *cipher;
	struct crypto_instance *inst = (void *)tfm->__crt_alg;
	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
	struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);

	cipher = crypto_spawn_cipher(spawn);
	if (IS_ERR(cipher))
		return PTR_ERR(cipher);

	ctx->child = cipher;
	return 0;
}

static void vmac_exit_tfm(struct crypto_tfm *tfm)
{
	struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);
	crypto_free_cipher(ctx->child);
}

static int vmac_create(struct crypto_template *tmpl, struct rtattr **tb)
{
	struct shash_instance *inst;
	struct crypto_alg *alg;
	int err;

	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
	if (err)
		return err;

	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
			CRYPTO_ALG_TYPE_MASK);
	if (IS_ERR(alg))
		return PTR_ERR(alg);

	inst = shash_alloc_instance("vmac", alg);
	err = PTR_ERR(inst);
	if (IS_ERR(inst))
		goto out_put_alg;

	err = crypto_init_spawn(shash_instance_ctx(inst), alg,
			shash_crypto_instance(inst),
			CRYPTO_ALG_TYPE_MASK);
	if (err)
		goto out_free_inst;

	inst->alg.base.cra_priority = alg->cra_priority;
	inst->alg.base.cra_blocksize = alg->cra_blocksize;
	inst->alg.base.cra_alignmask = alg->cra_alignmask;

	inst->alg.digestsize = sizeof(vmac_t);
	inst->alg.base.cra_ctxsize = sizeof(struct vmac_ctx_t);
	inst->alg.base.cra_init = vmac_init_tfm;
	inst->alg.base.cra_exit = vmac_exit_tfm;

	inst->alg.init = vmac_init;
	inst->alg.update = vmac_update;
	inst->alg.final = vmac_final;
	inst->alg.setkey = vmac_setkey;

	err = shash_register_instance(tmpl, inst);
	if (err) {
out_free_inst:
		shash_free_instance(shash_crypto_instance(inst));
	}

out_put_alg:
	crypto_mod_put(alg);
	return err;
}

static struct crypto_template vmac_tmpl = {
	.name = "vmac",
	.create = vmac_create,
	.free = shash_free_instance,
	.module = THIS_MODULE,
};

static int __init vmac_module_init(void)
{
	return crypto_register_template(&vmac_tmpl);
}

static void __exit vmac_module_exit(void)
{
	crypto_unregister_template(&vmac_tmpl);
}

module_init(vmac_module_init);
module_exit(vmac_module_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("VMAC hash algorithm");
Commit	Line	Data
f1939f7c SW	1	/*
	2	* Modified to interface to the Linux kernel
	3	* Copyright (c) 2009, Intel Corporation.
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms and conditions of the GNU General Public License,
	7	* version 2, as published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope it will be useful, but WITHOUT
	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	12	* more details.
	13	*
	14	* You should have received a copy of the GNU General Public License along with
	15	* this program; if not, write to the Free Software Foundation, Inc., 59 Temple
	16	* Place - Suite 330, Boston, MA 02111-1307 USA.
	17	*/
	18
	19	/* --------------------------------------------------------------------------
	20	* VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai.
	21	* This implementation is herby placed in the public domain.
	22	* The authors offers no warranty. Use at your own risk.
	23	* Please send bug reports to the authors.
	24	* Last modified: 17 APR 08, 1700 PDT
	25	* ----------------------------------------------------------------------- */
	26
	27	#include <linux/init.h>
	28	#include <linux/types.h>
	29	#include <linux/crypto.h>
4bb33cc8	30	#include <linux/module.h>
f1939f7c SW	31	#include <linux/scatterlist.h>
	32	#include <asm/byteorder.h>
	33	#include <crypto/scatterwalk.h>
	34	#include <crypto/vmac.h>
	35	#include <crypto/internal/hash.h>
	36
	37	/*
	38	* Constants and masks
	39	*/
	40	#define UINT64_C(x) x##ULL
66ce0b0f JK	41	static const u64 p64 = UINT64_C(0xfffffffffffffeff); /* 2^64 - 257 prime */
	42	static const u64 m62 = UINT64_C(0x3fffffffffffffff); /* 62-bit mask */
	43	static const u64 m63 = UINT64_C(0x7fffffffffffffff); /* 63-bit mask */
	44	static const u64 m64 = UINT64_C(0xffffffffffffffff); /* 64-bit mask */
	45	static const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */
f1939f7c	46
304a204e SW	47	#define pe64_to_cpup le64_to_cpup /* Prefer little endian */
304a204e SW	48
f1939f7c SW	49	#ifdef __LITTLE_ENDIAN
	50	#define INDEX_HIGH 1
	51	#define INDEX_LOW 0
	52	#else
	53	#define INDEX_HIGH 0
	54	#define INDEX_LOW 1
	55	#endif
	56
	57	/*
	58	* The following routines are used in this implementation. They are
	59	* written via macros to simulate zero-overhead call-by-reference.
	60	*
	61	* MUL64: 64x64->128-bit multiplication
	62	* PMUL64: assumes top bits cleared on inputs
	63	* ADD128: 128x128->128-bit addition
	64	*/
	65
	66	#define ADD128(rh, rl, ih, il) \
	67	do { \
	68	u64 _il = (il); \
	69	(rl) += (_il); \
	70	if ((rl) < (_il)) \
	71	(rh)++; \
	72	(rh) += (ih); \
	73	} while (0)
	74
	75	#define MUL32(i1, i2) ((u64)(u32)(i1)*(u32)(i2))
	76
	77	#define PMUL64(rh, rl, i1, i2) /* Assumes m doesn't overflow */ \
	78	do { \
	79	u64 _i1 = (i1), _i2 = (i2); \
	80	u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2); \
	81	rh = MUL32(_i1>>32, _i2>>32); \
	82	rl = MUL32(_i1, _i2); \
	83	ADD128(rh, rl, (m >> 32), (m << 32)); \
	84	} while (0)
	85
	86	#define MUL64(rh, rl, i1, i2) \
	87	do { \
	88	u64 _i1 = (i1), _i2 = (i2); \
	89	u64 m1 = MUL32(_i1, _i2>>32); \
	90	u64 m2 = MUL32(_i1>>32, _i2); \
	91	rh = MUL32(_i1>>32, _i2>>32); \
	92	rl = MUL32(_i1, _i2); \
	93	ADD128(rh, rl, (m1 >> 32), (m1 << 32)); \
	94	ADD128(rh, rl, (m2 >> 32), (m2 << 32)); \
	95	} while (0)
	96
	97	/*
	98	* For highest performance the L1 NH and L2 polynomial hashes should be
25985edc	99	* carefully implemented to take advantage of one's target architecture.
f1939f7c SW	100	* Here these two hash functions are defined multiple time; once for
	101	* 64-bit architectures, once for 32-bit SSE2 architectures, and once
	102	* for the rest (32-bit) architectures.
	103	* For each, nh_16 must be defined (works on multiples of 16 bytes).
	104	* Optionally, nh_vmac_nhbytes can be defined (for multiples of
	105	* VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two
	106	* NH computations at once).
	107	*/
	108
	109	#ifdef CONFIG_64BIT
	110
	111	#define nh_16(mp, kp, nw, rh, rl) \
	112	do { \
	113	int i; u64 th, tl; \
	114	rh = rl = 0; \
	115	for (i = 0; i < nw; i += 2) { \
304a204e SW	116	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
304a204e SW	117	pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
f1939f7c SW	118	ADD128(rh, rl, th, tl); \
	119	} \
	120	} while (0)
	121
	122	#define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1) \
	123	do { \
	124	int i; u64 th, tl; \
	125	rh1 = rl1 = rh = rl = 0; \
	126	for (i = 0; i < nw; i += 2) { \
304a204e SW	127	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
304a204e SW	128	pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
f1939f7c	129	ADD128(rh, rl, th, tl); \
304a204e SW	130	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \
304a204e SW	131	pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \
f1939f7c SW	132	ADD128(rh1, rl1, th, tl); \
	133	} \
	134	} while (0)
	135
	136	#if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */
	137	#define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \
	138	do { \
	139	int i; u64 th, tl; \
	140	rh = rl = 0; \
	141	for (i = 0; i < nw; i += 8) { \
304a204e SW	142	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
304a204e SW	143	pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
f1939f7c	144	ADD128(rh, rl, th, tl); \
304a204e SW	145	MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
304a204e SW	146	pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \
f1939f7c	147	ADD128(rh, rl, th, tl); \
304a204e SW	148	MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
304a204e SW	149	pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \
f1939f7c	150	ADD128(rh, rl, th, tl); \
304a204e SW	151	MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
304a204e SW	152	pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \
f1939f7c SW	153	ADD128(rh, rl, th, tl); \
	154	} \
	155	} while (0)
	156
	157	#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1) \
	158	do { \
	159	int i; u64 th, tl; \
	160	rh1 = rl1 = rh = rl = 0; \
	161	for (i = 0; i < nw; i += 8) { \
304a204e SW	162	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
304a204e SW	163	pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
f1939f7c	164	ADD128(rh, rl, th, tl); \
304a204e SW	165	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \
304a204e SW	166	pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \
f1939f7c	167	ADD128(rh1, rl1, th, tl); \
304a204e SW	168	MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
304a204e SW	169	pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \
f1939f7c	170	ADD128(rh, rl, th, tl); \
304a204e SW	171	MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4], \
304a204e SW	172	pe64_to_cpup((mp)+i+3)+(kp)[i+5]); \
f1939f7c	173	ADD128(rh1, rl1, th, tl); \
304a204e SW	174	MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
304a204e SW	175	pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \
f1939f7c	176	ADD128(rh, rl, th, tl); \
304a204e SW	177	MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6], \
304a204e SW	178	pe64_to_cpup((mp)+i+5)+(kp)[i+7]); \
f1939f7c	179	ADD128(rh1, rl1, th, tl); \
304a204e SW	180	MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
304a204e SW	181	pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \
f1939f7c	182	ADD128(rh, rl, th, tl); \
304a204e SW	183	MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8], \
304a204e SW	184	pe64_to_cpup((mp)+i+7)+(kp)[i+9]); \
f1939f7c SW	185	ADD128(rh1, rl1, th, tl); \
	186	} \
	187	} while (0)
	188	#endif
	189
	190	#define poly_step(ah, al, kh, kl, mh, ml) \
	191	do { \
	192	u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0; \
	193	/* compute abcd, put bd into result registers / \
	194	PMUL64(t3h, t3l, al, kh); \
	195	PMUL64(t2h, t2l, ah, kl); \
	196	PMUL64(t1h, t1l, ah, 2*kh); \
	197	PMUL64(ah, al, al, kl); \
	198	/* add 2 * ac to result */ \
	199	ADD128(ah, al, t1h, t1l); \
	200	/* add together ad + bc */ \
	201	ADD128(t2h, t2l, t3h, t3l); \
	202	/* now (ah,al), (t2l,2t2h) need summing / \
	203	/* first add the high registers, carrying into t2h */ \
	204	ADD128(t2h, ah, z, t2l); \
	205	/* double t2h and add top bit of ah */ \
	206	t2h = 2 * t2h + (ah >> 63); \
	207	ah &= m63; \
	208	/* now add the low registers */ \
	209	ADD128(ah, al, mh, ml); \
	210	ADD128(ah, al, z, t2h); \
	211	} while (0)
	212
	213	#else /* ! CONFIG_64BIT */
	214
	215	#ifndef nh_16
	216	#define nh_16(mp, kp, nw, rh, rl) \
	217	do { \
	218	u64 t1, t2, m1, m2, t; \
	219	int i; \
	220	rh = rl = t = 0; \
	221	for (i = 0; i < nw; i += 2) { \
304a204e SW	222	t1 = pe64_to_cpup(mp+i) + kp[i]; \
304a204e SW	223	t2 = pe64_to_cpup(mp+i+1) + kp[i+1]; \
f1939f7c SW	224	m2 = MUL32(t1 >> 32, t2); \
	225	m1 = MUL32(t1, t2 >> 32); \
	226	ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32), \
	227	MUL32(t1, t2)); \
	228	rh += (u64)(u32)(m1 >> 32) \
	229	+ (u32)(m2 >> 32); \
	230	t += (u64)(u32)m1 + (u32)m2; \
	231	} \
	232	ADD128(rh, rl, (t >> 32), (t << 32)); \
	233	} while (0)
	234	#endif
	235
	236	static void poly_step_func(u64 ahi, u64 alo,
	237	const u64 kh, const u64 kl,
	238	const u64 mh, const u64 ml)
	239	{
	240	#define a0 ((((u32 )alo)+INDEX_LOW))
	241	#define a1 ((((u32 )alo)+INDEX_HIGH))
	242	#define a2 ((((u32 )ahi)+INDEX_LOW))
	243	#define a3 ((((u32 )ahi)+INDEX_HIGH))
	244	#define k0 ((((u32 )kl)+INDEX_LOW))
	245	#define k1 ((((u32 )kl)+INDEX_HIGH))
	246	#define k2 ((((u32 )kh)+INDEX_LOW))
	247	#define k3 ((((u32 )kh)+INDEX_HIGH))
	248
	249	u64 p, q, t;
	250	u32 t2;
	251
	252	p = MUL32(a3, k3);
	253	p += p;
	254	p += (u64 )mh;
	255	p += MUL32(a0, k2);
	256	p += MUL32(a1, k1);
	257	p += MUL32(a2, k0);
	258	t = (u32)(p);
	259	p >>= 32;
	260	p += MUL32(a0, k3);
	261	p += MUL32(a1, k2);
	262	p += MUL32(a2, k1);
	263	p += MUL32(a3, k0);
	264	t \|= ((u64)((u32)p & 0x7fffffff)) << 32;
	265	p >>= 31;
	266	p += (u64)(((u32 *)ml)[INDEX_LOW]);
	267	p += MUL32(a0, k0);
	268	q = MUL32(a1, k3);
	269	q += MUL32(a2, k2);
	270	q += MUL32(a3, k1);
	271	q += q;
	272	p += q;
	273	t2 = (u32)(p);
	274	p >>= 32;
	275	p += (u64)(((u32 *)ml)[INDEX_HIGH]);
	276	p += MUL32(a0, k1);
	277	p += MUL32(a1, k0);
	278	q = MUL32(a2, k3);
	279	q += MUL32(a3, k2);
	280	q += q;
	281	p += q;
	282	(u64 )(alo) = (p << 32) \| t2;
	283	p >>= 32;
	284	(u64 )(ahi) = p + t;
	285
	286	#undef a0
	287	#undef a1
288	#undef a2
289	#undef a3
290	#undef k0
291	#undef k1
292	#undef k2
293	#undef k3
294	}
295
296	#define poly_step(ah, al, kh, kl, mh, ml) \
297	poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml))
298
299	#endif /* end of specialized NH and poly definitions */
300
301	/* At least nh_16 is defined. Defined others as needed here */
302	#ifndef nh_16_2
303	#define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2) \
304	do { \
305	nh_16(mp, kp, nw, rh, rl); \
306	nh_16(mp, ((kp)+2), nw, rh2, rl2); \
307	} while (0)
308	#endif
309	#ifndef nh_vmac_nhbytes
310	#define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \
311	nh_16(mp, kp, nw, rh, rl)
312	#endif
313	#ifndef nh_vmac_nhbytes_2
314	#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2) \
315	do { \
316	nh_vmac_nhbytes(mp, kp, nw, rh, rl); \
317	nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2); \
318	} while (0)
319	#endif
320
321	static void vhash_abort(struct vmac_ctx *ctx)
322	{
323	ctx->polytmp[0] = ctx->polykey[0] ;
324	ctx->polytmp[1] = ctx->polykey[1] ;
325	ctx->first_block_processed = 0;
326	}
327
304a204e	328	static u64 l3hash(u64 p1, u64 p2, u64 k1, u64 k2, u64 len)
f1939f7c SW	329	{
	330	u64 rh, rl, t, z = 0;
	331
	332	/* fully reduce (p1,p2)+(len,0) mod p127 */
	333	t = p1 >> 63;
	334	p1 &= m63;
	335	ADD128(p1, p2, len, t);
	336	/* At this point, (p1,p2) is at most 2^127+(len<<64) */
	337	t = (p1 > m63) + ((p1 == m63) && (p2 == m64));
	338	ADD128(p1, p2, z, t);
	339	p1 &= m63;
	340
	341	/* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */
	342	t = p1 + (p2 >> 32);
	343	t += (t >> 32);
	344	t += (u32)t > 0xfffffffeu;
	345	p1 += (t >> 32);
	346	p2 += (p1 << 32);
	347
	348	/* compute (p1+k1)%p64 and (p2+k2)%p64 */
	349	p1 += k1;
	350	p1 += (0 - (p1 < k1)) & 257;
	351	p2 += k2;
	352	p2 += (0 - (p2 < k2)) & 257;
	353
	354	/* compute (p1+k1)(p2+k2)%p64 /
	355	MUL64(rh, rl, p1, p2);
	356	t = rh >> 56;
	357	ADD128(t, rl, z, rh);
	358	rh <<= 8;
	359	ADD128(t, rl, z, rh);
	360	t += t << 8;
	361	rl += t;
	362	rl += (0 - (rl < t)) & 257;
	363	rl += (0 - (rl > p64-1)) & 257;
	364	return rl;
	365	}
	366
	367	static void vhash_update(const unsigned char *m,
	368	unsigned int mbytes, /* Pos multiple of VMAC_NHBYTES */
	369	struct vmac_ctx *ctx)
	370	{
	371	u64 rh, rl, *mptr;
	372	const u64 kptr = (u64 )ctx->nhkey;
	373	int i;
	374	u64 ch, cl;
	375	u64 pkh = ctx->polykey[0];
	376	u64 pkl = ctx->polykey[1];
	377
	378	mptr = (u64 *)m;
	379	i = mbytes / VMAC_NHBYTES; /* Must be non-zero */
	380
	381	ch = ctx->polytmp[0];
	382	cl = ctx->polytmp[1];
	383
	384	if (!ctx->first_block_processed) {
	385	ctx->first_block_processed = 1;
	386	nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
	387	rh &= m62;
	388	ADD128(ch, cl, rh, rl);
	389	mptr += (VMAC_NHBYTES/sizeof(u64));
	390	i--;
	391	}
	392
393	while (i--) {
394	nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
395	rh &= m62;
396	poly_step(ch, cl, pkh, pkl, rh, rl);
397	mptr += (VMAC_NHBYTES/sizeof(u64));
398	}
399
400	ctx->polytmp[0] = ch;
401	ctx->polytmp[1] = cl;
402	}
403
404	static u64 vhash(unsigned char m[], unsigned int mbytes,
405	u64 tagl, struct vmac_ctx ctx)
406	{
407	u64 rh, rl, *mptr;
408	const u64 kptr = (u64 )ctx->nhkey;
409	int i, remaining;
410	u64 ch, cl;
411	u64 pkh = ctx->polykey[0];
412	u64 pkl = ctx->polykey[1];
413
414	mptr = (u64 *)m;
415	i = mbytes / VMAC_NHBYTES;
416	remaining = mbytes % VMAC_NHBYTES;
417
418	if (ctx->first_block_processed) {
419	ch = ctx->polytmp[0];
420	cl = ctx->polytmp[1];
421	} else if (i) {
422	nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, ch, cl);
423	ch &= m62;
424	ADD128(ch, cl, pkh, pkl);
425	mptr += (VMAC_NHBYTES/sizeof(u64));
426	i--;
427	} else if (remaining) {
428	nh_16(mptr, kptr, 2*((remaining+15)/16), ch, cl);
429	ch &= m62;
430	ADD128(ch, cl, pkh, pkl);
431	mptr += (VMAC_NHBYTES/sizeof(u64));
432	goto do_l3;
433	} else {/* Empty String */
434	ch = pkh; cl = pkl;
435	goto do_l3;
436	}
437
438	while (i--) {
439	nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
440	rh &= m62;
441	poly_step(ch, cl, pkh, pkl, rh, rl);
442	mptr += (VMAC_NHBYTES/sizeof(u64));
443	}
444	if (remaining) {
445	nh_16(mptr, kptr, 2*((remaining+15)/16), rh, rl);
446	rh &= m62;
447	poly_step(ch, cl, pkh, pkl, rh, rl);
448	}
449
450	do_l3:
451	vhash_abort(ctx);
452	remaining *= 8;
453	return l3hash(ch, cl, ctx->l3key[0], ctx->l3key[1], remaining);
454	}
455
456	static u64 vmac(unsigned char m[], unsigned int mbytes,
457	unsigned char n[16], u64 *tagl,
458	struct vmac_ctx_t *ctx)
459	{
460	u64 in_n, out_p;
461	u64 p, h;
462	int i;
463
464	in_n = ctx->__vmac_ctx.cached_nonce;
465	out_p = ctx->__vmac_ctx.cached_aes;
466
467	i = n[15] & 1;
468	if (((u64 )(n+8) != in_n[1]) \|\| ((u64 )(n) != in_n[0])) {
469	in_n[0] = (u64 )(n);
470	in_n[1] = (u64 )(n+8);
471	((unsigned char *)in_n)[15] &= 0xFE;
472	crypto_cipher_encrypt_one(ctx->child,
473	(unsigned char )out_p, (unsigned char )in_n);
474
475	((unsigned char *)in_n)[15] \|= (unsigned char)(1-i);
476	}
477	p = be64_to_cpup(out_p + i);
478	h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx);
304a204e	479	return le64_to_cpu(p + h);
f1939f7c SW	480	}
	481
	482	static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx)
	483	{
	484	u64 in[2] = {0}, out[2];
	485	unsigned i;
	486	int err = 0;
	487
	488	err = crypto_cipher_setkey(ctx->child, user_key, VMAC_KEY_LEN);
	489	if (err)
	490	return err;
	491
	492	/* Fill nh key */
	493	((unsigned char *)in)[0] = 0x80;
	494	for (i = 0; i < sizeof(ctx->__vmac_ctx.nhkey)/8; i += 2) {
	495	crypto_cipher_encrypt_one(ctx->child,
	496	(unsigned char )out, (unsigned char )in);
	497	ctx->__vmac_ctx.nhkey[i] = be64_to_cpup(out);
	498	ctx->__vmac_ctx.nhkey[i+1] = be64_to_cpup(out+1);
	499	((unsigned char *)in)[15] += 1;
	500	}
	501
	502	/* Fill poly key */
	503	((unsigned char *)in)[0] = 0xC0;
	504	in[1] = 0;
	505	for (i = 0; i < sizeof(ctx->__vmac_ctx.polykey)/8; i += 2) {
	506	crypto_cipher_encrypt_one(ctx->child,
	507	(unsigned char )out, (unsigned char )in);
	508	ctx->__vmac_ctx.polytmp[i] =
	509	ctx->__vmac_ctx.polykey[i] =
	510	be64_to_cpup(out) & mpoly;
	511	ctx->__vmac_ctx.polytmp[i+1] =
	512	ctx->__vmac_ctx.polykey[i+1] =
	513	be64_to_cpup(out+1) & mpoly;
	514	((unsigned char *)in)[15] += 1;
	515	}
	516
	517	/* Fill ip key */
	518	((unsigned char *)in)[0] = 0xE0;
	519	in[1] = 0;
	520	for (i = 0; i < sizeof(ctx->__vmac_ctx.l3key)/8; i += 2) {
	521	do {
	522	crypto_cipher_encrypt_one(ctx->child,
	523	(unsigned char )out, (unsigned char )in);
	524	ctx->__vmac_ctx.l3key[i] = be64_to_cpup(out);
	525	ctx->__vmac_ctx.l3key[i+1] = be64_to_cpup(out+1);
	526	((unsigned char *)in)[15] += 1;
	527	} while (ctx->__vmac_ctx.l3key[i] >= p64
	528	\|\| ctx->__vmac_ctx.l3key[i+1] >= p64);
	529	}
	530
	531	/* Invalidate nonce/aes cache and reset other elements */
	532	ctx->__vmac_ctx.cached_nonce[0] = (u64)-1; /* Ensure illegal nonce */
	533	ctx->__vmac_ctx.cached_nonce[1] = (u64)0; /* Ensure illegal nonce */
	534	ctx->__vmac_ctx.first_block_processed = 0;
	535
	536	return err;
	537	}
	538
	539	static int vmac_setkey(struct crypto_shash *parent,
	540	const u8 *key, unsigned int keylen)
	541	{
	542	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
	543
544	if (keylen != VMAC_KEY_LEN) {
545	crypto_shash_set_flags(parent, CRYPTO_TFM_RES_BAD_KEY_LEN);
546	return -EINVAL;
547	}
548
549	return vmac_set_key((u8 *)key, ctx);
550	}
551
552	static int vmac_init(struct shash_desc *pdesc)
553	{
f1939f7c SW	554	return 0;
	555	}
	556
	557	static int vmac_update(struct shash_desc pdesc, const u8 p,
	558	unsigned int len)
	559	{
	560	struct crypto_shash *parent = pdesc->tfm;
	561	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
	562
	563	vhash_update(p, len, &ctx->__vmac_ctx);
	564
	565	return 0;
	566	}
	567
	568	static int vmac_final(struct shash_desc pdesc, u8 out)
	569	{
	570	struct crypto_shash *parent = pdesc->tfm;
	571	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
	572	vmac_t mac;
	573	u8 nonce[16] = {};
	574
	575	mac = vmac(NULL, 0, nonce, NULL, ctx);
	576	memcpy(out, &mac, sizeof(vmac_t));
	577	memset(&mac, 0, sizeof(vmac_t));
	578	memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx));
	579	return 0;
	580	}
	581
	582	static int vmac_init_tfm(struct crypto_tfm *tfm)
	583	{
	584	struct crypto_cipher *cipher;
	585	struct crypto_instance inst = (void )tfm->__crt_alg;
	586	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
	587	struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);
	588
	589	cipher = crypto_spawn_cipher(spawn);
	590	if (IS_ERR(cipher))
	591	return PTR_ERR(cipher);
	592
	593	ctx->child = cipher;
	594	return 0;
	595	}
	596
	597	static void vmac_exit_tfm(struct crypto_tfm *tfm)
	598	{
	599	struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);
	600	crypto_free_cipher(ctx->child);
	601	}
	602
	603	static int vmac_create(struct crypto_template tmpl, struct rtattr *tb)
	604	{
	605	struct shash_instance *inst;
	606	struct crypto_alg *alg;
	607	int err;
	608
	609	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
	610	if (err)
	611	return err;
	612
	613	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
	614	CRYPTO_ALG_TYPE_MASK);
	615	if (IS_ERR(alg))
	616	return PTR_ERR(alg);
	617
618	inst = shash_alloc_instance("vmac", alg);
619	err = PTR_ERR(inst);
620	if (IS_ERR(inst))
621	goto out_put_alg;
622
623	err = crypto_init_spawn(shash_instance_ctx(inst), alg,
624	shash_crypto_instance(inst),
625	CRYPTO_ALG_TYPE_MASK);
626	if (err)
627	goto out_free_inst;
628
629	inst->alg.base.cra_priority = alg->cra_priority;
630	inst->alg.base.cra_blocksize = alg->cra_blocksize;
631	inst->alg.base.cra_alignmask = alg->cra_alignmask;
632
633	inst->alg.digestsize = sizeof(vmac_t);
634	inst->alg.base.cra_ctxsize = sizeof(struct vmac_ctx_t);
635	inst->alg.base.cra_init = vmac_init_tfm;
636	inst->alg.base.cra_exit = vmac_exit_tfm;
637
638	inst->alg.init = vmac_init;
639	inst->alg.update = vmac_update;
640	inst->alg.final = vmac_final;
641	inst->alg.setkey = vmac_setkey;
642
643	err = shash_register_instance(tmpl, inst);
644	if (err) {
645	out_free_inst:
646	shash_free_instance(shash_crypto_instance(inst));
647	}
648
649	out_put_alg:
650	crypto_mod_put(alg);
651	return err;
652	}
653
654	static struct crypto_template vmac_tmpl = {
655	.name = "vmac",
656	.create = vmac_create,
657	.free = shash_free_instance,
658	.module = THIS_MODULE,
659	};
660
661	static int __init vmac_module_init(void)
662	{
663	return crypto_register_template(&vmac_tmpl);
664	}
665
666	static void __exit vmac_module_exit(void)
667	{
668	crypto_unregister_template(&vmac_tmpl);
669	}
670
671	module_init(vmac_module_init);
672	module_exit(vmac_module_exit);
673
674	MODULE_LICENSE("GPL");
675	MODULE_DESCRIPTION("VMAC hash algorithm");
676