[GitHub/mt8127/android_kernel_alcatel_ttab.git] / arch / x86 / crypto / crc32-pclmul_asm.S

/* GPL HEADER START
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 only,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License version 2 for more details (a copy is included
 * in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU General Public License
 * version 2 along with this program; If not, see http://www.gnu.org/licenses
 *
 * Please  visit http://www.xyratex.com/contact if you need additional
 * information or have any questions.
 *
 * GPL HEADER END
 */

/*
 * Copyright 2012 Xyratex Technology Limited
 *
 * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
 * calculation.
 * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
 * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
 * at:
 * http://www.intel.com/products/processor/manuals/
 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
 * Volume 2B: Instruction Set Reference, N-Z
 *
 * Authors:   Gregory Prestas <Gregory_Prestas@us.xyratex.com>
 *	      Alexander Boyko <Alexander_Boyko@xyratex.com>
 */

#include <linux/linkage.h>
#include <asm/inst.h>


.align 16
/*
 * [x4*128+32 mod P(x) << 32)]'  << 1   = 0x154442bd4
 * #define CONSTANT_R1  0x154442bd4LL
 *
 * [(x4*128-32 mod P(x) << 32)]' << 1   = 0x1c6e41596
 * #define CONSTANT_R2  0x1c6e41596LL
 */
.Lconstant_R2R1:
	.octa 0x00000001c6e415960000000154442bd4
/*
 * [(x128+32 mod P(x) << 32)]'   << 1   = 0x1751997d0
 * #define CONSTANT_R3  0x1751997d0LL
 *
 * [(x128-32 mod P(x) << 32)]'   << 1   = 0x0ccaa009e
 * #define CONSTANT_R4  0x0ccaa009eLL
 */
.Lconstant_R4R3:
	.octa 0x00000000ccaa009e00000001751997d0
/*
 * [(x64 mod P(x) << 32)]'       << 1   = 0x163cd6124
 * #define CONSTANT_R5  0x163cd6124LL
 */
.Lconstant_R5:
	.octa 0x00000000000000000000000163cd6124
.Lconstant_mask32:
	.octa 0x000000000000000000000000FFFFFFFF
/*
 * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
 *
 * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))` = 0x1F7011641LL
 * #define CONSTANT_RU  0x1F7011641LL
 */
.Lconstant_RUpoly:
	.octa 0x00000001F701164100000001DB710641

#define CONSTANT %xmm0

#ifdef __x86_64__
#define BUF     %rdi
#define LEN     %rsi
#define CRC     %edx
#else
#define BUF     %eax
#define LEN     %edx
#define CRC     %ecx
#endif


.text
/**
 *      Calculate crc32
 *      BUF - buffer (16 bytes aligned)
 *      LEN - sizeof buffer (16 bytes aligned), LEN should be grater than 63
 *      CRC - initial crc32
 *      return %eax crc32
 *      uint crc32_pclmul_le_16(unsigned char const *buffer,
 *	                     size_t len, uint crc32)
 */

ENTRY(crc32_pclmul_le_16) /* buffer and buffer size are 16 bytes aligned */
	movdqa  (BUF), %xmm1
	movdqa  0x10(BUF), %xmm2
	movdqa  0x20(BUF), %xmm3
	movdqa  0x30(BUF), %xmm4
	movd    CRC, CONSTANT
	pxor    CONSTANT, %xmm1
	sub     $0x40, LEN
	add     $0x40, BUF
#ifndef __x86_64__
	/* This is for position independent code(-fPIC) support for 32bit */
	call    delta
delta:
	pop     %ecx
#endif
	cmp     $0x40, LEN
	jb      less_64

#ifdef __x86_64__
	movdqa .Lconstant_R2R1(%rip), CONSTANT
#else
	movdqa .Lconstant_R2R1 - delta(%ecx), CONSTANT
#endif

loop_64:/*  64 bytes Full cache line folding */
	prefetchnta    0x40(BUF)
	movdqa  %xmm1, %xmm5
	movdqa  %xmm2, %xmm6
	movdqa  %xmm3, %xmm7
#ifdef __x86_64__
	movdqa  %xmm4, %xmm8
#endif
	PCLMULQDQ 00, CONSTANT, %xmm1
	PCLMULQDQ 00, CONSTANT, %xmm2
	PCLMULQDQ 00, CONSTANT, %xmm3
#ifdef __x86_64__
	PCLMULQDQ 00, CONSTANT, %xmm4
#endif
	PCLMULQDQ 0x11, CONSTANT, %xmm5
	PCLMULQDQ 0x11, CONSTANT, %xmm6
	PCLMULQDQ 0x11, CONSTANT, %xmm7
#ifdef __x86_64__
	PCLMULQDQ 0x11, CONSTANT, %xmm8
#endif
	pxor    %xmm5, %xmm1
	pxor    %xmm6, %xmm2
	pxor    %xmm7, %xmm3
#ifdef __x86_64__
	pxor    %xmm8, %xmm4
#else
	/* xmm8 unsupported for x32 */
	movdqa  %xmm4, %xmm5
	PCLMULQDQ 00, CONSTANT, %xmm4
	PCLMULQDQ 0x11, CONSTANT, %xmm5
	pxor    %xmm5, %xmm4
#endif

	pxor    (BUF), %xmm1
	pxor    0x10(BUF), %xmm2
	pxor    0x20(BUF), %xmm3
	pxor    0x30(BUF), %xmm4

	sub     $0x40, LEN
	add     $0x40, BUF
	cmp     $0x40, LEN
	jge     loop_64
less_64:/*  Folding cache line into 128bit */
#ifdef __x86_64__
	movdqa  .Lconstant_R4R3(%rip), CONSTANT
#else
	movdqa  .Lconstant_R4R3 - delta(%ecx), CONSTANT
#endif
	prefetchnta     (BUF)

	movdqa  %xmm1, %xmm5
	PCLMULQDQ 0x00, CONSTANT, %xmm1
	PCLMULQDQ 0x11, CONSTANT, %xmm5
	pxor    %xmm5, %xmm1
	pxor    %xmm2, %xmm1

	movdqa  %xmm1, %xmm5
	PCLMULQDQ 0x00, CONSTANT, %xmm1
	PCLMULQDQ 0x11, CONSTANT, %xmm5
	pxor    %xmm5, %xmm1
	pxor    %xmm3, %xmm1

	movdqa  %xmm1, %xmm5
	PCLMULQDQ 0x00, CONSTANT, %xmm1
	PCLMULQDQ 0x11, CONSTANT, %xmm5
	pxor    %xmm5, %xmm1
	pxor    %xmm4, %xmm1

	cmp     $0x10, LEN
	jb      fold_64
loop_16:/* Folding rest buffer into 128bit */
	movdqa  %xmm1, %xmm5
	PCLMULQDQ 0x00, CONSTANT, %xmm1
	PCLMULQDQ 0x11, CONSTANT, %xmm5
	pxor    %xmm5, %xmm1
	pxor    (BUF), %xmm1
	sub     $0x10, LEN
	add     $0x10, BUF
	cmp     $0x10, LEN
	jge     loop_16

fold_64:
	/* perform the last 64 bit fold, also adds 32 zeroes
	 * to the input stream */
	PCLMULQDQ 0x01, %xmm1, CONSTANT /* R4 * xmm1.low */
	psrldq  $0x08, %xmm1
	pxor    CONSTANT, %xmm1

	/* final 32-bit fold */
	movdqa  %xmm1, %xmm2
#ifdef __x86_64__
	movdqa  .Lconstant_R5(%rip), CONSTANT
	movdqa  .Lconstant_mask32(%rip), %xmm3
#else
	movdqa  .Lconstant_R5 - delta(%ecx), CONSTANT
	movdqa  .Lconstant_mask32 - delta(%ecx), %xmm3
#endif
	psrldq  $0x04, %xmm2
	pand    %xmm3, %xmm1
	PCLMULQDQ 0x00, CONSTANT, %xmm1
	pxor    %xmm2, %xmm1

	/* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
#ifdef __x86_64__
	movdqa  .Lconstant_RUpoly(%rip), CONSTANT
#else
	movdqa  .Lconstant_RUpoly - delta(%ecx), CONSTANT
#endif
	movdqa  %xmm1, %xmm2
	pand    %xmm3, %xmm1
	PCLMULQDQ 0x10, CONSTANT, %xmm1
	pand    %xmm3, %xmm1
	PCLMULQDQ 0x00, CONSTANT, %xmm1
	pxor    %xmm2, %xmm1
	PEXTRD  0x01, %xmm1, %eax

	ret
ENDPROC(crc32_pclmul_le_16)
Commit	Line	Data
78c37d19 AB	1	/* GPL HEADER START
	2	*
	3	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 only,
	7	* as published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful, but
	10	* WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	12	* General Public License version 2 for more details (a copy is included
	13	* in the LICENSE file that accompanied this code).
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* version 2 along with this program; If not, see http://www.gnu.org/licenses
	17	*
	18	* Please visit http://www.xyratex.com/contact if you need additional
	19	* information or have any questions.
	20	*
	21	* GPL HEADER END
	22	*/
	23
	24	/*
	25	* Copyright 2012 Xyratex Technology Limited
	26	*
	27	* Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
	28	* calculation.
	29	* CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
	30	* PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
	31	* at:
	32	* http://www.intel.com/products/processor/manuals/
	33	* Intel(R) 64 and IA-32 Architectures Software Developer's Manual
	34	* Volume 2B: Instruction Set Reference, N-Z
	35	*
	36	* Authors: Gregory Prestas <Gregory_Prestas@us.xyratex.com>
	37	* Alexander Boyko <Alexander_Boyko@xyratex.com>
	38	*/
	39
	40	#include <linux/linkage.h>
	41	#include <asm/inst.h>
	42
	43
	44	.align 16
	45	/*
	46	* [x4*128+32 mod P(x) << 32)]' << 1 = 0x154442bd4
	47	* #define CONSTANT_R1 0x154442bd4LL
	48	*
	49	* [(x4*128-32 mod P(x) << 32)]' << 1 = 0x1c6e41596
	50	* #define CONSTANT_R2 0x1c6e41596LL
	51	*/
	52	.Lconstant_R2R1:
	53	.octa 0x00000001c6e415960000000154442bd4
	54	/*
	55	* [(x128+32 mod P(x) << 32)]' << 1 = 0x1751997d0
	56	* #define CONSTANT_R3 0x1751997d0LL
	57	*
	58	* [(x128-32 mod P(x) << 32)]' << 1 = 0x0ccaa009e
	59	* #define CONSTANT_R4 0x0ccaa009eLL
	60	*/
	61	.Lconstant_R4R3:
	62	.octa 0x00000000ccaa009e00000001751997d0
	63	/*
	64	* [(x64 mod P(x) << 32)]' << 1 = 0x163cd6124
65	* #define CONSTANT_R5 0x163cd6124LL
66	*/
67	.Lconstant_R5:
68	.octa 0x00000000000000000000000163cd6124
69	.Lconstant_mask32:
70	.octa 0x000000000000000000000000FFFFFFFF
71	/*
72	* #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
73	*
74	* Barrett Reduction constant (u64`) = u` = (x**64 / P(x))` = 0x1F7011641LL
75	* #define CONSTANT_RU 0x1F7011641LL
76	*/
77	.Lconstant_RUpoly:
78	.octa 0x00000001F701164100000001DB710641
79
80	#define CONSTANT %xmm0
81
82	#ifdef __x86_64__
83	#define BUF %rdi
84	#define LEN %rsi
85	#define CRC %edx
86	#else
78c37d19 AB	87	#define BUF %eax
	88	#define LEN %edx
	89	#define CRC %ecx
	90	#endif
	91
	92
	93
	94	.text
	95	/**
	96	* Calculate crc32
	97	* BUF - buffer (16 bytes aligned)
	98	* LEN - sizeof buffer (16 bytes aligned), LEN should be grater than 63
	99	* CRC - initial crc32
	100	* return %eax crc32
	101	* uint crc32_pclmul_le_16(unsigned char const *buffer,
	102	* size_t len, uint crc32)
	103	*/
eca17269 JK	104
eca17269 JK	105	ENTRY(crc32_pclmul_le_16) /* buffer and buffer size are 16 bytes aligned */
78c37d19 AB	106	movdqa (BUF), %xmm1
	107	movdqa 0x10(BUF), %xmm2
	108	movdqa 0x20(BUF), %xmm3
	109	movdqa 0x30(BUF), %xmm4
	110	movd CRC, CONSTANT
	111	pxor CONSTANT, %xmm1
	112	sub $0x40, LEN
	113	add $0x40, BUF
	114	#ifndef __x86_64__
	115	/* This is for position independent code(-fPIC) support for 32bit */
	116	call delta
	117	delta:
	118	pop %ecx
	119	#endif
	120	cmp $0x40, LEN
	121	jb less_64
	122
	123	#ifdef __x86_64__
	124	movdqa .Lconstant_R2R1(%rip), CONSTANT
	125	#else
	126	movdqa .Lconstant_R2R1 - delta(%ecx), CONSTANT
	127	#endif
	128
	129	loop_64:/* 64 bytes Full cache line folding */
	130	prefetchnta 0x40(BUF)
	131	movdqa %xmm1, %xmm5
	132	movdqa %xmm2, %xmm6
	133	movdqa %xmm3, %xmm7
	134	#ifdef __x86_64__
	135	movdqa %xmm4, %xmm8
	136	#endif
	137	PCLMULQDQ 00, CONSTANT, %xmm1
	138	PCLMULQDQ 00, CONSTANT, %xmm2
	139	PCLMULQDQ 00, CONSTANT, %xmm3
	140	#ifdef __x86_64__
	141	PCLMULQDQ 00, CONSTANT, %xmm4
	142	#endif
	143	PCLMULQDQ 0x11, CONSTANT, %xmm5
	144	PCLMULQDQ 0x11, CONSTANT, %xmm6
	145	PCLMULQDQ 0x11, CONSTANT, %xmm7
	146	#ifdef __x86_64__
	147	PCLMULQDQ 0x11, CONSTANT, %xmm8
	148	#endif
	149	pxor %xmm5, %xmm1
	150	pxor %xmm6, %xmm2
	151	pxor %xmm7, %xmm3
	152	#ifdef __x86_64__
	153	pxor %xmm8, %xmm4
	154	#else
	155	/* xmm8 unsupported for x32 */
	156	movdqa %xmm4, %xmm5
	157	PCLMULQDQ 00, CONSTANT, %xmm4
	158	PCLMULQDQ 0x11, CONSTANT, %xmm5
	159	pxor %xmm5, %xmm4
	160	#endif
	161
	162	pxor (BUF), %xmm1
	163	pxor 0x10(BUF), %xmm2
	164	pxor 0x20(BUF), %xmm3
	165	pxor 0x30(BUF), %xmm4
	166
	167	sub $0x40, LEN
	168	add $0x40, BUF
	169	cmp $0x40, LEN
170	jge loop_64
171	less_64:/* Folding cache line into 128bit */
172	#ifdef __x86_64__
173	movdqa .Lconstant_R4R3(%rip), CONSTANT
174	#else
175	movdqa .Lconstant_R4R3 - delta(%ecx), CONSTANT
176	#endif
177	prefetchnta (BUF)
178
179	movdqa %xmm1, %xmm5
180	PCLMULQDQ 0x00, CONSTANT, %xmm1
181	PCLMULQDQ 0x11, CONSTANT, %xmm5
182	pxor %xmm5, %xmm1
183	pxor %xmm2, %xmm1
184
185	movdqa %xmm1, %xmm5
186	PCLMULQDQ 0x00, CONSTANT, %xmm1
187	PCLMULQDQ 0x11, CONSTANT, %xmm5
188	pxor %xmm5, %xmm1
189	pxor %xmm3, %xmm1
190
191	movdqa %xmm1, %xmm5
192	PCLMULQDQ 0x00, CONSTANT, %xmm1
193	PCLMULQDQ 0x11, CONSTANT, %xmm5
194	pxor %xmm5, %xmm1
195	pxor %xmm4, %xmm1
196
197	cmp $0x10, LEN
198	jb fold_64
199	loop_16:/* Folding rest buffer into 128bit */
200	movdqa %xmm1, %xmm5
201	PCLMULQDQ 0x00, CONSTANT, %xmm1
202	PCLMULQDQ 0x11, CONSTANT, %xmm5
203	pxor %xmm5, %xmm1
204	pxor (BUF), %xmm1
205	sub $0x10, LEN
206	add $0x10, BUF
207	cmp $0x10, LEN
208	jge loop_16
209
210	fold_64:
211	/* perform the last 64 bit fold, also adds 32 zeroes
212	* to the input stream */
213	PCLMULQDQ 0x01, %xmm1, CONSTANT /* R4 * xmm1.low */
214	psrldq $0x08, %xmm1
215	pxor CONSTANT, %xmm1
216
217	/* final 32-bit fold */
218	movdqa %xmm1, %xmm2
219	#ifdef __x86_64__
220	movdqa .Lconstant_R5(%rip), CONSTANT
221	movdqa .Lconstant_mask32(%rip), %xmm3
222	#else
223	movdqa .Lconstant_R5 - delta(%ecx), CONSTANT
224	movdqa .Lconstant_mask32 - delta(%ecx), %xmm3
225	#endif
226	psrldq $0x04, %xmm2
227	pand %xmm3, %xmm1
228	PCLMULQDQ 0x00, CONSTANT, %xmm1
229	pxor %xmm2, %xmm1
230
231	/* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
232	#ifdef __x86_64__
233	movdqa .Lconstant_RUpoly(%rip), CONSTANT
234	#else
235	movdqa .Lconstant_RUpoly - delta(%ecx), CONSTANT
236	#endif
237	movdqa %xmm1, %xmm2
238	pand %xmm3, %xmm1
239	PCLMULQDQ 0x10, CONSTANT, %xmm1
240	pand %xmm3, %xmm1
241	PCLMULQDQ 0x00, CONSTANT, %xmm1
242	pxor %xmm2, %xmm1
2baad612	243	PEXTRD 0x01, %xmm1, %eax
78c37d19 AB	244
78c37d19 AB	245	ret
eca17269	246	ENDPROC(crc32_pclmul_le_16)