[GitHub/mt8127/android_kernel_alcatel_ttab.git] / arch / x86 / lib / copy_user_64.S

/*
 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
 * Copyright 2002 Andi Kleen, SuSE Labs.
 * Subject to the GNU Public License v2.
 *
 * Functions to copy from and to user space.
 */

#include <linux/linkage.h>
#include <asm/dwarf2.h>

#define FIX_ALIGNMENT 1

#include <asm/current.h>
#include <asm/asm-offsets.h>
#include <asm/thread_info.h>
#include <asm/cpufeature.h>
#include <asm/alternative-asm.h>
#include <asm/asm.h>
#include <asm/smap.h>

/*
 * By placing feature2 after feature1 in altinstructions section, we logically
 * implement:
 * If CPU has feature2, jmp to alt2 is used
 * else if CPU has feature1, jmp to alt1 is used
 * else jmp to orig is used.
 */
	.macro ALTERNATIVE_JUMP feature1,feature2,orig,alt1,alt2
0:
	.byte 0xe9	/* 32bit jump */
	.long \orig-1f	/* by default jump to orig */
1:
	.section .altinstr_replacement,"ax"
2:	.byte 0xe9			/* near jump with 32bit immediate */
	.long \alt1-1b /* offset */   /* or alternatively to alt1 */
3:	.byte 0xe9			/* near jump with 32bit immediate */
	.long \alt2-1b /* offset */   /* or alternatively to alt2 */
	.previous

	.section .altinstructions,"a"
	altinstruction_entry 0b,2b,\feature1,5,5
	altinstruction_entry 0b,3b,\feature2,5,5
	.previous
	.endm

	.macro ALIGN_DESTINATION
#ifdef FIX_ALIGNMENT
	/* check for bad alignment of destination */
	movl %edi,%ecx
	andl $7,%ecx
	jz 102f				/* already aligned */
	subl $8,%ecx
	negl %ecx
	subl %ecx,%edx
100:	movb (%rsi),%al
101:	movb %al,(%rdi)
	incq %rsi
	incq %rdi
	decl %ecx
	jnz 100b
102:
	.section .fixup,"ax"
103:	addl %ecx,%edx			/* ecx is zerorest also */
	jmp copy_user_handle_tail
	.previous

	_ASM_EXTABLE(100b,103b)
	_ASM_EXTABLE(101b,103b)
#endif
	.endm

/* Standard copy_to_user with segment limit checking */
ENTRY(_copy_to_user)
	CFI_STARTPROC
	GET_THREAD_INFO(%rax)
	movq %rdi,%rcx
	addq %rdx,%rcx
	jc bad_to_user
	cmpq TI_addr_limit(%rax),%rcx
	ja bad_to_user
	ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS,	\
		copy_user_generic_unrolled,copy_user_generic_string,	\
		copy_user_enhanced_fast_string
	CFI_ENDPROC
ENDPROC(_copy_to_user)

/* Standard copy_from_user with segment limit checking */
ENTRY(_copy_from_user)
	CFI_STARTPROC
	GET_THREAD_INFO(%rax)
	movq %rsi,%rcx
	addq %rdx,%rcx
	jc bad_from_user
	cmpq TI_addr_limit(%rax),%rcx
	ja bad_from_user
	ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS,	\
		copy_user_generic_unrolled,copy_user_generic_string,	\
		copy_user_enhanced_fast_string
	CFI_ENDPROC
ENDPROC(_copy_from_user)

	.section .fixup,"ax"
	/* must zero dest */
ENTRY(bad_from_user)
bad_from_user:
	CFI_STARTPROC
	movl %edx,%ecx
	xorl %eax,%eax
	rep
	stosb
bad_to_user:
	movl %edx,%eax
	ret
	CFI_ENDPROC
ENDPROC(bad_from_user)
	.previous

/*
 * copy_user_generic_unrolled - memory copy with exception handling.
 * This version is for CPUs like P4 that don't have efficient micro
 * code for rep movsq
 *
 * Input:
 * rdi destination
 * rsi source
 * rdx count
 *
 * Output:
 * eax uncopied bytes or 0 if successful.
 */
ENTRY(copy_user_generic_unrolled)
	CFI_STARTPROC
	ASM_STAC
	cmpl $8,%edx
	jb 20f		/* less then 8 bytes, go to byte copy loop */
	ALIGN_DESTINATION
	movl %edx,%ecx
	andl $63,%edx
	shrl $6,%ecx
	jz 17f
1:	movq (%rsi),%r8
2:	movq 1*8(%rsi),%r9
3:	movq 2*8(%rsi),%r10
4:	movq 3*8(%rsi),%r11
5:	movq %r8,(%rdi)
6:	movq %r9,1*8(%rdi)
7:	movq %r10,2*8(%rdi)
8:	movq %r11,3*8(%rdi)
9:	movq 4*8(%rsi),%r8
10:	movq 5*8(%rsi),%r9
11:	movq 6*8(%rsi),%r10
12:	movq 7*8(%rsi),%r11
13:	movq %r8,4*8(%rdi)
14:	movq %r9,5*8(%rdi)
15:	movq %r10,6*8(%rdi)
16:	movq %r11,7*8(%rdi)
	leaq 64(%rsi),%rsi
	leaq 64(%rdi),%rdi
	decl %ecx
	jnz 1b
17:	movl %edx,%ecx
	andl $7,%edx
	shrl $3,%ecx
	jz 20f
18:	movq (%rsi),%r8
19:	movq %r8,(%rdi)
	leaq 8(%rsi),%rsi
	leaq 8(%rdi),%rdi
	decl %ecx
	jnz 18b
20:	andl %edx,%edx
	jz 23f
	movl %edx,%ecx
21:	movb (%rsi),%al
22:	movb %al,(%rdi)
	incq %rsi
	incq %rdi
	decl %ecx
	jnz 21b
23:	xor %eax,%eax
	ASM_CLAC
	ret

	.section .fixup,"ax"
30:	shll $6,%ecx
	addl %ecx,%edx
	jmp 60f
40:	lea (%rdx,%rcx,8),%rdx
	jmp 60f
50:	movl %ecx,%edx
60:	jmp copy_user_handle_tail /* ecx is zerorest also */
	.previous

	_ASM_EXTABLE(1b,30b)
	_ASM_EXTABLE(2b,30b)
	_ASM_EXTABLE(3b,30b)
	_ASM_EXTABLE(4b,30b)
	_ASM_EXTABLE(5b,30b)
	_ASM_EXTABLE(6b,30b)
	_ASM_EXTABLE(7b,30b)
	_ASM_EXTABLE(8b,30b)
	_ASM_EXTABLE(9b,30b)
	_ASM_EXTABLE(10b,30b)
	_ASM_EXTABLE(11b,30b)
	_ASM_EXTABLE(12b,30b)
	_ASM_EXTABLE(13b,30b)
	_ASM_EXTABLE(14b,30b)
	_ASM_EXTABLE(15b,30b)
	_ASM_EXTABLE(16b,30b)
	_ASM_EXTABLE(18b,40b)
	_ASM_EXTABLE(19b,40b)
	_ASM_EXTABLE(21b,50b)
	_ASM_EXTABLE(22b,50b)
	CFI_ENDPROC
ENDPROC(copy_user_generic_unrolled)

/* Some CPUs run faster using the string copy instructions.
 * This is also a lot simpler. Use them when possible.
 *
 * Only 4GB of copy is supported. This shouldn't be a problem
 * because the kernel normally only writes from/to page sized chunks
 * even if user space passed a longer buffer.
 * And more would be dangerous because both Intel and AMD have
 * errata with rep movsq > 4GB. If someone feels the need to fix
 * this please consider this.
 *
 * Input:
 * rdi destination
 * rsi source
 * rdx count
 *
 * Output:
 * eax uncopied bytes or 0 if successful.
 */
ENTRY(copy_user_generic_string)
	CFI_STARTPROC
	ASM_STAC
	andl %edx,%edx
	jz 4f
	cmpl $8,%edx
	jb 2f		/* less than 8 bytes, go to byte copy loop */
	ALIGN_DESTINATION
	movl %edx,%ecx
	shrl $3,%ecx
	andl $7,%edx
1:	rep
	movsq
2:	movl %edx,%ecx
3:	rep
	movsb
4:	xorl %eax,%eax
	ASM_CLAC
	ret

	.section .fixup,"ax"
11:	lea (%rdx,%rcx,8),%rcx
12:	movl %ecx,%edx		/* ecx is zerorest also */
	jmp copy_user_handle_tail
	.previous

	_ASM_EXTABLE(1b,11b)
	_ASM_EXTABLE(3b,12b)
	CFI_ENDPROC
ENDPROC(copy_user_generic_string)

/*
 * Some CPUs are adding enhanced REP MOVSB/STOSB instructions.
 * It's recommended to use enhanced REP MOVSB/STOSB if it's enabled.
 *
 * Input:
 * rdi destination
 * rsi source
 * rdx count
 *
 * Output:
 * eax uncopied bytes or 0 if successful.
 */
ENTRY(copy_user_enhanced_fast_string)
	CFI_STARTPROC
	ASM_STAC
	andl %edx,%edx
	jz 2f
	movl %edx,%ecx
1:	rep
	movsb
2:	xorl %eax,%eax
	ASM_CLAC
	ret

	.section .fixup,"ax"
12:	movl %ecx,%edx		/* ecx is zerorest also */
	jmp copy_user_handle_tail
	.previous

	_ASM_EXTABLE(1b,12b)
	CFI_ENDPROC
ENDPROC(copy_user_enhanced_fast_string)
Commit	Line	Data
ad2fc2cd VM	1	/*
	2	* Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
	3	* Copyright 2002 Andi Kleen, SuSE Labs.
1da177e4	4	* Subject to the GNU Public License v2.
ad2fc2cd VM	5	*
	6	* Functions to copy from and to user space.
	7	*/
1da177e4	8
8d379dad JB	9	#include <linux/linkage.h>
	10	#include <asm/dwarf2.h>
	11
7bcd3f34 AK	12	#define FIX_ALIGNMENT 1
7bcd3f34 AK	13
3022d734 AK	14	#include <asm/current.h>
	15	#include <asm/asm-offsets.h>
	16	#include <asm/thread_info.h>
	17	#include <asm/cpufeature.h>
4307bec9	18	#include <asm/alternative-asm.h>
9732da8c	19	#include <asm/asm.h>
63bcff2a	20	#include <asm/smap.h>
3022d734	21
4307bec9 FY	22	/*
	23	* By placing feature2 after feature1 in altinstructions section, we logically
	24	* implement:
	25	* If CPU has feature2, jmp to alt2 is used
	26	* else if CPU has feature1, jmp to alt1 is used
	27	* else jmp to orig is used.
	28	*/
	29	.macro ALTERNATIVE_JUMP feature1,feature2,orig,alt1,alt2
3022d734 AK	30	0:
	31	.byte 0xe9 /* 32bit jump */
	32	.long \orig-1f /* by default jump to orig */
	33	1:
	34	.section .altinstr_replacement,"ax"
ad2fc2cd	35	2: .byte 0xe9 /* near jump with 32bit immediate */
4307bec9 FY	36	.long \alt1-1b /* offset / / or alternatively to alt1 */
	37	3: .byte 0xe9 /* near jump with 32bit immediate */
	38	.long \alt2-1b /* offset / / or alternatively to alt2 */
3022d734	39	.previous
4307bec9	40
3022d734	41	.section .altinstructions,"a"
4307bec9 FY	42	altinstruction_entry 0b,2b,\feature1,5,5
4307bec9 FY	43	altinstruction_entry 0b,3b,\feature2,5,5
3022d734 AK	44	.previous
3022d734 AK	45	.endm
1da177e4	46
ad2fc2cd VM	47	.macro ALIGN_DESTINATION
	48	#ifdef FIX_ALIGNMENT
	49	/* check for bad alignment of destination */
	50	movl %edi,%ecx
	51	andl $7,%ecx
	52	jz 102f /* already aligned */
	53	subl $8,%ecx
	54	negl %ecx
	55	subl %ecx,%edx
	56	100: movb (%rsi),%al
	57	101: movb %al,(%rdi)
	58	incq %rsi
	59	incq %rdi
	60	decl %ecx
	61	jnz 100b
	62	102:
	63	.section .fixup,"ax"
afd962a9	64	103: addl %ecx,%edx /* ecx is zerorest also */
ad2fc2cd VM	65	jmp copy_user_handle_tail
	66	.previous
	67
9732da8c PA	68	_ASM_EXTABLE(100b,103b)
9732da8c PA	69	_ASM_EXTABLE(101b,103b)
ad2fc2cd VM	70	#endif
	71	.endm
	72
	73	/* Standard copy_to_user with segment limit checking */
3c93ca00	74	ENTRY(_copy_to_user)
8d379dad	75	CFI_STARTPROC
1da177e4 LT	76	GET_THREAD_INFO(%rax)
	77	movq %rdi,%rcx
	78	addq %rdx,%rcx
ad2fc2cd	79	jc bad_to_user
26ccb8a7	80	cmpq TI_addr_limit(%rax),%rcx
26afb7c6	81	ja bad_to_user
4307bec9 FY	82	ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
	83	copy_user_generic_unrolled,copy_user_generic_string, \
	84	copy_user_enhanced_fast_string
8d379dad	85	CFI_ENDPROC
3c93ca00	86	ENDPROC(_copy_to_user)
7bcd3f34	87
ad2fc2cd	88	/* Standard copy_from_user with segment limit checking */
9f0cf4ad	89	ENTRY(_copy_from_user)
3022d734	90	CFI_STARTPROC
ad2fc2cd VM	91	GET_THREAD_INFO(%rax)
	92	movq %rsi,%rcx
	93	addq %rdx,%rcx
	94	jc bad_from_user
	95	cmpq TI_addr_limit(%rax),%rcx
26afb7c6	96	ja bad_from_user
4307bec9 FY	97	ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
	98	copy_user_generic_unrolled,copy_user_generic_string, \
	99	copy_user_enhanced_fast_string
3022d734	100	CFI_ENDPROC
9f0cf4ad	101	ENDPROC(_copy_from_user)
3022d734	102
1da177e4 LT	103	.section .fixup,"ax"
1da177e4 LT	104	/* must zero dest */
ad2fc2cd	105	ENTRY(bad_from_user)
1da177e4	106	bad_from_user:
8d379dad	107	CFI_STARTPROC
1da177e4 LT	108	movl %edx,%ecx
	109	xorl %eax,%eax
	110	rep
	111	stosb
	112	bad_to_user:
ad2fc2cd	113	movl %edx,%eax
1da177e4	114	ret
8d379dad	115	CFI_ENDPROC
ad2fc2cd	116	ENDPROC(bad_from_user)
1da177e4	117	.previous
ad2fc2cd	118
1da177e4	119	/*
3022d734	120	* copy_user_generic_unrolled - memory copy with exception handling.
ad2fc2cd VM	121	* This version is for CPUs like P4 that don't have efficient micro
	122	* code for rep movsq
	123	*
	124	* Input:
1da177e4 LT	125	* rdi destination
	126	* rsi source
	127	* rdx count
	128	*
ad2fc2cd	129	* Output:
0d2eb44f	130	* eax uncopied bytes or 0 if successful.
1da177e4	131	*/
3022d734	132	ENTRY(copy_user_generic_unrolled)
8d379dad	133	CFI_STARTPROC
63bcff2a	134	ASM_STAC
ad2fc2cd VM	135	cmpl $8,%edx
	136	jb 20f /* less then 8 bytes, go to byte copy loop */
	137	ALIGN_DESTINATION
	138	movl %edx,%ecx
	139	andl $63,%edx
	140	shrl $6,%ecx
	141	jz 17f
	142	1: movq (%rsi),%r8
	143	2: movq 1*8(%rsi),%r9
	144	3: movq 2*8(%rsi),%r10
	145	4: movq 3*8(%rsi),%r11
	146	5: movq %r8,(%rdi)
	147	6: movq %r9,1*8(%rdi)
	148	7: movq %r10,2*8(%rdi)
	149	8: movq %r11,3*8(%rdi)
	150	9: movq 4*8(%rsi),%r8
	151	10: movq 5*8(%rsi),%r9
	152	11: movq 6*8(%rsi),%r10
	153	12: movq 7*8(%rsi),%r11
	154	13: movq %r8,4*8(%rdi)
	155	14: movq %r9,5*8(%rdi)
	156	15: movq %r10,6*8(%rdi)
	157	16: movq %r11,7*8(%rdi)
7bcd3f34 AK	158	leaq 64(%rsi),%rsi
7bcd3f34 AK	159	leaq 64(%rdi),%rdi
7bcd3f34	160	decl %ecx
ad2fc2cd VM	161	jnz 1b
	162	17: movl %edx,%ecx
	163	andl $7,%edx
	164	shrl $3,%ecx
	165	jz 20f
	166	18: movq (%rsi),%r8
	167	19: movq %r8,(%rdi)
7bcd3f34	168	leaq 8(%rsi),%rsi
ad2fc2cd VM	169	leaq 8(%rdi),%rdi
	170	decl %ecx
	171	jnz 18b
	172	20: andl %edx,%edx
	173	jz 23f
7bcd3f34	174	movl %edx,%ecx
ad2fc2cd VM	175	21: movb (%rsi),%al
ad2fc2cd VM	176	22: movb %al,(%rdi)
7bcd3f34	177	incq %rsi
ad2fc2cd	178	incq %rdi
7bcd3f34	179	decl %ecx
ad2fc2cd VM	180	jnz 21b
ad2fc2cd VM	181	23: xor %eax,%eax
63bcff2a	182	ASM_CLAC
7bcd3f34 AK	183	ret
7bcd3f34 AK	184
ad2fc2cd VM	185	.section .fixup,"ax"
	186	30: shll $6,%ecx
	187	addl %ecx,%edx
	188	jmp 60f
27cb0a75	189	40: lea (%rdx,%rcx,8),%rdx
ad2fc2cd VM	190	jmp 60f
	191	50: movl %ecx,%edx
	192	60: jmp copy_user_handle_tail /* ecx is zerorest also */
	193	.previous
7bcd3f34	194
9732da8c PA	195	_ASM_EXTABLE(1b,30b)
	196	_ASM_EXTABLE(2b,30b)
	197	_ASM_EXTABLE(3b,30b)
	198	_ASM_EXTABLE(4b,30b)
	199	_ASM_EXTABLE(5b,30b)
	200	_ASM_EXTABLE(6b,30b)
	201	_ASM_EXTABLE(7b,30b)
	202	_ASM_EXTABLE(8b,30b)
	203	_ASM_EXTABLE(9b,30b)
	204	_ASM_EXTABLE(10b,30b)
	205	_ASM_EXTABLE(11b,30b)
	206	_ASM_EXTABLE(12b,30b)
	207	_ASM_EXTABLE(13b,30b)
	208	_ASM_EXTABLE(14b,30b)
	209	_ASM_EXTABLE(15b,30b)
	210	_ASM_EXTABLE(16b,30b)
	211	_ASM_EXTABLE(18b,40b)
	212	_ASM_EXTABLE(19b,40b)
	213	_ASM_EXTABLE(21b,50b)
	214	_ASM_EXTABLE(22b,50b)
8d379dad	215	CFI_ENDPROC
ad2fc2cd	216	ENDPROC(copy_user_generic_unrolled)
8d379dad	217
ad2fc2cd VM	218	/* Some CPUs run faster using the string copy instructions.
	219	* This is also a lot simpler. Use them when possible.
	220	*
	221	* Only 4GB of copy is supported. This shouldn't be a problem
	222	* because the kernel normally only writes from/to page sized chunks
	223	* even if user space passed a longer buffer.
	224	* And more would be dangerous because both Intel and AMD have
	225	* errata with rep movsq > 4GB. If someone feels the need to fix
	226	* this please consider this.
	227	*
	228	* Input:
	229	* rdi destination
	230	* rsi source
	231	* rdx count
	232	*
	233	* Output:
	234	* eax uncopied bytes or 0 if successful.
	235	*/
3022d734	236	ENTRY(copy_user_generic_string)
8d379dad	237	CFI_STARTPROC
63bcff2a	238	ASM_STAC
ad2fc2cd VM	239	andl %edx,%edx
	240	jz 4f
	241	cmpl $8,%edx
	242	jb 2f /* less than 8 bytes, go to byte copy loop */
	243	ALIGN_DESTINATION
1da177e4 LT	244	movl %edx,%ecx
1da177e4 LT	245	shrl $3,%ecx
ad2fc2cd VM	246	andl $7,%edx
ad2fc2cd VM	247	1: rep
3022d734	248	movsq
ad2fc2cd VM	249	2: movl %edx,%ecx
	250	3: rep
	251	movsb
	252	4: xorl %eax,%eax
63bcff2a	253	ASM_CLAC
1da177e4	254	ret
3022d734	255
ad2fc2cd	256	.section .fixup,"ax"
27cb0a75	257	11: lea (%rdx,%rcx,8),%rcx
ad2fc2cd VM	258	12: movl %ecx,%edx /* ecx is zerorest also */
	259	jmp copy_user_handle_tail
	260	.previous
2cbc9ee3	261
9732da8c PA	262	_ASM_EXTABLE(1b,11b)
9732da8c PA	263	_ASM_EXTABLE(3b,12b)
ad2fc2cd VM	264	CFI_ENDPROC
ad2fc2cd VM	265	ENDPROC(copy_user_generic_string)
4307bec9 FY	266
	267	/*
	268	* Some CPUs are adding enhanced REP MOVSB/STOSB instructions.
	269	* It's recommended to use enhanced REP MOVSB/STOSB if it's enabled.
	270	*
	271	* Input:
	272	* rdi destination
	273	* rsi source
	274	* rdx count
	275	*
	276	* Output:
	277	* eax uncopied bytes or 0 if successful.
	278	*/
	279	ENTRY(copy_user_enhanced_fast_string)
	280	CFI_STARTPROC
63bcff2a	281	ASM_STAC
4307bec9 FY	282	andl %edx,%edx
	283	jz 2f
	284	movl %edx,%ecx
	285	1: rep
	286	movsb
	287	2: xorl %eax,%eax
63bcff2a	288	ASM_CLAC
4307bec9 FY	289	ret
	290
	291	.section .fixup,"ax"
	292	12: movl %ecx,%edx /* ecx is zerorest also */
	293	jmp copy_user_handle_tail
	294	.previous
	295
9732da8c	296	_ASM_EXTABLE(1b,12b)
4307bec9 FY	297	CFI_ENDPROC
4307bec9 FY	298	ENDPROC(copy_user_enhanced_fast_string)