[GitHub/mt8127/android_kernel_alcatel_ttab.git] / arch / sparc / lib / memcpy.S

/* memcpy.S: Sparc optimized memcpy and memmove code
 * Hand optimized from GNU libc's memcpy and memmove
 * Copyright (C) 1991,1996 Free Software Foundation
 * Copyright (C) 1995 Linus Torvalds (Linus.Torvalds@helsinki.fi)
 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
 * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
 */

#define FUNC(x) 		\
	.globl	x;		\
	.type	x,@function;	\
	.align	4;		\
x:

/* Both these macros have to start with exactly the same insn */
#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
	ldd	[%src + (offset) + 0x00], %t0; \
	ldd	[%src + (offset) + 0x08], %t2; \
	ldd	[%src + (offset) + 0x10], %t4; \
	ldd	[%src + (offset) + 0x18], %t6; \
	st	%t0, [%dst + (offset) + 0x00]; \
	st	%t1, [%dst + (offset) + 0x04]; \
	st	%t2, [%dst + (offset) + 0x08]; \
	st	%t3, [%dst + (offset) + 0x0c]; \
	st	%t4, [%dst + (offset) + 0x10]; \
	st	%t5, [%dst + (offset) + 0x14]; \
	st	%t6, [%dst + (offset) + 0x18]; \
	st	%t7, [%dst + (offset) + 0x1c];

#define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
	ldd	[%src + (offset) + 0x00], %t0; \
	ldd	[%src + (offset) + 0x08], %t2; \
	ldd	[%src + (offset) + 0x10], %t4; \
	ldd	[%src + (offset) + 0x18], %t6; \
	std	%t0, [%dst + (offset) + 0x00]; \
	std	%t2, [%dst + (offset) + 0x08]; \
	std	%t4, [%dst + (offset) + 0x10]; \
	std	%t6, [%dst + (offset) + 0x18];

#define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
	ldd	[%src - (offset) - 0x10], %t0; \
	ldd	[%src - (offset) - 0x08], %t2; \
	st	%t0, [%dst - (offset) - 0x10]; \
	st	%t1, [%dst - (offset) - 0x0c]; \
	st	%t2, [%dst - (offset) - 0x08]; \
	st	%t3, [%dst - (offset) - 0x04];

#define MOVE_LASTALIGNCHUNK(src, dst, offset, t0, t1, t2, t3) \
	ldd	[%src - (offset) - 0x10], %t0; \
	ldd	[%src - (offset) - 0x08], %t2; \
	std	%t0, [%dst - (offset) - 0x10]; \
	std	%t2, [%dst - (offset) - 0x08];

#define MOVE_SHORTCHUNK(src, dst, offset, t0, t1) \
	ldub	[%src - (offset) - 0x02], %t0; \
	ldub	[%src - (offset) - 0x01], %t1; \
	stb	%t0, [%dst - (offset) - 0x02]; \
	stb	%t1, [%dst - (offset) - 0x01];

/* Both these macros have to start with exactly the same insn */
#define RMOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
	ldd	[%src - (offset) - 0x20], %t0; \
	ldd	[%src - (offset) - 0x18], %t2; \
	ldd	[%src - (offset) - 0x10], %t4; \
	ldd	[%src - (offset) - 0x08], %t6; \
	st	%t0, [%dst - (offset) - 0x20]; \
	st	%t1, [%dst - (offset) - 0x1c]; \
	st	%t2, [%dst - (offset) - 0x18]; \
	st	%t3, [%dst - (offset) - 0x14]; \
	st	%t4, [%dst - (offset) - 0x10]; \
	st	%t5, [%dst - (offset) - 0x0c]; \
	st	%t6, [%dst - (offset) - 0x08]; \
	st	%t7, [%dst - (offset) - 0x04];

#define RMOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
	ldd	[%src - (offset) - 0x20], %t0; \
	ldd	[%src - (offset) - 0x18], %t2; \
	ldd	[%src - (offset) - 0x10], %t4; \
	ldd	[%src - (offset) - 0x08], %t6; \
	std	%t0, [%dst - (offset) - 0x20]; \
	std	%t2, [%dst - (offset) - 0x18]; \
	std	%t4, [%dst - (offset) - 0x10]; \
	std	%t6, [%dst - (offset) - 0x08];

#define RMOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
	ldd	[%src + (offset) + 0x00], %t0; \
	ldd	[%src + (offset) + 0x08], %t2; \
	st	%t0, [%dst + (offset) + 0x00]; \
	st	%t1, [%dst + (offset) + 0x04]; \
	st	%t2, [%dst + (offset) + 0x08]; \
	st	%t3, [%dst + (offset) + 0x0c];

#define RMOVE_SHORTCHUNK(src, dst, offset, t0, t1) \
	ldub	[%src + (offset) + 0x00], %t0; \
	ldub	[%src + (offset) + 0x01], %t1; \
	stb	%t0, [%dst + (offset) + 0x00]; \
	stb	%t1, [%dst + (offset) + 0x01];

#define SMOVE_CHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, prev, shil, shir, offset2) \
	ldd	[%src + (offset) + 0x00], %t0; \
	ldd	[%src + (offset) + 0x08], %t2; \
	srl	%t0, shir, %t5; \
	srl	%t1, shir, %t6; \
	sll	%t0, shil, %t0; \
	or	%t5, %prev, %t5; \
	sll	%t1, shil, %prev; \
	or	%t6, %t0, %t0; \
	srl	%t2, shir, %t1; \
	srl	%t3, shir, %t6; \
	sll	%t2, shil, %t2; \
	or	%t1, %prev, %t1; \
	std	%t4, [%dst + (offset) + (offset2) - 0x04]; \
	std	%t0, [%dst + (offset) + (offset2) + 0x04]; \
	sll	%t3, shil, %prev; \
	or	%t6, %t2, %t4;

#define SMOVE_ALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, prev, shil, shir, offset2) \
	ldd	[%src + (offset) + 0x00], %t0; \
	ldd	[%src + (offset) + 0x08], %t2; \
	srl	%t0, shir, %t4;	\
	srl	%t1, shir, %t5;	\
	sll	%t0, shil, %t6;	\
	or	%t4, %prev, %t0; \
	sll	%t1, shil, %prev; \
	or	%t5, %t6, %t1; \
	srl	%t2, shir, %t4;	\
	srl	%t3, shir, %t5;	\
	sll	%t2, shil, %t6; \
	or	%t4, %prev, %t2; \
	sll	%t3, shil, %prev; \
	or	%t5, %t6, %t3; \
	std	%t0, [%dst + (offset) + (offset2) + 0x00]; \
	std	%t2, [%dst + (offset) + (offset2) + 0x08];

	.text
	.align	4

0:
	retl
	 nop		! Only bcopy returns here and it retuns void...

#ifdef __KERNEL__
FUNC(amemmove)
FUNC(__memmove)
#endif
FUNC(memmove)
	cmp		%o0, %o1
	mov		%o0, %g7
	bleu		9f
	 sub		%o0, %o1, %o4

	add		%o1, %o2, %o3
	cmp		%o3, %o0
	bleu		0f
	 andcc		%o4, 3, %o5

	add		%o1, %o2, %o1
	add		%o0, %o2, %o0
	sub		%o1, 1, %o1
	sub		%o0, 1, %o0
	
1:	/* reverse_bytes */

	ldub		[%o1], %o4
	subcc		%o2, 1, %o2
	stb		%o4, [%o0]
	sub		%o1, 1, %o1
	bne		1b
	 sub		%o0, 1, %o0

	retl
	 mov		%g7, %o0

/* NOTE: This code is executed just for the cases,
         where %src (=%o1) & 3 is != 0.
	 We need to align it to 4. So, for (%src & 3)
	 1 we need to do ldub,lduh
	 2 lduh
	 3 just ldub
         so even if it looks weird, the branches
         are correct here. -jj
 */
78:	/* dword_align */

	andcc		%o1, 1, %g0
	be		4f
	 andcc		%o1, 2, %g0

	ldub		[%o1], %g2
	add		%o1, 1, %o1
	stb		%g2, [%o0]
	sub		%o2, 1, %o2
	bne		3f
	 add		%o0, 1, %o0
4:
	lduh		[%o1], %g2
	add		%o1, 2, %o1
	sth		%g2, [%o0]
	sub		%o2, 2, %o2
	b		3f
	 add		%o0, 2, %o0

FUNC(memcpy)	/* %o0=dst %o1=src %o2=len */

	sub		%o0, %o1, %o4
	mov		%o0, %g7
9:
	andcc		%o4, 3, %o5
0:
	bne		86f
	 cmp		%o2, 15

	bleu		90f
	 andcc		%o1, 3, %g0

	bne		78b
3:
	 andcc		%o1, 4, %g0

	be		2f
	 mov		%o2, %g1

	ld		[%o1], %o4
	sub		%g1, 4, %g1
	st		%o4, [%o0]
	add		%o1, 4, %o1
	add		%o0, 4, %o0
2:
	andcc		%g1, 0xffffff80, %g0
	be		3f
	 andcc		%o0, 4, %g0

	be		82f + 4
5:
	MOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
	MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
	MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
	MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
	sub		%g1, 128, %g1
	add		%o1, 128, %o1
	cmp		%g1, 128
	bge		5b
	 add		%o0, 128, %o0
3:
	andcc		%g1, 0x70, %g4
	be		80f
	 andcc		%g1, 8, %g0

	sethi		%hi(80f), %o5
	srl		%g4, 1, %o4
	add		%g4, %o4, %o4
	add		%o1, %g4, %o1
	sub		%o5, %o4, %o5
	jmpl		%o5 + %lo(80f), %g0
	 add		%o0, %g4, %o0

79:	/* memcpy_table */

	MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
	MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
	MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
	MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
	MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
	MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
	MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5)

80:	/* memcpy_table_end */
	be		81f
	 andcc		%g1, 4, %g0

	ldd		[%o1], %g2
	add		%o0, 8, %o0
	st		%g2, [%o0 - 0x08]
	add		%o1, 8, %o1
	st		%g3, [%o0 - 0x04]

81:	/* memcpy_last7 */

	be		1f
	 andcc		%g1, 2, %g0

	ld		[%o1], %g2
	add		%o1, 4, %o1
	st		%g2, [%o0]
	add		%o0, 4, %o0
1:
	be		1f
	 andcc		%g1, 1, %g0

	lduh		[%o1], %g2
	add		%o1, 2, %o1
	sth		%g2, [%o0]
	add		%o0, 2, %o0
1:
	be		1f
	 nop

	ldub		[%o1], %g2
	stb		%g2, [%o0]
1:
	retl
	 mov		%g7, %o0

82:	/* ldd_std */
	MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
	MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
	MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
	MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
	subcc		%g1, 128, %g1
	add		%o1, 128, %o1
	cmp		%g1, 128
	bge		82b
	 add		%o0, 128, %o0

	andcc		%g1, 0x70, %g4
	be		84f
	 andcc		%g1, 8, %g0

	sethi		%hi(84f), %o5
	add		%o1, %g4, %o1
	sub		%o5, %g4, %o5
	jmpl		%o5 + %lo(84f), %g0
	 add		%o0, %g4, %o0

83:	/* amemcpy_table */

	MOVE_LASTALIGNCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
	MOVE_LASTALIGNCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
	MOVE_LASTALIGNCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
	MOVE_LASTALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
	MOVE_LASTALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
	MOVE_LASTALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
	MOVE_LASTALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5)

84:	/* amemcpy_table_end */
	be		85f
	 andcc		%g1, 4, %g0

	ldd		[%o1], %g2
	add		%o0, 8, %o0
	std		%g2, [%o0 - 0x08]
	add		%o1, 8, %o1
85:	/* amemcpy_last7 */
	be		1f
	 andcc		%g1, 2, %g0

	ld		[%o1], %g2
	add		%o1, 4, %o1
	st		%g2, [%o0]
	add		%o0, 4, %o0
1:
	be		1f
	 andcc		%g1, 1, %g0

	lduh		[%o1], %g2
	add		%o1, 2, %o1
	sth		%g2, [%o0]
	add		%o0, 2, %o0
1:
	be		1f
	 nop

	ldub		[%o1], %g2
	stb		%g2, [%o0]
1:
	retl
	 mov		%g7, %o0

86:	/* non_aligned */
	cmp		%o2, 6
	bleu		88f
	 nop

	save		%sp, -96, %sp
	andcc		%i0, 3, %g0
	be		61f
	 andcc		%i0, 1, %g0
	be		60f
	 andcc		%i0, 2, %g0

	ldub		[%i1], %g5
	add		%i1, 1, %i1
	stb		%g5, [%i0]
	sub		%i2, 1, %i2
	bne		61f
	 add		%i0, 1, %i0
60:
	ldub		[%i1], %g3
	add		%i1, 2, %i1
	stb		%g3, [%i0]
	sub		%i2, 2, %i2
	ldub		[%i1 - 1], %g3
	add		%i0, 2, %i0
	stb		%g3, [%i0 - 1]
61:
	and		%i1, 3, %g2
	and		%i2, 0xc, %g3
	and		%i1, -4, %i1
	cmp		%g3, 4
	sll		%g2, 3, %g4
	mov		32, %g2
	be		4f
	 sub		%g2, %g4, %l0
	
	blu		3f
	 cmp		%g3, 0x8

	be		2f
	 srl		%i2, 2, %g3

	ld		[%i1], %i3
	add		%i0, -8, %i0
	ld		[%i1 + 4], %i4
	b		8f
	 add		%g3, 1, %g3
2:
	ld		[%i1], %i4
	add		%i0, -12, %i0
	ld		[%i1 + 4], %i5
	add		%g3, 2, %g3
	b		9f
	 add		%i1, -4, %i1
3:
	ld		[%i1], %g1
	add		%i0, -4, %i0
	ld		[%i1 + 4], %i3
	srl		%i2, 2, %g3
	b		7f
	 add		%i1, 4, %i1
4:
	ld		[%i1], %i5
	cmp		%i2, 7
	ld		[%i1 + 4], %g1
	srl		%i2, 2, %g3
	bleu		10f
	 add		%i1, 8, %i1

	ld		[%i1], %i3
	add		%g3, -1, %g3
5:
	sll		%i5, %g4, %g2
	srl		%g1, %l0, %g5
	or		%g2, %g5, %g2
	st		%g2, [%i0]
7:
	ld		[%i1 + 4], %i4
	sll		%g1, %g4, %g2
	srl		%i3, %l0, %g5
	or		%g2, %g5, %g2
	st		%g2, [%i0 + 4]
8:
	ld		[%i1 + 8], %i5
	sll		%i3, %g4, %g2
	srl		%i4, %l0, %g5
	or		%g2, %g5, %g2
	st		%g2, [%i0 + 8]
9:
	ld		[%i1 + 12], %g1
	sll		%i4, %g4, %g2
	srl		%i5, %l0, %g5
	addcc		%g3, -4, %g3
	or		%g2, %g5, %g2
	add		%i1, 16, %i1
	st		%g2, [%i0 + 12]
	add		%i0, 16, %i0
	bne,a		5b
	 ld		[%i1], %i3
10:
	sll		%i5, %g4, %g2
	srl		%g1, %l0, %g5
	srl		%l0, 3, %g3
	or		%g2, %g5, %g2
	sub		%i1, %g3, %i1
	andcc		%i2, 2, %g0
	st		%g2, [%i0]
	be		1f
	 andcc		%i2, 1, %g0

	ldub		[%i1], %g2
	add		%i1, 2, %i1
	stb		%g2, [%i0 + 4]
	add		%i0, 2, %i0
	ldub		[%i1 - 1], %g2
	stb		%g2, [%i0 + 3]
1:
	be		1f
	 nop
	ldub		[%i1], %g2
	stb		%g2, [%i0 + 4]
1:
	ret
	 restore	%g7, %g0, %o0

88:	/* short_end */

	and		%o2, 0xe, %o3
20:
	sethi		%hi(89f), %o5
	sll		%o3, 3, %o4
	add		%o0, %o3, %o0
	sub		%o5, %o4, %o5
	add		%o1, %o3, %o1
	jmpl		%o5 + %lo(89f), %g0
	 andcc		%o2, 1, %g0

	MOVE_SHORTCHUNK(o1, o0, 0x0c, g2, g3)
	MOVE_SHORTCHUNK(o1, o0, 0x0a, g2, g3)
	MOVE_SHORTCHUNK(o1, o0, 0x08, g2, g3)
	MOVE_SHORTCHUNK(o1, o0, 0x06, g2, g3)
	MOVE_SHORTCHUNK(o1, o0, 0x04, g2, g3)
	MOVE_SHORTCHUNK(o1, o0, 0x02, g2, g3)
	MOVE_SHORTCHUNK(o1, o0, 0x00, g2, g3)

89:	/* short_table_end */

	be		1f
	 nop

	ldub		[%o1], %g2
	stb		%g2, [%o0]
1:
	retl
	 mov		%g7, %o0

90:	/* short_aligned_end */
	bne		88b
	 andcc		%o2, 8, %g0

	be		1f
	 andcc		%o2, 4, %g0

	ld		[%o1 + 0x00], %g2
	ld		[%o1 + 0x04], %g3
	add		%o1, 8, %o1
	st		%g2, [%o0 + 0x00]
	st		%g3, [%o0 + 0x04]
	add		%o0, 8, %o0
1:
	b		81b
	 mov		%o2, %g1
Commit	Line	Data
1da177e4 LT	1	/* memcpy.S: Sparc optimized memcpy and memmove code
	2	* Hand optimized from GNU libc's memcpy and memmove
	3	* Copyright (C) 1991,1996 Free Software Foundation
	4	* Copyright (C) 1995 Linus Torvalds (Linus.Torvalds@helsinki.fi)
	5	* Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
	6	* Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
	7	* Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
	8	*/
	9
045b7de9	10	#define FUNC(x) \
1da177e4 LT	11	.globl x; \
1da177e4 LT	12	.type x,@function; \
045b7de9	13	.align 4; \
1da177e4 LT	14	x:
1da177e4 LT	15
1da177e4 LT	16	/* Both these macros have to start with exactly the same insn */
	17	#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
	18	ldd [%src + (offset) + 0x00], %t0; \
	19	ldd [%src + (offset) + 0x08], %t2; \
	20	ldd [%src + (offset) + 0x10], %t4; \
	21	ldd [%src + (offset) + 0x18], %t6; \
	22	st %t0, [%dst + (offset) + 0x00]; \
	23	st %t1, [%dst + (offset) + 0x04]; \
	24	st %t2, [%dst + (offset) + 0x08]; \
	25	st %t3, [%dst + (offset) + 0x0c]; \
	26	st %t4, [%dst + (offset) + 0x10]; \
	27	st %t5, [%dst + (offset) + 0x14]; \
	28	st %t6, [%dst + (offset) + 0x18]; \
	29	st %t7, [%dst + (offset) + 0x1c];
	30
	31	#define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
	32	ldd [%src + (offset) + 0x00], %t0; \
	33	ldd [%src + (offset) + 0x08], %t2; \
	34	ldd [%src + (offset) + 0x10], %t4; \
	35	ldd [%src + (offset) + 0x18], %t6; \
	36	std %t0, [%dst + (offset) + 0x00]; \
	37	std %t2, [%dst + (offset) + 0x08]; \
	38	std %t4, [%dst + (offset) + 0x10]; \
	39	std %t6, [%dst + (offset) + 0x18];
	40
	41	#define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
	42	ldd [%src - (offset) - 0x10], %t0; \
	43	ldd [%src - (offset) - 0x08], %t2; \
	44	st %t0, [%dst - (offset) - 0x10]; \
	45	st %t1, [%dst - (offset) - 0x0c]; \
	46	st %t2, [%dst - (offset) - 0x08]; \
	47	st %t3, [%dst - (offset) - 0x04];
	48
	49	#define MOVE_LASTALIGNCHUNK(src, dst, offset, t0, t1, t2, t3) \
	50	ldd [%src - (offset) - 0x10], %t0; \
	51	ldd [%src - (offset) - 0x08], %t2; \
	52	std %t0, [%dst - (offset) - 0x10]; \
	53	std %t2, [%dst - (offset) - 0x08];
	54
	55	#define MOVE_SHORTCHUNK(src, dst, offset, t0, t1) \
	56	ldub [%src - (offset) - 0x02], %t0; \
	57	ldub [%src - (offset) - 0x01], %t1; \
	58	stb %t0, [%dst - (offset) - 0x02]; \
	59	stb %t1, [%dst - (offset) - 0x01];
	60
	61	/* Both these macros have to start with exactly the same insn */
	62	#define RMOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
	63	ldd [%src - (offset) - 0x20], %t0; \
	64	ldd [%src - (offset) - 0x18], %t2; \
	65	ldd [%src - (offset) - 0x10], %t4; \
	66	ldd [%src - (offset) - 0x08], %t6; \
	67	st %t0, [%dst - (offset) - 0x20]; \
	68	st %t1, [%dst - (offset) - 0x1c]; \
	69	st %t2, [%dst - (offset) - 0x18]; \
	70	st %t3, [%dst - (offset) - 0x14]; \
	71	st %t4, [%dst - (offset) - 0x10]; \
	72	st %t5, [%dst - (offset) - 0x0c]; \
	73	st %t6, [%dst - (offset) - 0x08]; \
	74	st %t7, [%dst - (offset) - 0x04];
	75
	76	#define RMOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \
	77	ldd [%src - (offset) - 0x20], %t0; \
	78	ldd [%src - (offset) - 0x18], %t2; \
	79	ldd [%src - (offset) - 0x10], %t4; \
80	ldd [%src - (offset) - 0x08], %t6; \
81	std %t0, [%dst - (offset) - 0x20]; \
82	std %t2, [%dst - (offset) - 0x18]; \
83	std %t4, [%dst - (offset) - 0x10]; \
84	std %t6, [%dst - (offset) - 0x08];
85
86	#define RMOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \
87	ldd [%src + (offset) + 0x00], %t0; \
88	ldd [%src + (offset) + 0x08], %t2; \
89	st %t0, [%dst + (offset) + 0x00]; \
90	st %t1, [%dst + (offset) + 0x04]; \
91	st %t2, [%dst + (offset) + 0x08]; \
92	st %t3, [%dst + (offset) + 0x0c];
93
94	#define RMOVE_SHORTCHUNK(src, dst, offset, t0, t1) \
95	ldub [%src + (offset) + 0x00], %t0; \
96	ldub [%src + (offset) + 0x01], %t1; \
97	stb %t0, [%dst + (offset) + 0x00]; \
98	stb %t1, [%dst + (offset) + 0x01];
99
100	#define SMOVE_CHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, prev, shil, shir, offset2) \
101	ldd [%src + (offset) + 0x00], %t0; \
102	ldd [%src + (offset) + 0x08], %t2; \
103	srl %t0, shir, %t5; \
104	srl %t1, shir, %t6; \
105	sll %t0, shil, %t0; \
106	or %t5, %prev, %t5; \
107	sll %t1, shil, %prev; \
108	or %t6, %t0, %t0; \
109	srl %t2, shir, %t1; \
110	srl %t3, shir, %t6; \
111	sll %t2, shil, %t2; \
112	or %t1, %prev, %t1; \
113	std %t4, [%dst + (offset) + (offset2) - 0x04]; \
114	std %t0, [%dst + (offset) + (offset2) + 0x04]; \
115	sll %t3, shil, %prev; \
116	or %t6, %t2, %t4;
117
118	#define SMOVE_ALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, prev, shil, shir, offset2) \
119	ldd [%src + (offset) + 0x00], %t0; \
120	ldd [%src + (offset) + 0x08], %t2; \
121	srl %t0, shir, %t4; \
122	srl %t1, shir, %t5; \
123	sll %t0, shil, %t6; \
124	or %t4, %prev, %t0; \
125	sll %t1, shil, %prev; \
126	or %t5, %t6, %t1; \
127	srl %t2, shir, %t4; \
128	srl %t3, shir, %t5; \
129	sll %t2, shil, %t6; \
130	or %t4, %prev, %t2; \
131	sll %t3, shil, %prev; \
132	or %t5, %t6, %t3; \
133	std %t0, [%dst + (offset) + (offset2) + 0x00]; \
134	std %t2, [%dst + (offset) + (offset2) + 0x08];
135
136	.text
137	.align 4
138
1da177e4 LT	139	0:
	140	retl
	141	nop ! Only bcopy returns here and it retuns void...
	142
	143	#ifdef __KERNEL__
	144	FUNC(amemmove)
	145	FUNC(__memmove)
	146	#endif
	147	FUNC(memmove)
	148	cmp %o0, %o1
a52312b8	149	mov %o0, %g7
1da177e4 LT	150	bleu 9f
	151	sub %o0, %o1, %o4
	152
	153	add %o1, %o2, %o3
	154	cmp %o3, %o0
	155	bleu 0f
	156	andcc %o4, 3, %o5
	157
1da177e4 LT	158	add %o1, %o2, %o1
	159	add %o0, %o2, %o0
	160	sub %o1, 1, %o1
	161	sub %o0, 1, %o0
	162
	163	1: /* reverse_bytes */
	164
	165	ldub [%o1], %o4
	166	subcc %o2, 1, %o2
	167	stb %o4, [%o0]
	168	sub %o1, 1, %o1
	169	bne 1b
	170	sub %o0, 1, %o0
	171
	172	retl
a52312b8	173	mov %g7, %o0
1da177e4	174
1da177e4 LT	175	/* NOTE: This code is executed just for the cases,
	176	where %src (=%o1) & 3 is != 0.
	177	We need to align it to 4. So, for (%src & 3)
	178	1 we need to do ldub,lduh
	179	2 lduh
	180	3 just ldub
	181	so even if it looks weird, the branches
	182	are correct here. -jj
	183	*/
	184	78: /* dword_align */
	185
	186	andcc %o1, 1, %g0
	187	be 4f
	188	andcc %o1, 2, %g0
	189
	190	ldub [%o1], %g2
	191	add %o1, 1, %o1
	192	stb %g2, [%o0]
	193	sub %o2, 1, %o2
	194	bne 3f
	195	add %o0, 1, %o0
	196	4:
	197	lduh [%o1], %g2
	198	add %o1, 2, %o1
	199	sth %g2, [%o0]
	200	sub %o2, 2, %o2
	201	b 3f
	202	add %o0, 2, %o0
	203
1da177e4 LT	204	FUNC(memcpy) /* %o0=dst %o1=src %o2=len */
	205
	206	sub %o0, %o1, %o4
a52312b8	207	mov %o0, %g7
1da177e4 LT	208	9:
	209	andcc %o4, 3, %o5
	210	0:
	211	bne 86f
	212	cmp %o2, 15
	213
	214	bleu 90f
	215	andcc %o1, 3, %g0
	216
	217	bne 78b
	218	3:
	219	andcc %o1, 4, %g0
	220
	221	be 2f
	222	mov %o2, %g1
	223
	224	ld [%o1], %o4
	225	sub %g1, 4, %g1
	226	st %o4, [%o0]
	227	add %o1, 4, %o1
	228	add %o0, 4, %o0
	229	2:
21f74d36	230	andcc %g1, 0xffffff80, %g0
1da177e4 LT	231	be 3f
	232	andcc %o0, 4, %g0
	233
	234	be 82f + 4
	235	5:
	236	MOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
	237	MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
	238	MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
	239	MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
21f74d36	240	sub %g1, 128, %g1
1da177e4	241	add %o1, 128, %o1
21f74d36 DM	242	cmp %g1, 128
21f74d36 DM	243	bge 5b
1da177e4 LT	244	add %o0, 128, %o0
1da177e4 LT	245	3:
21f74d36	246	andcc %g1, 0x70, %g4
1da177e4 LT	247	be 80f
	248	andcc %g1, 8, %g0
	249
	250	sethi %hi(80f), %o5
21f74d36 DM	251	srl %g4, 1, %o4
	252	add %g4, %o4, %o4
	253	add %o1, %g4, %o1
1da177e4 LT	254	sub %o5, %o4, %o5
1da177e4 LT	255	jmpl %o5 + %lo(80f), %g0
21f74d36	256	add %o0, %g4, %o0
1da177e4 LT	257
	258	79: /* memcpy_table */
	259
	260	MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
	261	MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
	262	MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
	263	MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
	264	MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
	265	MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
	266	MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
	267
	268	80: /* memcpy_table_end */
	269	be 81f
	270	andcc %g1, 4, %g0
	271
	272	ldd [%o1], %g2
	273	add %o0, 8, %o0
	274	st %g2, [%o0 - 0x08]
	275	add %o1, 8, %o1
	276	st %g3, [%o0 - 0x04]
	277
	278	81: /* memcpy_last7 */
	279
	280	be 1f
	281	andcc %g1, 2, %g0
	282
	283	ld [%o1], %g2
	284	add %o1, 4, %o1
	285	st %g2, [%o0]
	286	add %o0, 4, %o0
	287	1:
	288	be 1f
	289	andcc %g1, 1, %g0
	290
	291	lduh [%o1], %g2
	292	add %o1, 2, %o1
	293	sth %g2, [%o0]
	294	add %o0, 2, %o0
	295	1:
	296	be 1f
	297	nop
	298
	299	ldub [%o1], %g2
	300	stb %g2, [%o0]
	301	1:
	302	retl
a52312b8	303	mov %g7, %o0
1da177e4 LT	304
	305	82: /* ldd_std */
	306	MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5)
	307	MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5)
	308	MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5)
	309	MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5)
21f74d36	310	subcc %g1, 128, %g1
1da177e4	311	add %o1, 128, %o1
21f74d36 DM	312	cmp %g1, 128
21f74d36 DM	313	bge 82b
1da177e4 LT	314	add %o0, 128, %o0
1da177e4 LT	315
21f74d36	316	andcc %g1, 0x70, %g4
1da177e4 LT	317	be 84f
	318	andcc %g1, 8, %g0
	319
	320	sethi %hi(84f), %o5
21f74d36 DM	321	add %o1, %g4, %o1
21f74d36 DM	322	sub %o5, %g4, %o5
1da177e4	323	jmpl %o5 + %lo(84f), %g0
21f74d36	324	add %o0, %g4, %o0
1da177e4 LT	325
	326	83: /* amemcpy_table */
	327
	328	MOVE_LASTALIGNCHUNK(o1, o0, 0x60, g2, g3, g4, g5)
	329	MOVE_LASTALIGNCHUNK(o1, o0, 0x50, g2, g3, g4, g5)
	330	MOVE_LASTALIGNCHUNK(o1, o0, 0x40, g2, g3, g4, g5)
	331	MOVE_LASTALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5)
	332	MOVE_LASTALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5)
	333	MOVE_LASTALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5)
	334	MOVE_LASTALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5)
	335
	336	84: /* amemcpy_table_end */
	337	be 85f
	338	andcc %g1, 4, %g0
	339
	340	ldd [%o1], %g2
	341	add %o0, 8, %o0
	342	std %g2, [%o0 - 0x08]
	343	add %o1, 8, %o1
	344	85: /* amemcpy_last7 */
	345	be 1f
	346	andcc %g1, 2, %g0
	347
	348	ld [%o1], %g2
	349	add %o1, 4, %o1
	350	st %g2, [%o0]
	351	add %o0, 4, %o0
	352	1:
	353	be 1f
	354	andcc %g1, 1, %g0
	355
	356	lduh [%o1], %g2
	357	add %o1, 2, %o1
	358	sth %g2, [%o0]
	359	add %o0, 2, %o0
	360	1:
	361	be 1f
	362	nop
	363
	364	ldub [%o1], %g2
	365	stb %g2, [%o0]
	366	1:
	367	retl
a52312b8	368	mov %g7, %o0
1da177e4	369
1da177e4 LT	370	86: /* non_aligned */
	371	cmp %o2, 6
	372	bleu 88f
21f74d36 DM	373	nop
	374
	375	save %sp, -96, %sp
	376	andcc %i0, 3, %g0
1da177e4	377	be 61f
21f74d36	378	andcc %i0, 1, %g0
1da177e4	379	be 60f
21f74d36	380	andcc %i0, 2, %g0
1da177e4	381
21f74d36 DM	382	ldub [%i1], %g5
	383	add %i1, 1, %i1
	384	stb %g5, [%i0]
	385	sub %i2, 1, %i2
1da177e4	386	bne 61f
21f74d36	387	add %i0, 1, %i0
1da177e4	388	60:
21f74d36 DM	389	ldub [%i1], %g3
	390	add %i1, 2, %i1
	391	stb %g3, [%i0]
	392	sub %i2, 2, %i2
	393	ldub [%i1 - 1], %g3
	394	add %i0, 2, %i0
	395	stb %g3, [%i0 - 1]
1da177e4	396	61:
21f74d36 DM	397	and %i1, 3, %g2
	398	and %i2, 0xc, %g3
	399	and %i1, -4, %i1
1da177e4 LT	400	cmp %g3, 4
	401	sll %g2, 3, %g4
	402	mov 32, %g2
	403	be 4f
21f74d36	404	sub %g2, %g4, %l0
1da177e4 LT	405
	406	blu 3f
	407	cmp %g3, 0x8
	408
	409	be 2f
21f74d36	410	srl %i2, 2, %g3
1da177e4	411
21f74d36 DM	412	ld [%i1], %i3
	413	add %i0, -8, %i0
	414	ld [%i1 + 4], %i4
1da177e4 LT	415	b 8f
	416	add %g3, 1, %g3
	417	2:
21f74d36 DM	418	ld [%i1], %i4
	419	add %i0, -12, %i0
	420	ld [%i1 + 4], %i5
1da177e4 LT	421	add %g3, 2, %g3
1da177e4 LT	422	b 9f
21f74d36	423	add %i1, -4, %i1
1da177e4	424	3:
21f74d36 DM	425	ld [%i1], %g1
	426	add %i0, -4, %i0
	427	ld [%i1 + 4], %i3
	428	srl %i2, 2, %g3
1da177e4	429	b 7f
21f74d36	430	add %i1, 4, %i1
1da177e4	431	4:
21f74d36 DM	432	ld [%i1], %i5
	433	cmp %i2, 7
	434	ld [%i1 + 4], %g1
	435	srl %i2, 2, %g3
1da177e4	436	bleu 10f
21f74d36	437	add %i1, 8, %i1
1da177e4	438
21f74d36	439	ld [%i1], %i3
1da177e4 LT	440	add %g3, -1, %g3
1da177e4 LT	441	5:
21f74d36 DM	442	sll %i5, %g4, %g2
21f74d36 DM	443	srl %g1, %l0, %g5
1da177e4	444	or %g2, %g5, %g2
21f74d36	445	st %g2, [%i0]
1da177e4	446	7:
21f74d36	447	ld [%i1 + 4], %i4
1da177e4	448	sll %g1, %g4, %g2
21f74d36	449	srl %i3, %l0, %g5
1da177e4	450	or %g2, %g5, %g2
21f74d36	451	st %g2, [%i0 + 4]
1da177e4	452	8:
21f74d36 DM	453	ld [%i1 + 8], %i5
	454	sll %i3, %g4, %g2
	455	srl %i4, %l0, %g5
1da177e4	456	or %g2, %g5, %g2
21f74d36	457	st %g2, [%i0 + 8]
1da177e4	458	9:
21f74d36 DM	459	ld [%i1 + 12], %g1
	460	sll %i4, %g4, %g2
	461	srl %i5, %l0, %g5
1da177e4 LT	462	addcc %g3, -4, %g3
1da177e4 LT	463	or %g2, %g5, %g2
21f74d36 DM	464	add %i1, 16, %i1
	465	st %g2, [%i0 + 12]
	466	add %i0, 16, %i0
1da177e4	467	bne,a 5b
21f74d36	468	ld [%i1], %i3
1da177e4	469	10:
21f74d36 DM	470	sll %i5, %g4, %g2
	471	srl %g1, %l0, %g5
	472	srl %l0, 3, %g3
1da177e4	473	or %g2, %g5, %g2
21f74d36 DM	474	sub %i1, %g3, %i1
	475	andcc %i2, 2, %g0
	476	st %g2, [%i0]
1da177e4	477	be 1f
21f74d36 DM	478	andcc %i2, 1, %g0
	479
	480	ldub [%i1], %g2
	481	add %i1, 2, %i1
	482	stb %g2, [%i0 + 4]
	483	add %i0, 2, %i0
	484	ldub [%i1 - 1], %g2
	485	stb %g2, [%i0 + 3]
1da177e4 LT	486	1:
	487	be 1f
	488	nop
21f74d36 DM	489	ldub [%i1], %g2
21f74d36 DM	490	stb %g2, [%i0 + 4]
1da177e4	491	1:
21f74d36	492	ret
a52312b8	493	restore %g7, %g0, %o0
1da177e4	494
1da177e4 LT	495	88: /* short_end */
	496
	497	and %o2, 0xe, %o3
	498	20:
	499	sethi %hi(89f), %o5
	500	sll %o3, 3, %o4
	501	add %o0, %o3, %o0
	502	sub %o5, %o4, %o5
	503	add %o1, %o3, %o1
	504	jmpl %o5 + %lo(89f), %g0
	505	andcc %o2, 1, %g0
	506
	507	MOVE_SHORTCHUNK(o1, o0, 0x0c, g2, g3)
	508	MOVE_SHORTCHUNK(o1, o0, 0x0a, g2, g3)
	509	MOVE_SHORTCHUNK(o1, o0, 0x08, g2, g3)
	510	MOVE_SHORTCHUNK(o1, o0, 0x06, g2, g3)
	511	MOVE_SHORTCHUNK(o1, o0, 0x04, g2, g3)
	512	MOVE_SHORTCHUNK(o1, o0, 0x02, g2, g3)
	513	MOVE_SHORTCHUNK(o1, o0, 0x00, g2, g3)
	514
	515	89: /* short_table_end */
	516
	517	be 1f
	518	nop
	519
	520	ldub [%o1], %g2
	521	stb %g2, [%o0]
	522	1:
	523	retl
a52312b8	524	mov %g7, %o0
1da177e4 LT	525
	526	90: /* short_aligned_end */
	527	bne 88b
	528	andcc %o2, 8, %g0
	529
	530	be 1f
	531	andcc %o2, 4, %g0
	532
	533	ld [%o1 + 0x00], %g2
	534	ld [%o1 + 0x04], %g3
	535	add %o1, 8, %o1
	536	st %g2, [%o0 + 0x00]
	537	st %g3, [%o0 + 0x04]
	538	add %o0, 8, %o0
	539	1:
	540	b 81b
	541	mov %o2, %g1