--- zzzz-none-000/linux-3.10.107/arch/mips/lib/memcpy.S	2017-06-27 09:49:32.000000000 +0000
+++ vr9-7490-729/linux-3.10.107/arch/mips/lib/memcpy.S	2021-11-10 11:53:54.000000000 +0000
@@ -91,6 +91,31 @@
 	PTR	9b, handler;			\
 	.previous
 
+    .data
+bug_text1: .ascii __FILE__"\0"
+bug_text2: .ascii "memcpy\0"
+bug_text3: .ascii "invalid src-addr see: dst=a0 src=a1\0"
+
+/**--------------------------------------------------------------------------------**\
+ * AVM: mbahr:
+ * Grosses Problem: 
+ * fuer memcpy und copy_from_user wird gleiche Funktion verwendet
+ * die Load-Befehle werden mit Fixup-Exception-Handler "abgefangen"
+ * diese duerfen aber nur bei User-Copy zuschlagen - und erfordern ein gesetztes
+ * AT (entspricht src+len). Ein memcpy kann also je nach AT grosse Bereiche des Kernels zerschreiben, 
+ * ohne das es sofort auffaellt.
+ * Absicherung:
+ * Als Indicator wird bei memcpy() initial AT auf zero gesetzt und dann ein 
+ * BUG ausgeloest, wenn im Fixup-Exceptionhandler AT==zero 
+\**--------------------------------------------------------------------------------**/
+#define BUG_MEMCPY(dst, src, reg, line)  \
+    movz a0, dst, reg; \
+    movz a1, src, reg; \
+1:	teq zero, reg, 12; \
+	.section __bug_debug_table,"a";\
+	PTR	1b, bug_text1, line, bug_text2, bug_text3;\
+	.previous
+
 /*
  * Only on the 64-bit kernel we can made use of 64-bit registers.
  */
@@ -116,6 +141,8 @@
 #define NBYTES 8
 #define LOG_NBYTES 3
 
+#define LOADK  ld
+
 /*
  * As we are sharing code base with the mips32 tree (which use the o32 ABI
  * register definitions). We need to redefine the register definitions from
@@ -152,19 +179,21 @@
 #define NBYTES 4
 #define LOG_NBYTES 2
 
+#define LOADK  lw
+
 #endif /* USE_DOUBLE */
 
 #ifdef CONFIG_CPU_LITTLE_ENDIAN
 #define LDFIRST LOADR
-#define LDREST	LOADL
+#define LDREST  LOADL
 #define STFIRST STORER
-#define STREST	STOREL
+#define STREST  STOREL
 #define SHIFT_DISCARD SLLV
 #else
 #define LDFIRST LOADL
-#define LDREST	LOADR
+#define LDREST  LOADR
 #define STFIRST STOREL
-#define STREST	STORER
+#define STREST  STORER
 #define SHIFT_DISCARD SRLV
 #endif
 
@@ -200,6 +229,7 @@
 LEAF(memcpy)					/* a0=dst a1=src a2=len */
 	move	v0, dst				/* return value */
 .L__memcpy:
+    move    AT, zero            /* AVM: mbahr: kernel space-indicator */
 FEXPORT(__copy_user)
 	li	t6, 0	/* not inatomic */
 __copy_user_common:
@@ -235,7 +265,7 @@
 	 * src and dst are aligned; need to compute rem
 	 */
 .Lboth_aligned:
-	 SRL	t0, len, LOG_NBYTES+3	 # +3 for 8 units/iter
+	 SRL	t0, len, LOG_NBYTES+3    # +3 for 8 units/iter
 	beqz	t0, .Lcleanup_both_aligned # len < 8*NBYTES
 	 and	rem, len, (8*NBYTES-1)	 # rem = len % (8*NBYTES)
 	PREF(	0, 3*32(src) )
@@ -313,7 +343,7 @@
 	/*
 	 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
 	 * A loop would do only a byte at a time with possible branch
-	 * mispredicts.	 Can't do an explicit LOAD dst,mask,or,STORE
+	 * mispredicts.  Can't do an explicit LOAD dst,mask,or,STORE
 	 * because can't assume read-access to dst.  Instead, use
 	 * STREST dst, which doesn't require read access to dst.
 	 *
@@ -327,7 +357,7 @@
 	li	bits, 8*NBYTES
 	SLL	rem, len, 3	# rem = number of bits to keep
 EXC(	LOAD	t0, 0(src),		.Ll_exc)
-	SUB	bits, bits, rem # bits = number of bits to discard
+	SUB	bits, bits, rem	# bits = number of bits to discard
 	SHIFT_DISCARD t0, t0, bits
 EXC(	STREST	t0, -1(t1),		.Ls_exc)
 	jr	ra
@@ -343,7 +373,7 @@
 	 * Set match = (src and dst have same alignment)
 	 */
 #define match rem
-EXC(	LDFIRST t3, FIRST(0)(src),	.Ll_exc)
+EXC(	LDFIRST	t3, FIRST(0)(src),	.Ll_exc)
 	ADD	t2, zero, NBYTES
 EXC(	LDREST	t3, REST(0)(src),	.Ll_exc_copy)
 	SUB	t2, t2, t1	# t2 = number of bytes copied
@@ -357,10 +387,10 @@
 	 ADD	src, src, t2
 
 .Lsrc_unaligned_dst_aligned:
-	SRL	t0, len, LOG_NBYTES+2	 # +2 for 4 units/iter
+	SRL	t0, len, LOG_NBYTES+2    # +2 for 4 units/iter
 	PREF(	0, 3*32(src) )
 	beqz	t0, .Lcleanup_src_unaligned
-	 and	rem, len, (4*NBYTES-1)	 # rem = len % 4*NBYTES
+	 and	rem, len, (4*NBYTES-1)   # rem = len % 4*NBYTES
 	PREF(	1, 3*32(dst) )
 1:
 /*
@@ -370,13 +400,13 @@
  * are to the same unit (unless src is aligned, but it's not).
  */
 	R10KCBARRIER(0(ra))
-EXC(	LDFIRST t0, FIRST(0)(src),	.Ll_exc)
-EXC(	LDFIRST t1, FIRST(1)(src),	.Ll_exc_copy)
-	SUB	len, len, 4*NBYTES
+EXC(	LDFIRST	t0, FIRST(0)(src),	.Ll_exc)
+EXC(	LDFIRST	t1, FIRST(1)(src),	.Ll_exc_copy)
+	SUB     len, len, 4*NBYTES
 EXC(	LDREST	t0, REST(0)(src),	.Ll_exc_copy)
 EXC(	LDREST	t1, REST(1)(src),	.Ll_exc_copy)
-EXC(	LDFIRST t2, FIRST(2)(src),	.Ll_exc_copy)
-EXC(	LDFIRST t3, FIRST(3)(src),	.Ll_exc_copy)
+EXC(	LDFIRST	t2, FIRST(2)(src),	.Ll_exc_copy)
+EXC(	LDFIRST	t3, FIRST(3)(src),	.Ll_exc_copy)
 EXC(	LDREST	t2, REST(2)(src),	.Ll_exc_copy)
 EXC(	LDREST	t3, REST(3)(src),	.Ll_exc_copy)
 	PREF(	0, 9*32(src) )		# 0 is PREF_LOAD  (not streamed)
@@ -388,7 +418,7 @@
 EXC(	STORE	t1, UNIT(1)(dst),	.Ls_exc_p3u)
 EXC(	STORE	t2, UNIT(2)(dst),	.Ls_exc_p2u)
 EXC(	STORE	t3, UNIT(3)(dst),	.Ls_exc_p1u)
-	PREF(	1, 9*32(dst) )		# 1 is PREF_STORE (not streamed)
+	PREF(	1, 9*32(dst) )     	# 1 is PREF_STORE (not streamed)
 	.set	reorder				/* DADDI_WAR */
 	ADD	dst, dst, 4*NBYTES
 	bne	len, rem, 1b
@@ -451,9 +481,9 @@
 	 *
 	 * Assumes src < THREAD_BUADDR($28)
 	 */
-	LOAD	t0, TI_TASK($28)
+	LOADK   t0, TI_TASK($28)
 	 nop
-	LOAD	t0, THREAD_BUADDR(t0)
+	LOADK   t0, THREAD_BUADDR(t0)
 1:
 EXC(	lb	t1, 0(src),	.Ll_exc)
 	ADD	src, src, 1
@@ -463,12 +493,14 @@
 	bne	src, t0, 1b
 	.set	noreorder
 .Ll_exc:
-	LOAD	t0, TI_TASK($28)
+	LOADK   t0, TI_TASK($28)
 	 nop
-	LOAD	t0, THREAD_BUADDR(t0)	# t0 is just past last good address
+	LOADK   t0, THREAD_BUADDR(t0)   # t0 is just past last good address
 	 nop
-	SUB	len, AT, t0		# len number of uncopied bytes
 	bnez	t6, .Ldone	/* Skip the zeroing part if inatomic */
+	 SUB     len, AT, t0            # len number of uncopied bytes
+     BUG_MEMCPY(dst, src, AT, __LINE__)
+     
 	/*
 	 * Here's where we rely on src and dst being incremented in tandem,
 	 *   See (3) above.
@@ -502,7 +534,7 @@
 
 
 #define SEXC(n)							\
-	.set	reorder;			/* DADDI_WAR */ \
+	.set	reorder;			/* DADDI_WAR */	\
 .Ls_exc_p ## n ## u:						\
 	ADD	len, len, n*NBYTES;				\
 	jr	ra;						\
@@ -575,3 +607,942 @@
 	jr	ra
 	 move	a2, zero
 	END(__rmemcpy)
+
+#ifdef CONFIG_EVA
+
+	.set    eva
+
+LEAF(__copy_fromuser_inatomic)
+	b       __copy_fromuser_common
+	 li	t6, 1
+	END(__copy_fromuser_inatomic)
+
+#undef  LOAD
+#undef  LOADL
+#undef  LOADR
+#undef  STORE
+#undef  STOREL
+#undef  STORER
+#undef  LDFIRST
+#undef  LDREST
+#undef  STFIRST
+#undef  STREST
+#undef  SHIFT_DISCARD
+#undef  COPY_BYTE
+#undef  SEXC
+
+#define LOAD   lwe
+#define LOADL  lwle
+#define LOADR  lwre
+#define STOREL swl
+#define STORER swr
+#define STORE  sw
+
+#ifdef CONFIG_CPU_LITTLE_ENDIAN
+#define LDFIRST LOADR
+#define LDREST  LOADL
+#define STFIRST STORER
+#define STREST  STOREL
+#define SHIFT_DISCARD SLLV
+#else
+#define LDFIRST LOADL
+#define LDREST  LOADR
+#define STFIRST STOREL
+#define STREST  STORER
+#define SHIFT_DISCARD SRLV
+#endif
+
+LEAF(__copy_fromuser)
+	li	t6, 0	/* not inatomic */
+__copy_fromuser_common:
+	/*
+	 * Note: dst & src may be unaligned, len may be 0
+	 * Temps
+	 */
+
+	R10KCBARRIER(0(ra))
+	/*
+	 * The "issue break"s below are very approximate.
+	 * Issue delays for dcache fills will perturb the schedule, as will
+	 * load queue full replay traps, etc.
+	 *
+	 * If len < NBYTES use byte operations.
+	 */
+	PREFE(  0, 0(src) )
+	PREF(	1, 0(dst) )
+	sltu	t2, len, NBYTES
+	and	t1, dst, ADDRMASK
+	PREFE(  0, 1*32(src) )
+	PREF(	1, 1*32(dst) )
+	bnez    t2, .LFcopy_bytes_checklen
+	 and	t0, src, ADDRMASK
+	PREFE(  0, 2*32(src) )
+	PREF(	1, 2*32(dst) )
+	bnez    t1, .LFdst_unaligned
+	 nop
+	bnez    t0, .LFsrc_unaligned_dst_aligned
+	/*
+	 * use delay slot for fall-through
+	 * src and dst are aligned; need to compute rem
+	 */
+.LFboth_aligned:
+	 SRL	t0, len, LOG_NBYTES+3    # +3 for 8 units/iter
+	beqz    t0, .LFcleanup_both_aligned # len < 8*NBYTES
+	 and	rem, len, (8*NBYTES-1)	 # rem = len % (8*NBYTES)
+	PREFE(  0, 3*32(src) )
+	PREF(	1, 3*32(dst) )
+	.align	4
+1:
+	R10KCBARRIER(0(ra))
+EXC(    LOAD    t0, UNIT(0)(src),       .LFl_exc)
+EXC(    LOAD    t1, UNIT(1)(src),       .LFl_exc_copy)
+EXC(    LOAD    t2, UNIT(2)(src),       .LFl_exc_copy)
+EXC(    LOAD    t3, UNIT(3)(src),       .LFl_exc_copy)
+	SUB	len, len, 8*NBYTES
+EXC(    LOAD    t4, UNIT(4)(src),       .LFl_exc_copy)
+EXC(    LOAD    t7, UNIT(5)(src),       .LFl_exc_copy)
+	STORE   t0, UNIT(0)(dst)
+	STORE   t1, UNIT(1)(dst)
+EXC(    LOAD    t0, UNIT(6)(src),       .LFl_exc_copy)
+EXC(    LOAD    t1, UNIT(7)(src),       .LFl_exc_copy)
+	ADD	src, src, 8*NBYTES
+	ADD	dst, dst, 8*NBYTES
+	STORE   t2, UNIT(-6)(dst)
+	STORE   t3, UNIT(-5)(dst)
+	STORE   t4, UNIT(-4)(dst)
+	STORE   t7, UNIT(-3)(dst)
+	STORE   t0, UNIT(-2)(dst)
+	STORE   t1, UNIT(-1)(dst)
+	PREFE(  0, 8*32(src) )
+	PREF(	1, 8*32(dst) )
+	bne	len, rem, 1b
+	 nop
+
+	/*
+	 * len == rem == the number of bytes left to copy < 8*NBYTES
+	 */
+.LFcleanup_both_aligned:
+	beqz	len, .Ldone
+	 sltu	t0, len, 4*NBYTES
+	bnez    t0, .LFless_than_4units
+	 and	rem, len, (NBYTES-1)	# rem = len % NBYTES
+	/*
+	 * len >= 4*NBYTES
+	 */
+EXC(    LOAD    t0, UNIT(0)(src),       .LFl_exc)
+EXC(    LOAD    t1, UNIT(1)(src),       .LFl_exc_copy)
+EXC(    LOAD    t2, UNIT(2)(src),       .LFl_exc_copy)
+EXC(    LOAD    t3, UNIT(3)(src),       .LFl_exc_copy)
+	SUB	len, len, 4*NBYTES
+	ADD	src, src, 4*NBYTES
+	R10KCBARRIER(0(ra))
+	STORE   t0, UNIT(0)(dst)
+	STORE   t1, UNIT(1)(dst)
+	STORE   t2, UNIT(2)(dst)
+	STORE   t3, UNIT(3)(dst)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, 4*NBYTES
+	beqz	len, .Ldone
+	.set	noreorder
+.LFless_than_4units:
+	/*
+	 * rem = len % NBYTES
+	 */
+	beq     rem, len, .LFcopy_bytes
+	 nop
+1:
+	R10KCBARRIER(0(ra))
+EXC(    LOAD    t0, 0(src),             .LFl_exc)
+	ADD	src, src, NBYTES
+	SUB	len, len, NBYTES
+	STORE   t0, 0(dst)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, NBYTES
+	bne	rem, len, 1b
+	.set	noreorder
+
+	/*
+	 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
+	 * A loop would do only a byte at a time with possible branch
+	 * mispredicts.  Can't do an explicit LOAD dst,mask,or,STORE
+	 * because can't assume read-access to dst.  Instead, use
+	 * STREST dst, which doesn't require read access to dst.
+	 *
+	 * This code should perform better than a simple loop on modern,
+	 * wide-issue mips processors because the code has fewer branches and
+	 * more instruction-level parallelism.
+	 */
+	beqz	len, .Ldone
+	 ADD	t1, dst, len	# t1 is just past last byte of dst
+	li	bits, 8*NBYTES
+	SLL	rem, len, 3	# rem = number of bits to keep
+EXC(    LOAD    t0, 0(src),             .LFl_exc)
+	SUB	bits, bits, rem	# bits = number of bits to discard
+	SHIFT_DISCARD t0, t0, bits
+	STREST  t0, -1(t1)
+	jr	ra
+	 move	len, zero
+.LFdst_unaligned:
+	/*
+	 * dst is unaligned
+	 * t0 = src & ADDRMASK
+	 * t1 = dst & ADDRMASK; T1 > 0
+	 * len >= NBYTES
+	 *
+	 * Copy enough bytes to align dst
+	 * Set match = (src and dst have same alignment)
+	 */
+#define match rem
+EXC(    LDFIRST t3, FIRST(0)(src),      .LFl_exc)
+	ADD	t2, zero, NBYTES
+EXC(    LDREST  t3, REST(0)(src),       .LFl_exc_copy)
+	SUB	t2, t2, t1	# t2 = number of bytes copied
+	xor	match, t0, t1
+	R10KCBARRIER(0(ra))
+	STFIRST t3, FIRST(0)(dst)
+	beq	len, t2, .Ldone
+	 SUB	len, len, t2
+	ADD	dst, dst, t2
+	beqz    match, .LFboth_aligned
+	 ADD	src, src, t2
+
+.LFsrc_unaligned_dst_aligned:
+	SRL	t0, len, LOG_NBYTES+2    # +2 for 4 units/iter
+	PREFE(  0, 3*32(src) )
+	beqz    t0, .LFcleanup_src_unaligned
+	 and	rem, len, (4*NBYTES-1)   # rem = len % 4*NBYTES
+	PREF(	1, 3*32(dst) )
+1:
+/*
+ * Avoid consecutive LD*'s to the same register since some mips
+ * implementations can't issue them in the same cycle.
+ * It's OK to load FIRST(N+1) before REST(N) because the two addresses
+ * are to the same unit (unless src is aligned, but it's not).
+ */
+	R10KCBARRIER(0(ra))
+EXC(    LDFIRST t0, FIRST(0)(src),      .LFl_exc)
+EXC(    LDFIRST t1, FIRST(1)(src),      .LFl_exc_copy)
+	SUB     len, len, 4*NBYTES
+EXC(    LDREST  t0, REST(0)(src),       .LFl_exc_copy)
+EXC(    LDREST  t1, REST(1)(src),       .LFl_exc_copy)
+EXC(    LDFIRST t2, FIRST(2)(src),      .LFl_exc_copy)
+EXC(    LDFIRST t3, FIRST(3)(src),      .LFl_exc_copy)
+EXC(    LDREST  t2, REST(2)(src),       .LFl_exc_copy)
+EXC(    LDREST  t3, REST(3)(src),       .LFl_exc_copy)
+	PREFE(  0, 9*32(src) )          # 0 is PREF_LOAD  (not streamed)
+	ADD	src, src, 4*NBYTES
+#ifdef CONFIG_CPU_SB1
+	nop				# improves slotting
+#endif
+	STORE   t0, UNIT(0)(dst)
+	STORE   t1, UNIT(1)(dst)
+	STORE   t2, UNIT(2)(dst)
+	STORE   t3, UNIT(3)(dst)
+	PREF(	1, 9*32(dst) )     	# 1 is PREF_STORE (not streamed)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, 4*NBYTES
+	bne	len, rem, 1b
+	.set	noreorder
+
+.LFcleanup_src_unaligned:
+	beqz	len, .Ldone
+	 and	rem, len, NBYTES-1  # rem = len % NBYTES
+	beq     rem, len, .LFcopy_bytes
+	 nop
+1:
+	R10KCBARRIER(0(ra))
+EXC(    LDFIRST t0, FIRST(0)(src),      .LFl_exc)
+EXC(    LDREST  t0, REST(0)(src),       .LFl_exc_copy)
+	ADD	src, src, NBYTES
+	SUB	len, len, NBYTES
+	STORE   t0, 0(dst)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, NBYTES
+	bne	len, rem, 1b
+	.set	noreorder
+
+.LFcopy_bytes_checklen:
+	beqz	len, .Ldone
+	 nop
+.LFcopy_bytes:
+	/* 0 < len < NBYTES  */
+	R10KCBARRIER(0(ra))
+#define COPY_BYTE(N)			\
+EXC(    lbe      t0, N(src), .LFl_exc);   \
+	SUB	len, len, 1;		\
+	beqz	len, .Ldone;		\
+	 sb     t0, N(dst)
+
+	COPY_BYTE(0)
+	COPY_BYTE(1)
+#ifdef USE_DOUBLE
+	COPY_BYTE(2)
+	COPY_BYTE(3)
+	COPY_BYTE(4)
+	COPY_BYTE(5)
+#endif
+EXC(    lbe     t0, NBYTES-2(src), .LFl_exc)
+	SUB	len, len, 1
+	jr	ra
+	 sb     t0, NBYTES-2(dst)
+
+.LFl_exc_copy:
+	/*
+	 * Copy bytes from src until faulting load address (or until a
+	 * lb faults)
+	 *
+	 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
+	 * may be more than a byte beyond the last address.
+	 * Hence, the lb below may get an exception.
+	 *
+	 * Assumes src < THREAD_BUADDR($28)
+	 */
+	LOADK   t0, TI_TASK($28)
+	addi    t0, t0, THREAD_BUADDR
+	LOADK   t0, 0(t0)
+1:
+EXC(    lbe     t1, 0(src),     .LFl_exc)
+	ADD	src, src, 1
+	sb	t1, 0(dst)	# can't fault -- we're copy_from_user
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, 1
+	bne	src, t0, 1b
+	.set	noreorder
+.LFl_exc:
+	LOADK   t0, TI_TASK($28)
+	addi    t0, t0, THREAD_BUADDR
+	LOADK   t0, 0(t0)   # t0 is just past last good address
+	bnez	t6, .Ldone	/* Skip the zeroing part if inatomic */
+	SUB    len, AT, t0             # len number of uncopied bytes
+    BUG_MEMCPY(dst, src, AT, __LINE__)
+	/*
+	 * Here's where we rely on src and dst being incremented in tandem,
+	 *   See (3) above.
+	 * dst += (fault addr - src) to put dst at first byte to clear
+	 */
+	ADD	dst, t0			# compute start address in a1
+	SUB	dst, src
+	/*
+	 * Clear len bytes starting at dst.  Can't call __bzero because it
+	 * might modify len.  An inefficient loop for these rare times...
+	 */
+	.set	reorder				/* DADDI_WAR */
+	SUB	src, len, 1
+	beqz	len, .Ldone
+	.set	noreorder
+1:	sb	zero, 0(dst)
+	ADD	dst, dst, 1
+#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
+	bnez	src, 1b
+	 SUB	src, src, 1
+#else
+	.set	push
+	.set	noat
+	li	v1, 1
+	bnez	src, 1b
+	 SUB	src, src, v1
+	.set	pop
+#endif
+	jr	ra
+	 nop
+	END(__copy_fromuser)
+
+
+#undef  LOAD
+#undef  LOADL
+#undef  LOADR
+#undef  STORE
+#undef  STOREL
+#undef  STORER
+#undef  LDFIRST
+#undef  LDREST
+#undef  STFIRST
+#undef  STREST
+#undef  SHIFT_DISCARD
+#undef  COPY_BYTE
+#undef  SEXC
+
+#define LOAD   lw
+#define LOADL  lwl
+#define LOADR  lwr
+#define STOREL swle
+#define STORER swre
+#define STORE  swe
+
+#ifdef CONFIG_CPU_LITTLE_ENDIAN
+#define LDFIRST LOADR
+#define LDREST  LOADL
+#define STFIRST STORER
+#define STREST  STOREL
+#define SHIFT_DISCARD SLLV
+#else
+#define LDFIRST LOADL
+#define LDREST  LOADR
+#define STFIRST STOREL
+#define STREST  STORER
+#define SHIFT_DISCARD SRLV
+#endif
+
+LEAF(__copy_touser)
+	/*
+	 * Note: dst & src may be unaligned, len may be 0
+	 * Temps
+	 */
+
+	R10KCBARRIER(0(ra))
+	/*
+	 * The "issue break"s below are very approximate.
+	 * Issue delays for dcache fills will perturb the schedule, as will
+	 * load queue full replay traps, etc.
+	 *
+	 * If len < NBYTES use byte operations.
+	 */
+	PREF(	0, 0(src) )
+	PREFE(  1, 0(dst) )
+	sltu	t2, len, NBYTES
+	and	t1, dst, ADDRMASK
+	PREF(	0, 1*32(src) )
+	PREFE(  1, 1*32(dst) )
+	bnez    t2, .LTcopy_bytes_checklen
+	 and	t0, src, ADDRMASK
+	PREF(	0, 2*32(src) )
+	PREFE(  1, 2*32(dst) )
+	bnez    t1, .LTdst_unaligned
+	 nop
+	bnez    t0, .LTsrc_unaligned_dst_aligned
+	/*
+	 * use delay slot for fall-through
+	 * src and dst are aligned; need to compute rem
+	 */
+.LTboth_aligned:
+	 SRL	t0, len, LOG_NBYTES+3    # +3 for 8 units/iter
+	beqz    t0, .LTcleanup_both_aligned # len < 8*NBYTES
+	 and	rem, len, (8*NBYTES-1)	 # rem = len % (8*NBYTES)
+	PREF(	0, 3*32(src) )
+	PREFE(  1, 3*32(dst) )
+	.align	4
+1:
+	R10KCBARRIER(0(ra))
+	LOAD    t0, UNIT(0)(src)
+	LOAD    t1, UNIT(1)(src)
+	LOAD    t2, UNIT(2)(src)
+	LOAD    t3, UNIT(3)(src)
+	SUB     len, len, 8*NBYTES
+	LOAD    t4, UNIT(4)(src)
+	LOAD    t7, UNIT(5)(src)
+EXC(    STORE   t0, UNIT(0)(dst),       .Ls_exc_p8u)
+EXC(    STORE   t1, UNIT(1)(dst),       .Ls_exc_p7u)
+	LOAD    t0, UNIT(6)(src)
+	LOAD    t1, UNIT(7)(src)
+	ADD     src, src, 8*NBYTES
+	ADD     dst, dst, 8*NBYTES
+EXC(    STORE   t2, UNIT(-6)(dst),      .Ls_exc_p6u)
+EXC(    STORE   t3, UNIT(-5)(dst),      .Ls_exc_p5u)
+EXC(    STORE   t4, UNIT(-4)(dst),      .Ls_exc_p4u)
+EXC(    STORE   t7, UNIT(-3)(dst),      .Ls_exc_p3u)
+EXC(    STORE   t0, UNIT(-2)(dst),      .Ls_exc_p2u)
+EXC(    STORE   t1, UNIT(-1)(dst),      .Ls_exc_p1u)
+	PREF(	0, 8*32(src) )
+	PREFE(  1, 8*32(dst) )
+	bne	len, rem, 1b
+	 nop
+
+	/*
+	 * len == rem == the number of bytes left to copy < 8*NBYTES
+	 */
+.LTcleanup_both_aligned:
+	beqz	len, .Ldone
+	 sltu	t0, len, 4*NBYTES
+	bnez    t0, .LTless_than_4units
+	 and	rem, len, (NBYTES-1)	# rem = len % NBYTES
+	/*
+	 * len >= 4*NBYTES
+	 */
+	LOAD    t0, UNIT(0)(src)
+	LOAD    t1, UNIT(1)(src)
+	LOAD    t2, UNIT(2)(src)
+	LOAD    t3, UNIT(3)(src)
+	SUB	len, len, 4*NBYTES
+	ADD	src, src, 4*NBYTES
+	R10KCBARRIER(0(ra))
+EXC(    STORE   t0, UNIT(0)(dst),       .Ls_exc_p4u)
+EXC(    STORE   t1, UNIT(1)(dst),       .Ls_exc_p3u)
+EXC(    STORE   t2, UNIT(2)(dst),       .Ls_exc_p2u)
+EXC(    STORE   t3, UNIT(3)(dst),       .Ls_exc_p1u)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, 4*NBYTES
+	beqz	len, .Ldone
+	.set	noreorder
+.LTless_than_4units:
+	/*
+	 * rem = len % NBYTES
+	 */
+	beq     rem, len, .LTcopy_bytes
+	 nop
+1:
+	R10KCBARRIER(0(ra))
+	LOAD    t0, 0(src)
+	ADD	src, src, NBYTES
+	SUB	len, len, NBYTES
+EXC(    STORE   t0, 0(dst),             .Ls_exc_p1u)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, NBYTES
+	bne	rem, len, 1b
+	.set	noreorder
+
+	/*
+	 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
+	 * A loop would do only a byte at a time with possible branch
+	 * mispredicts.  Can't do an explicit LOAD dst,mask,or,STORE
+	 * because can't assume read-access to dst.  Instead, use
+	 * STREST dst, which doesn't require read access to dst.
+	 *
+	 * This code should perform better than a simple loop on modern,
+	 * wide-issue mips processors because the code has fewer branches and
+	 * more instruction-level parallelism.
+	 */
+	beqz	len, .Ldone
+	 ADD	t1, dst, len	# t1 is just past last byte of dst
+	li	bits, 8*NBYTES
+	SLL	rem, len, 3	# rem = number of bits to keep
+	LOAD    t0, 0(src)
+	SUB	bits, bits, rem	# bits = number of bits to discard
+	SHIFT_DISCARD t0, t0, bits
+EXC(    STREST  t0, -1(t1),             .Ls_exc)
+	jr	ra
+	 move	len, zero
+.LTdst_unaligned:
+	/*
+	 * dst is unaligned
+	 * t0 = src & ADDRMASK
+	 * t1 = dst & ADDRMASK; T1 > 0
+	 * len >= NBYTES
+	 *
+	 * Copy enough bytes to align dst
+	 * Set match = (src and dst have same alignment)
+	 */
+	LDFIRST t3, FIRST(0)(src)
+	ADD     t2, zero, NBYTES
+	LDREST  t3, REST(0)(src)
+	SUB	t2, t2, t1	# t2 = number of bytes copied
+	xor	match, t0, t1
+	R10KCBARRIER(0(ra))
+EXC(    STFIRST t3, FIRST(0)(dst),      .Ls_exc)
+	beq	len, t2, .Ldone
+	 SUB	len, len, t2
+	ADD	dst, dst, t2
+	beqz    match, .LTboth_aligned
+	 ADD	src, src, t2
+
+.LTsrc_unaligned_dst_aligned:
+	SRL	t0, len, LOG_NBYTES+2    # +2 for 4 units/iter
+	PREF(	0, 3*32(src) )
+	beqz    t0, .LTcleanup_src_unaligned
+	 and	rem, len, (4*NBYTES-1)   # rem = len % 4*NBYTES
+	PREFE(  1, 3*32(dst) )
+1:
+/*
+ * Avoid consecutive LD*'s to the same register since some mips
+ * implementations can't issue them in the same cycle.
+ * It's OK to load FIRST(N+1) before REST(N) because the two addresses
+ * are to the same unit (unless src is aligned, but it's not).
+ */
+	R10KCBARRIER(0(ra))
+	LDFIRST t0, FIRST(0)(src)
+	LDFIRST t1, FIRST(1)(src)
+	SUB     len, len, 4*NBYTES
+	LDREST  t0, REST(0)(src)
+	LDREST  t1, REST(1)(src)
+	LDFIRST t2, FIRST(2)(src)
+	LDFIRST t3, FIRST(3)(src)
+	LDREST  t2, REST(2)(src)
+	LDREST  t3, REST(3)(src)
+	PREF(	0, 9*32(src) )		# 0 is PREF_LOAD  (not streamed)
+	ADD	src, src, 4*NBYTES
+#ifdef CONFIG_CPU_SB1
+	nop				# improves slotting
+#endif
+EXC(    STORE   t0, UNIT(0)(dst),       .Ls_exc_p4u)
+EXC(    STORE   t1, UNIT(1)(dst),       .Ls_exc_p3u)
+EXC(    STORE   t2, UNIT(2)(dst),       .Ls_exc_p2u)
+EXC(    STORE   t3, UNIT(3)(dst),       .Ls_exc_p1u)
+	PREFE(  1, 9*32(dst) )          # 1 is PREF_STORE (not streamed)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, 4*NBYTES
+	bne	len, rem, 1b
+	.set	noreorder
+
+.LTcleanup_src_unaligned:
+	beqz	len, .Ldone
+	 and	rem, len, NBYTES-1  # rem = len % NBYTES
+	beq     rem, len, .LTcopy_bytes
+	 nop
+1:
+	R10KCBARRIER(0(ra))
+	LDFIRST t0, FIRST(0)(src)
+	LDREST  t0, REST(0)(src)
+	ADD	src, src, NBYTES
+	SUB	len, len, NBYTES
+EXC(    STORE   t0, 0(dst),             .Ls_exc_p1u)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, NBYTES
+	bne	len, rem, 1b
+	.set	noreorder
+
+.LTcopy_bytes_checklen:
+	beqz	len, .Ldone
+	 nop
+.LTcopy_bytes:
+	/* 0 < len < NBYTES  */
+	R10KCBARRIER(0(ra))
+#define COPY_BYTE(N)			\
+	lb      t0, N(src);             \
+	SUB	len, len, 1;		\
+	beqz	len, .Ldone;		\
+EXC(     sbe    t0, N(dst), .Ls_exc_p1)
+
+	COPY_BYTE(0)
+	COPY_BYTE(1)
+#ifdef USE_DOUBLE
+	COPY_BYTE(2)
+	COPY_BYTE(3)
+	COPY_BYTE(4)
+	COPY_BYTE(5)
+#endif
+	lb      t0, NBYTES-2(src)
+	SUB	len, len, 1
+	jr	ra
+EXC(     sbe    t0, NBYTES-2(dst), .Ls_exc_p1)
+	END(__copy_touser)
+
+
+#undef  LOAD
+#undef  LOADL
+#undef  LOADR
+#undef  STORE
+#undef  STOREL
+#undef  STORER
+#undef  LDFIRST
+#undef  LDREST
+#undef  STFIRST
+#undef  STREST
+#undef  SHIFT_DISCARD
+#undef  COPY_BYTE
+#undef  SEXC
+
+#define LOAD   lwe
+#define LOADL  lwle
+#define LOADR  lwre
+#define STOREL swle
+#define STORER swre
+#define STORE  swe
+
+#ifdef CONFIG_CPU_LITTLE_ENDIAN
+#define LDFIRST LOADR
+#define LDREST  LOADL
+#define STFIRST STORER
+#define STREST  STOREL
+#define SHIFT_DISCARD SLLV
+#else
+#define LDFIRST LOADL
+#define LDREST  LOADR
+#define STFIRST STOREL
+#define STREST  STORER
+#define SHIFT_DISCARD SRLV
+#endif
+
+
+LEAF(__copy_inuser)
+	/*
+	 * Note: dst & src may be unaligned, len may be 0
+	 * Temps
+	 */
+
+	R10KCBARRIER(0(ra))
+	/*
+	 * The "issue break"s below are very approximate.
+	 * Issue delays for dcache fills will perturb the schedule, as will
+	 * load queue full replay traps, etc.
+	 *
+	 * If len < NBYTES use byte operations.
+	 */
+	PREFE(  0, 0(src) )
+	PREFE(  1, 0(dst) )
+	sltu	t2, len, NBYTES
+	and	t1, dst, ADDRMASK
+	PREFE(  0, 1*32(src) )
+	PREFE(  1, 1*32(dst) )
+	bnez    t2, .LIcopy_bytes_checklen
+	 and	t0, src, ADDRMASK
+	PREFE(  0, 2*32(src) )
+	PREFE(  1, 2*32(dst) )
+	bnez    t1, .LIdst_unaligned
+	 nop
+	bnez    t0, .LIsrc_unaligned_dst_aligned
+	/*
+	 * use delay slot for fall-through
+	 * src and dst are aligned; need to compute rem
+	 */
+.LIboth_aligned:
+	 SRL	t0, len, LOG_NBYTES+3    # +3 for 8 units/iter
+	beqz    t0, .LIcleanup_both_aligned # len < 8*NBYTES
+	 and	rem, len, (8*NBYTES-1)	 # rem = len % (8*NBYTES)
+	PREFE(  0, 3*32(src) )
+	PREFE(  1, 3*32(dst) )
+	.align	4
+1:
+	R10KCBARRIER(0(ra))
+EXC(    LOAD    t0, UNIT(0)(src),       .LIl_exc)
+EXC(    LOAD    t1, UNIT(1)(src),       .LIl_exc_copy)
+EXC(    LOAD    t2, UNIT(2)(src),       .LIl_exc_copy)
+EXC(    LOAD    t3, UNIT(3)(src),       .LIl_exc_copy)
+	SUB	len, len, 8*NBYTES
+EXC(    LOAD    t4, UNIT(4)(src),       .LIl_exc_copy)
+EXC(    LOAD    t7, UNIT(5)(src),       .LIl_exc_copy)
+EXC(    STORE   t0, UNIT(0)(dst),       .Ls_exc_p8u)
+EXC(    STORE   t1, UNIT(1)(dst),       .Ls_exc_p7u)
+EXC(    LOAD    t0, UNIT(6)(src),       .LIl_exc_copy)
+EXC(    LOAD    t1, UNIT(7)(src),       .LIl_exc_copy)
+	ADD	src, src, 8*NBYTES
+	ADD	dst, dst, 8*NBYTES
+EXC(    STORE   t2, UNIT(-6)(dst),      .Ls_exc_p6u)
+EXC(    STORE   t3, UNIT(-5)(dst),      .Ls_exc_p5u)
+EXC(    STORE   t4, UNIT(-4)(dst),      .Ls_exc_p4u)
+EXC(    STORE   t7, UNIT(-3)(dst),      .Ls_exc_p3u)
+EXC(    STORE   t0, UNIT(-2)(dst),      .Ls_exc_p2u)
+EXC(    STORE   t1, UNIT(-1)(dst),      .Ls_exc_p1u)
+	PREFE(  0, 8*32(src) )
+	PREFE(  1, 8*32(dst) )
+	bne	len, rem, 1b
+	 nop
+
+	/*
+	 * len == rem == the number of bytes left to copy < 8*NBYTES
+	 */
+.LIcleanup_both_aligned:
+	beqz	len, .Ldone
+	 sltu	t0, len, 4*NBYTES
+	bnez    t0, .LIless_than_4units
+	 and	rem, len, (NBYTES-1)	# rem = len % NBYTES
+	/*
+	 * len >= 4*NBYTES
+	 */
+EXC(    LOAD    t0, UNIT(0)(src),       .LIl_exc)
+EXC(    LOAD    t1, UNIT(1)(src),       .LIl_exc_copy)
+EXC(    LOAD    t2, UNIT(2)(src),       .LIl_exc_copy)
+EXC(    LOAD    t3, UNIT(3)(src),       .LIl_exc_copy)
+	SUB	len, len, 4*NBYTES
+	ADD	src, src, 4*NBYTES
+	R10KCBARRIER(0(ra))
+EXC(    STORE   t0, UNIT(0)(dst),       .Ls_exc_p4u)
+EXC(    STORE   t1, UNIT(1)(dst),       .Ls_exc_p3u)
+EXC(    STORE   t2, UNIT(2)(dst),       .Ls_exc_p2u)
+EXC(    STORE   t3, UNIT(3)(dst),       .Ls_exc_p1u)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, 4*NBYTES
+	beqz	len, .Ldone
+	.set	noreorder
+.LIless_than_4units:
+	/*
+	 * rem = len % NBYTES
+	 */
+	beq     rem, len, .LIcopy_bytes
+	 nop
+1:
+	R10KCBARRIER(0(ra))
+EXC(    LOAD    t0, 0(src),             .LIl_exc)
+	ADD	src, src, NBYTES
+	SUB	len, len, NBYTES
+EXC(    STORE   t0, 0(dst),             .Ls_exc_p1u)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, NBYTES
+	bne	rem, len, 1b
+	.set	noreorder
+
+	/*
+	 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
+	 * A loop would do only a byte at a time with possible branch
+	 * mispredicts.  Can't do an explicit LOAD dst,mask,or,STORE
+	 * because can't assume read-access to dst.  Instead, use
+	 * STREST dst, which doesn't require read access to dst.
+	 *
+	 * This code should perform better than a simple loop on modern,
+	 * wide-issue mips processors because the code has fewer branches and
+	 * more instruction-level parallelism.
+	 */
+	beqz	len, .Ldone
+	 ADD	t1, dst, len	# t1 is just past last byte of dst
+	li	bits, 8*NBYTES
+	SLL	rem, len, 3	# rem = number of bits to keep
+EXC(    LOAD    t0, 0(src),             .LIl_exc)
+	SUB	bits, bits, rem	# bits = number of bits to discard
+	SHIFT_DISCARD t0, t0, bits
+EXC(    STREST  t0, -1(t1),             .Ls_exc)
+	jr	ra
+	 move	len, zero
+.LIdst_unaligned:
+	/*
+	 * dst is unaligned
+	 * t0 = src & ADDRMASK
+	 * t1 = dst & ADDRMASK; T1 > 0
+	 * len >= NBYTES
+	 *
+	 * Copy enough bytes to align dst
+	 * Set match = (src and dst have same alignment)
+	 */
+EXC(    LDFIRST t3, FIRST(0)(src),      .LIl_exc)
+	ADD	t2, zero, NBYTES
+EXC(    LDREST  t3, REST(0)(src),       .LIl_exc_copy)
+	SUB	t2, t2, t1	# t2 = number of bytes copied
+	xor	match, t0, t1
+	R10KCBARRIER(0(ra))
+EXC(    STFIRST t3, FIRST(0)(dst),      .Ls_exc)
+	beq	len, t2, .Ldone
+	 SUB	len, len, t2
+	ADD	dst, dst, t2
+	beqz    match, .LIboth_aligned
+	 ADD	src, src, t2
+
+.LIsrc_unaligned_dst_aligned:
+	SRL	t0, len, LOG_NBYTES+2    # +2 for 4 units/iter
+	PREFE(  0, 3*32(src) )
+	beqz    t0, .LIcleanup_src_unaligned
+	 and	rem, len, (4*NBYTES-1)   # rem = len % 4*NBYTES
+	PREFE(  1, 3*32(dst) )
+1:
+/*
+ * Avoid consecutive LD*'s to the same register since some mips
+ * implementations can't issue them in the same cycle.
+ * It's OK to load FIRST(N+1) before REST(N) because the two addresses
+ * are to the same unit (unless src is aligned, but it's not).
+ */
+	R10KCBARRIER(0(ra))
+EXC(    LDFIRST t0, FIRST(0)(src),      .LIl_exc)
+EXC(    LDFIRST t1, FIRST(1)(src),      .LIl_exc_copy)
+	SUB     len, len, 4*NBYTES
+EXC(    LDREST  t0, REST(0)(src),       .LIl_exc_copy)
+EXC(    LDREST  t1, REST(1)(src),       .LIl_exc_copy)
+EXC(    LDFIRST t2, FIRST(2)(src),      .LIl_exc_copy)
+EXC(    LDFIRST t3, FIRST(3)(src),      .LIl_exc_copy)
+EXC(    LDREST  t2, REST(2)(src),       .LIl_exc_copy)
+EXC(    LDREST  t3, REST(3)(src),       .LIl_exc_copy)
+	PREFE(  0, 9*32(src) )          # 0 is PREF_LOAD  (not streamed)
+	ADD	src, src, 4*NBYTES
+#ifdef CONFIG_CPU_SB1
+	nop				# improves slotting
+#endif
+EXC(    STORE   t0, UNIT(0)(dst),       .Ls_exc_p4u)
+EXC(    STORE   t1, UNIT(1)(dst),       .Ls_exc_p3u)
+EXC(    STORE   t2, UNIT(2)(dst),       .Ls_exc_p2u)
+EXC(    STORE   t3, UNIT(3)(dst),       .Ls_exc_p1u)
+	PREFE(  1, 9*32(dst) )          # 1 is PREF_STORE (not streamed)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, 4*NBYTES
+	bne	len, rem, 1b
+	.set	noreorder
+
+.LIcleanup_src_unaligned:
+	beqz	len, .Ldone
+	 and	rem, len, NBYTES-1  # rem = len % NBYTES
+	beq     rem, len, .LIcopy_bytes
+	 nop
+1:
+	R10KCBARRIER(0(ra))
+EXC(    LDFIRST t0, FIRST(0)(src),      .LIl_exc)
+EXC(    LDREST  t0, REST(0)(src),       .LIl_exc_copy)
+	ADD	src, src, NBYTES
+	SUB	len, len, NBYTES
+EXC(    STORE   t0, 0(dst),             .Ls_exc_p1u)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, NBYTES
+	bne	len, rem, 1b
+	.set	noreorder
+
+.LIcopy_bytes_checklen:
+	beqz	len, .Ldone
+	 nop
+.LIcopy_bytes:
+	/* 0 < len < NBYTES  */
+	R10KCBARRIER(0(ra))
+#define COPY_BYTE(N)                    \
+EXC(    lbe     t0, N(src), .LIl_exc);  \
+	SUB	len, len, 1;		\
+	beqz	len, .Ldone;		\
+EXC(     sbe    t0, N(dst), .Ls_exc_p1)
+
+	COPY_BYTE(0)
+	COPY_BYTE(1)
+#ifdef USE_DOUBLE
+	COPY_BYTE(2)
+	COPY_BYTE(3)
+	COPY_BYTE(4)
+	COPY_BYTE(5)
+#endif
+EXC(    lbe     t0, NBYTES-2(src), .LIl_exc)
+	SUB	len, len, 1
+	jr	ra
+EXC(     sbe    t0, NBYTES-2(dst), .Ls_exc_p1)
+
+.LIl_exc_copy:
+	/*
+	 * Copy bytes from src until faulting load address (or until a
+	 * lb faults)
+	 *
+	 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
+	 * may be more than a byte beyond the last address.
+	 * Hence, the lb below may get an exception.
+	 *
+	 * Assumes src < THREAD_BUADDR($28)
+	 */
+	LOADK   t0, TI_TASK($28)
+	addi    t0, t0, THREAD_BUADDR
+	LOADK   t0, 0(t0)
+1:
+EXC(    lbe     t1, 0(src),     .LIl_exc)
+	ADD	src, src, 1
+EXC(    sbe     t1, 0(dst),     .Ls_exc)
+	.set	reorder				/* DADDI_WAR */
+	ADD	dst, dst, 1
+	SUB     len, len, 1             # need this because of sbe above
+	bne	src, t0, 1b
+	.set	noreorder
+.LIl_exc:
+	LOADK   t0, TI_TASK($28)
+	addi    t0, t0, THREAD_BUADDR
+	LOADK   t0, 0(t0)               # t0 is just past last good address
+	SUB	len, AT, t0		# len number of uncopied bytes
+    BUG_MEMCPY(dst, src, AT, __LINE__)
+	/*
+	 * Here's where we rely on src and dst being incremented in tandem,
+	 *   See (3) above.
+	 * dst += (fault addr - src) to put dst at first byte to clear
+	 */
+	ADD	dst, t0			# compute start address in a1
+	SUB	dst, src
+	/*
+	 * Clear len bytes starting at dst.  Can't call __bzero because it
+	 * might modify len.  An inefficient loop for these rare times...
+	 */
+	.set	reorder				/* DADDI_WAR */
+	SUB	src, len, 1
+	beqz	len, .Ldone
+	.set	noreorder
+1:
+EXC(    sbe     zero, 0(dst),   .Ls_exc)
+	ADD	dst, dst, 1
+#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
+	bnez	src, 1b
+	 SUB	src, src, 1
+#else
+	.set	push
+	.set	noat
+	li	v1, 1
+	bnez	src, 1b
+	 SUB	src, src, v1
+	.set	pop
+#endif
+	jr	ra
+	 nop
+	END(__copy_inuser)
+
+#endif