--- zzzz-none-000/linux-3.10.107/arch/mips/kernel/unaligned.c 2017-06-27 09:49:32.000000000 +0000 +++ scorpion-7490-727/linux-3.10.107/arch/mips/kernel/unaligned.c 2021-02-04 17:41:59.000000000 +0000 @@ -7,6 +7,7 @@ * * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle * Copyright (C) 1999 Silicon Graphics, Inc. + * Copyright (C) 2014 Imagination Technologies Ltd. * * This file contains exception handler for address error exception with the * special capability to execute faulting instructions in software. The @@ -72,6 +73,7 @@ * A store crossing a page boundary might be executed only partially. * Undo the partial store in this case. */ +#include #include #include #include @@ -83,12 +85,17 @@ #include #include #include +#include #include #include #include #include -#include -#include +#if defined(CONFIG_AVM_ENHANCED) +#include +#endif/*--- #if defined(CONFIG_AVM_ENHANCED) ---*/ +#if defined(CONFIG_AVM_SIMPLE_PROFILING) +#include +#endif /*--- #if defined(CONFIG_AVM_SIMPLE_PROFILING) ---*/ #define STR(x) __STR(x) #define __STR(x) #x @@ -107,10 +114,11 @@ extern void show_registers(struct pt_regs *regs); #ifdef __BIG_ENDIAN -#define LoadHW(addr, value, res) \ +#define _LoadHW(addr, value, res, type) \ +do { \ __asm__ __volatile__ (".set\tnoat\n" \ - "1:\tlb\t%0, 0(%2)\n" \ - "2:\tlbu\t$1, 1(%2)\n\t" \ + "1:\t"type##_lb("%0", "0(%2)")"\n" \ + "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ "sll\t%0, 0x8\n\t" \ "or\t%0, $1\n\t" \ "li\t%1, 0\n" \ @@ -125,12 +133,15 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); + : "r" (addr), "i" (-EFAULT)); \ +} while(0) -#define LoadW(addr, value, res) \ +#ifndef CONFIG_CPU_MIPSR6 +#define _LoadW(addr, value, res, type) \ +do { \ __asm__ __volatile__ ( \ - "1:\tlwl\t%0, (%2)\n" \ - "2:\tlwr\t%0, 3(%2)\n\t" \ + "1:\t"type##_lwl("%0", "(%2)")"\n" \ + "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ "li\t%1, 0\n" \ "3:\n\t" \ ".insn\n\t" \ @@ -143,13 +154,52 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); + : "r" (addr), "i" (-EFAULT)); \ +} while(0) + +#else +/* MIPSR6 has no lwl instruction */ +#define _LoadW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n" \ + ".set\tnoat\n\t" \ + "1:"type##_lb("%0", "0(%2)")"\n\t" \ + "2:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "3(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while(0) -#define LoadHWU(addr, value, res) \ +#endif /* CONFIG_CPU_MIPSR6 */ + +#define _LoadHWU(addr, value, res, type) \ +do { \ __asm__ __volatile__ ( \ ".set\tnoat\n" \ - "1:\tlbu\t%0, 0(%2)\n" \ - "2:\tlbu\t$1, 1(%2)\n\t" \ + "1:\t"type##_lbu("%0", "0(%2)")"\n" \ + "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ "sll\t%0, 0x8\n\t" \ "or\t%0, $1\n\t" \ "li\t%1, 0\n" \ @@ -165,12 +215,15 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); + : "r" (addr), "i" (-EFAULT)); \ +} while(0) -#define LoadWU(addr, value, res) \ +#ifndef CONFIG_CPU_MIPSR6 +#define _LoadWU(addr, value, res, type) \ +do { \ __asm__ __volatile__ ( \ - "1:\tlwl\t%0, (%2)\n" \ - "2:\tlwr\t%0, 3(%2)\n\t" \ + "1:\t"type##_lwl("%0", "(%2)")"\n" \ + "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ "dsll\t%0, %0, 32\n\t" \ "dsrl\t%0, %0, 32\n\t" \ "li\t%1, 0\n" \ @@ -185,9 +238,11 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); + : "r" (addr), "i" (-EFAULT)); \ +} while(0) -#define LoadDW(addr, value, res) \ +#define _LoadDW(addr, value, res) \ +do { \ __asm__ __volatile__ ( \ "1:\tldl\t%0, (%2)\n" \ "2:\tldr\t%0, 7(%2)\n\t" \ @@ -203,14 +258,103 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); + : "r" (addr), "i" (-EFAULT)); \ +} while(0) + +#else +/* MIPSR6 has not lwl and ldl instructions */ +#define _LoadWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_lbu("%0", "0(%2)")"\n\t" \ + "2:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "3(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while(0) + +#define _LoadDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:lb\t%0, 0(%2)\n\t" \ + "2:lbu\t $1, 1(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:lbu\t$1, 2(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:lbu\t$1, 3(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "5:lbu\t$1, 4(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "6:lbu\t$1, 5(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "7:lbu\t$1, 6(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "8:lbu\t$1, 7(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n\t" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while(0) -#define StoreHW(addr, value, res) \ +#endif /* CONFIG_CPU_MIPSR6 */ + + +#define _StoreHW(addr, value, res, type) \ +do { \ __asm__ __volatile__ ( \ ".set\tnoat\n" \ - "1:\tsb\t%1, 1(%2)\n\t" \ + "1:\t"type##_sb("%1", "1(%2)")"\n" \ "srl\t$1, %1, 0x8\n" \ - "2:\tsb\t$1, 0(%2)\n\t" \ + "2:\t"type##_sb("$1", "0(%2)")"\n" \ ".set\tat\n\t" \ "li\t%0, 0\n" \ "3:\n\t" \ @@ -224,12 +368,15 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); + : "r" (value), "r" (addr), "i" (-EFAULT));\ +} while(0) -#define StoreW(addr, value, res) \ +#ifndef CONFIG_CPU_MIPSR6 +#define _StoreW(addr, value, res, type) \ +do { \ __asm__ __volatile__ ( \ - "1:\tswl\t%1,(%2)\n" \ - "2:\tswr\t%1, 3(%2)\n\t" \ + "1:\t"type##_swl("%1", "(%2)")"\n" \ + "2:\t"type##_swr("%1", "3(%2)")"\n\t"\ "li\t%0, 0\n" \ "3:\n\t" \ ".insn\n\t" \ @@ -242,9 +389,11 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while(0) -#define StoreDW(addr, value, res) \ +#define _StoreDW(addr, value, res) \ +do { \ __asm__ __volatile__ ( \ "1:\tsdl\t%1,(%2)\n" \ "2:\tsdr\t%1, 7(%2)\n\t" \ @@ -260,14 +409,95 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); -#endif + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while(0) + +#else +/* MIPSR6 has no swl and sdl instructions */ +#define _StoreW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_sb("%1", "3(%2)")"\n\t" \ + "srl\t$1, %1, 0x8\n\t" \ + "2:"type##_sb("$1", "2(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "3:"type##_sb("$1", "1(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "4:"type##_sb("$1", "0(%2)")"\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while(0) + +#define _StoreDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:sb\t%1, 7(%2)\n\t" \ + "dsrl\t$1, %1, 0x8\n\t" \ + "2:sb\t$1, 6(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "3:sb\t$1, 5(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "4:sb\t$1, 4(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "5:sb\t$1, 3(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "6:sb\t$1, 2(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "7:sb\t$1, 1(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "8:sb\t$1, 0(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while(0) -#ifdef __LITTLE_ENDIAN -#define LoadHW(addr, value, res) \ +#endif /* CONFIG_CPU_MIPSR6 */ + +#else /* __BIG_ENDIAN */ + +#define _LoadHW(addr, value, res, type) \ +do { \ __asm__ __volatile__ (".set\tnoat\n" \ - "1:\tlb\t%0, 1(%2)\n" \ - "2:\tlbu\t$1, 0(%2)\n\t" \ + "1:\t"type##_lb("%0", "1(%2)")"\n" \ + "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ "sll\t%0, 0x8\n\t" \ "or\t%0, $1\n\t" \ "li\t%1, 0\n" \ @@ -282,12 +512,15 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); + : "r" (addr), "i" (-EFAULT)); \ +} while(0) -#define LoadW(addr, value, res) \ +#ifndef CONFIG_CPU_MIPSR6 +#define _LoadW(addr, value, res, type) \ +do { \ __asm__ __volatile__ ( \ - "1:\tlwl\t%0, 3(%2)\n" \ - "2:\tlwr\t%0, (%2)\n\t" \ + "1:\t"type##_lwl("%0", "3(%2)")"\n" \ + "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ "li\t%1, 0\n" \ "3:\n\t" \ ".insn\n\t" \ @@ -300,13 +533,53 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); + : "r" (addr), "i" (-EFAULT)); \ +} while(0) + +#else +/* MIPSR6 has no lwl instruction */ +#define _LoadW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n" \ + ".set\tnoat\n\t" \ + "1:"type##_lb("%0", "3(%2)")"\n\t" \ + "2:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "0(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while(0) -#define LoadHWU(addr, value, res) \ +#endif /* CONFIG_CPU_MIPSR6 */ + + +#define _LoadHWU(addr, value, res, type) \ +do { \ __asm__ __volatile__ ( \ ".set\tnoat\n" \ - "1:\tlbu\t%0, 1(%2)\n" \ - "2:\tlbu\t$1, 0(%2)\n\t" \ + "1:\t"type##_lbu("%0", "1(%2)")"\n" \ + "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ "sll\t%0, 0x8\n\t" \ "or\t%0, $1\n\t" \ "li\t%1, 0\n" \ @@ -322,12 +595,15 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); + : "r" (addr), "i" (-EFAULT)); \ +} while(0) -#define LoadWU(addr, value, res) \ +#ifndef CONFIG_CPU_MIPSR6 +#define _LoadWU(addr, value, res, type) \ +do { \ __asm__ __volatile__ ( \ - "1:\tlwl\t%0, 3(%2)\n" \ - "2:\tlwr\t%0, (%2)\n\t" \ + "1:\t"type##_lwl("%0", "3(%2)")"\n" \ + "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ "dsll\t%0, %0, 32\n\t" \ "dsrl\t%0, %0, 32\n\t" \ "li\t%1, 0\n" \ @@ -342,9 +618,11 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); + : "r" (addr), "i" (-EFAULT)); \ +} while(0) -#define LoadDW(addr, value, res) \ +#define _LoadDW(addr, value, res) \ +do { \ __asm__ __volatile__ ( \ "1:\tldl\t%0, 7(%2)\n" \ "2:\tldr\t%0, (%2)\n\t" \ @@ -360,14 +638,101 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); + : "r" (addr), "i" (-EFAULT)); \ +} while(0) + +#else +/* MIPSR6 has not lwl and ldl instructions */ +#define _LoadWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_lbu("%0", "3(%2)")"\n\t" \ + "2:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "0(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while(0) + +#define _LoadDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:lb\t%0, 7(%2)\n\t" \ + "2:lbu\t$1, 6(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:lbu\t$1, 5(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:lbu\t$1, 4(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "5:lbu\t$1, 3(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "6:lbu\t$1, 2(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "7:lbu\t$1, 1(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "8:lbu\t$1, 0(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n\t" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while(0) +#endif /* CONFIG_CPU_MIPSR6 */ -#define StoreHW(addr, value, res) \ +#define _StoreHW(addr, value, res, type) \ +do { \ __asm__ __volatile__ ( \ ".set\tnoat\n" \ - "1:\tsb\t%1, 0(%2)\n\t" \ + "1:\t"type##_sb("%1", "0(%2)")"\n" \ "srl\t$1,%1, 0x8\n" \ - "2:\tsb\t$1, 1(%2)\n\t" \ + "2:\t"type##_sb("$1", "1(%2)")"\n" \ ".set\tat\n\t" \ "li\t%0, 0\n" \ "3:\n\t" \ @@ -381,12 +746,15 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); + : "r" (value), "r" (addr), "i" (-EFAULT));\ +} while(0) -#define StoreW(addr, value, res) \ +#ifndef CONFIG_CPU_MIPSR6 +#define _StoreW(addr, value, res, type) \ +do { \ __asm__ __volatile__ ( \ - "1:\tswl\t%1, 3(%2)\n" \ - "2:\tswr\t%1, (%2)\n\t" \ + "1:\t"type##_swl("%1", "3(%2)")"\n" \ + "2:\t"type##_swr("%1", "(%2)")"\n\t"\ "li\t%0, 0\n" \ "3:\n\t" \ ".insn\n\t" \ @@ -399,9 +767,11 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while(0) -#define StoreDW(addr, value, res) \ +#define _StoreDW(addr, value, res) \ +do { \ __asm__ __volatile__ ( \ "1:\tsdl\t%1, 7(%2)\n" \ "2:\tsdr\t%1, (%2)\n\t" \ @@ -417,19 +787,123 @@ STR(PTR)"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while(0) + +#else +/* MIPSR6 has no swl and sdl instructions */ +#define _StoreW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_sb("%1", "0(%2)")"\n\t" \ + "srl\t$1, %1, 0x8\n\t" \ + "2:"type##_sb("$1", "1(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "3:"type##_sb("$1", "2(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "4:"type##_sb("$1", "3(%2)")"\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while(0) + +#define _StoreDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:sb\t%1, 0(%2)\n\t" \ + "dsrl\t$1, %1, 0x8\n\t" \ + "2:sb\t$1, 1(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "3:sb\t$1, 2(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "4:sb\t$1, 3(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "5:sb\t$1, 4(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "6:sb\t$1, 5(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "7:sb\t$1, 6(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "8:sb\t$1, 7(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while(0) + +#endif /* CONFIG_CPU_MIPSR6 */ #endif +#define LoadHWU(addr, value, res) _LoadHWU(addr, value, res, kernel) +#define LoadHWUE(addr, value, res) _LoadHWU(addr, value, res, user) +#define LoadWU(addr, value, res) _LoadWU(addr, value, res, kernel) +#define LoadWUE(addr, value, res) _LoadWU(addr, value, res, user) +#define LoadHW(addr, value, res) _LoadHW(addr, value, res, kernel) +#define LoadHWE(addr, value, res) _LoadHW(addr, value, res, user) +#define LoadW(addr, value, res) _LoadW(addr, value, res, kernel) +#define LoadWE(addr, value, res) _LoadW(addr, value, res, user) +#define LoadDW(addr, value, res) _LoadDW(addr, value, res) + +#define StoreHW(addr, value, res) _StoreHW(addr, value, res, kernel) +#define StoreHWE(addr, value, res) _StoreHW(addr, value, res, user) +#define StoreW(addr, value, res) _StoreW(addr, value, res, kernel) +#define StoreWE(addr, value, res) _StoreW(addr, value, res, user) +#define StoreDW(addr, value, res) _StoreDW(addr, value, res) + static void emulate_load_store_insn(struct pt_regs *regs, void __user *addr, unsigned int __user *pc) { union mips_instruction insn; unsigned long value; - unsigned int res; + unsigned int res, preempted; unsigned long origpc; unsigned long orig31; void __user *fault_addr = NULL; - +#ifdef CONFIG_EVA + mm_segment_t seg; +#endif +#if defined(CONFIG_AVM_ENHANCED) + siginfo_t info; +#endif + union fpureg *fpr; + enum msa_2b_fmt df; + unsigned int wd; origpc = (unsigned long)pc; orig31 = regs->regs[31]; @@ -474,11 +948,101 @@ * The remaining opcodes are the ones that are really of * interest. */ +#ifdef CONFIG_EVA + case spec3_op: + /* + * we can land here only from kernel accessing user memory, + * so we need to "switch" the address limit to user space, so + * address check can work properly. + */ + seg = get_fs(); + set_fs(USER_DS); + switch (insn.spec3_format.func) { + case lhe_op: + if (!access_ok(VERIFY_READ, addr, 2)) { + set_fs(seg); + goto sigbus; + } + LoadHWE(addr, value, res); + if (res) { + set_fs(seg); + goto fault; + } + compute_return_epc(regs); + regs->regs[insn.spec3_format.rt] = value; + break; + case lwe_op: + if (!access_ok(VERIFY_READ, addr, 4)) { + set_fs(seg); + goto sigbus; + } + LoadWE(addr, value, res); + if (res) { + set_fs(seg); + goto fault; + } + compute_return_epc(regs); + regs->regs[insn.spec3_format.rt] = value; + break; + case lhue_op: + if (!access_ok(VERIFY_READ, addr, 2)) { + set_fs(seg); + goto sigbus; + } + LoadHWUE(addr, value, res); + if (res) { + set_fs(seg); + goto fault; + } + compute_return_epc(regs); + regs->regs[insn.spec3_format.rt] = value; + break; + case she_op: + if (!access_ok(VERIFY_WRITE, addr, 2)) { + set_fs(seg); + goto sigbus; + } + compute_return_epc(regs); + value = regs->regs[insn.spec3_format.rt]; + StoreHWE(addr, value, res); + if (res) { + set_fs(seg); + goto fault; + } + break; + case swe_op: + if (!access_ok(VERIFY_WRITE, addr, 4)) { + set_fs(seg); + goto sigbus; + } + compute_return_epc(regs); + value = regs->regs[insn.spec3_format.rt]; + StoreWE(addr, value, res); + if (res) { + set_fs(seg); + goto fault; + } + break; + default: + set_fs(seg); + goto sigill; + } + set_fs(seg); + break; +#endif case lh_op: if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; - LoadHW(addr, value, res); + if (config_enabled(CONFIG_EVA)) { + if (segment_eq(get_fs(), get_ds())) + LoadHW(addr, value, res); + else + LoadHWE(addr, value, res); + } else { + LoadHW(addr, value, res); + } + if (res) goto fault; compute_return_epc(regs); @@ -489,7 +1053,15 @@ if (!access_ok(VERIFY_READ, addr, 4)) goto sigbus; - LoadW(addr, value, res); + if (config_enabled(CONFIG_EVA)) { + if (segment_eq(get_fs(), get_ds())) + LoadW(addr, value, res); + else + LoadWE(addr, value, res); + } else { + LoadW(addr, value, res); + } + if (res) goto fault; compute_return_epc(regs); @@ -500,7 +1072,15 @@ if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; - LoadHWU(addr, value, res); + if (config_enabled(CONFIG_EVA)) { + if (segment_eq(get_fs(), get_ds())) + LoadHWU(addr, value, res); + else + LoadHWUE(addr, value, res); + } else { + LoadHWU(addr, value, res); + } + if (res) goto fault; compute_return_epc(regs); @@ -559,7 +1139,16 @@ compute_return_epc(regs); value = regs->regs[insn.i_format.rt]; - StoreHW(addr, value, res); + + if (config_enabled(CONFIG_EVA)) { + if (segment_eq(get_fs(), get_ds())) + StoreHW(addr, value, res); + else + StoreHWE(addr, value, res); + } else { + StoreHW(addr, value, res); + } + if (res) goto fault; break; @@ -570,7 +1159,16 @@ compute_return_epc(regs); value = regs->regs[insn.i_format.rt]; - StoreW(addr, value, res); + + if (config_enabled(CONFIG_EVA)) { + if (segment_eq(get_fs(), get_ds())) + StoreW(addr, value, res); + else + StoreWE(addr, value, res); + } else { + StoreW(addr, value, res); + } + if (res) goto fault; break; @@ -611,16 +1209,97 @@ own_fpu(1); /* Restore FPU state. */ /* Signal if something went wrong. */ - process_fpemu_return(res, fault_addr); + process_fpemu_return(res, fault_addr, 0); if (res == 0) break; return; + case msa_op: + if (!cpu_has_msa) + goto sigill; + + /* + * If we've reached this point then userland should have taken + * the MSA disabled exception & initialised vector context at + * some point in the past. + */ + BUG_ON(!thread_msa_context_live()); + + df = insn.msa_mi10_format.df; + wd = insn.msa_mi10_format.wd; + fpr = ¤t->thread.fpu.fpr[wd]; + + switch (insn.msa_mi10_format.func) { + case msa_ld_op: + if (!access_ok(VERIFY_READ, addr, sizeof(*fpr))) + goto sigbus; + + do { + /* + * If we have live MSA context keep track of + * whether we get preempted in order to avoid + * the register context we load being clobbered + * by the live context as it's saved during + * preemption. If we don't have live context + * then it can't be saved to clobber the value + * we load. + */ + preempted = test_thread_flag(TIF_USEDMSA); + + res = __copy_from_user_inatomic(fpr, addr, + sizeof(*fpr)); + if (res) + goto fault; + + /* + * Update the hardware register if it is in use + * by the task in this quantum, in order to + * avoid having to save & restore the whole + * vector context. + */ + preempt_disable(); + if (test_thread_flag(TIF_USEDMSA)) { + write_msa_wr(wd, fpr, df); + preempted = 0; + } + preempt_enable(); + } while (preempted); + break; + + case msa_st_op: + if (!access_ok(VERIFY_WRITE, addr, sizeof(*fpr))) + goto sigbus; + + /* + * Update from the hardware register if it is in use by + * the task in this quantum, in order to avoid having to + * save & restore the whole vector context. + */ + preempt_disable(); + if (test_thread_flag(TIF_USEDMSA)) + read_msa_wr(wd, fpr, df); + preempt_enable(); + + res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr)); + if (res) + goto fault; + break; + + default: + goto sigbus; + } + + compute_return_epc(regs); + break; + +#ifndef CONFIG_CPU_MIPSR6 /* * COP2 is available to implementor for application specific use. * It's up to applications to register a notifier chain and do * whatever they have to do, including possible sending of signals. + * + * This instruction has been reallocated in Release 6 */ case lwc2_op: cu2_notifier_call_chain(CU2_LWC2_OP, regs); @@ -637,7 +1316,7 @@ case sdc2_op: cu2_notifier_call_chain(CU2_SDC2_OP, regs); break; - +#endif default: /* * Pheeee... We encountered an yet unknown instruction or @@ -661,20 +1340,42 @@ return; die_if_kernel("Unhandled kernel unaligned access", regs); +#if defined(CONFIG_AVM_ENHANCED) + info.si_code = SEGV_ACCERR; + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_addr = (void __user *) regs->cp0_badvaddr; + force_sig_info(SIGSEGV, &info, current); +#else force_sig(SIGSEGV, current); - +#endif return; sigbus: die_if_kernel("Unhandled kernel unaligned access", regs); +#if defined(CONFIG_AVM_ENHANCED) + info.si_code = BUS_ADRERR; + info.si_signo = SIGBUS; + info.si_errno = 0; + info.si_addr = (void __user *) regs->cp0_badvaddr; + force_sig_info(SIGBUS, &info, current); +#else force_sig(SIGBUS, current); - +#endif return; sigill: die_if_kernel ("Unhandled kernel unaligned access or invalid instruction", regs); +#if defined(CONFIG_AVM_ENHANCED) + info.si_code = ILL_ILLOPC; + info.si_signo = SIGILL; + info.si_errno = 0; + info.si_addr = (void __user *) regs->cp0_badvaddr; + force_sig_info(SIGILL, &info, current); +#else force_sig(SIGILL, current); +#endif } /* Recode table from 16-bit register notation to 32-bit GPR. */ @@ -683,7 +1384,8 @@ /* Recode table from 16-bit STORE register notation to 32-bit GPR. */ const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 }; -void emulate_load_store_microMIPS(struct pt_regs *regs, void __user * addr) +static void emulate_load_store_microMIPS(struct pt_regs *regs, + void __user *addr) { unsigned long value; unsigned int res; @@ -697,6 +1399,9 @@ union mips_instruction insn; struct mm_decoded_insn mminsn; void __user *fault_addr = NULL; +#if defined(CONFIG_AVM_ENHANCED) + siginfo_t info; +#endif origpc = regs->cp0_epc; orig31 = regs->regs[31]; @@ -1042,7 +1747,7 @@ own_fpu(1); /* restore FPU state */ /* If something went wrong, signal */ - process_fpemu_return(res, fault_addr); + process_fpemu_return(res, fault_addr, 0); if (res == 0) goto success; @@ -1287,20 +1992,42 @@ return; die_if_kernel("Unhandled kernel unaligned access", regs); +#if defined(CONFIG_AVM_ENHANCED) + info.si_code = SEGV_ACCERR; + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_addr = (void __user *) regs->cp0_badvaddr; + force_sig_info(SIGSEGV, &info, current); +#else force_sig(SIGSEGV, current); - +#endif return; sigbus: die_if_kernel("Unhandled kernel unaligned access", regs); +#if defined(CONFIG_AVM_ENHANCED) + info.si_code = BUS_ADRERR; + info.si_signo = SIGBUS; + info.si_errno = 0; + info.si_addr = (void __user *) regs->cp0_badvaddr; + force_sig_info(SIGBUS, &info, current); +#else force_sig(SIGBUS, current); - +#endif return; sigill: die_if_kernel ("Unhandled kernel unaligned access or invalid instruction", regs); +#if defined(CONFIG_AVM_ENHANCED) + info.si_code = ILL_ILLOPC; + info.si_signo = SIGILL; + info.si_errno = 0; + info.si_addr = (void __user *) regs->cp0_badvaddr; + force_sig_info(SIGILL, &info, current); +#else force_sig(SIGILL, current); +#endif } static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) @@ -1312,6 +2039,9 @@ u16 __user *pc16; unsigned long origpc; union mips16e_instruction mips16inst, oldinst; +#if defined(CONFIG_AVM_ENHANCED) + siginfo_t info; +#endif origpc = regs->cp0_epc; orig31 = regs->regs[31]; @@ -1532,26 +2262,58 @@ return; die_if_kernel("Unhandled kernel unaligned access", regs); +#if defined(CONFIG_AVM_ENHANCED) + info.si_code = SEGV_ACCERR; + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_addr = (void __user *) regs->cp0_badvaddr; + force_sig_info(SIGSEGV, &info, current); +#else force_sig(SIGSEGV, current); - +#endif return; sigbus: die_if_kernel("Unhandled kernel unaligned access", regs); +#if defined(CONFIG_AVM_ENHANCED) + info.si_code = BUS_ADRERR; + info.si_signo = SIGBUS; + info.si_errno = 0; + info.si_addr = (void __user *) regs->cp0_badvaddr; + force_sig_info(SIGBUS, &info, current); +#else force_sig(SIGBUS, current); - +#endif return; sigill: die_if_kernel ("Unhandled kernel unaligned access or invalid instruction", regs); +#if defined(CONFIG_AVM_ENHANCED) + info.si_code = ILL_ILLOPC; + info.si_signo = SIGILL; + info.si_errno = 0; + info.si_addr = (void __user *) regs->cp0_badvaddr; + force_sig_info(SIGILL, &info, current); +#else force_sig(SIGILL, current); +#endif } + asmlinkage void do_ade(struct pt_regs *regs) { + enum ctx_state prev_state; unsigned int __user *pc; mm_segment_t seg; +#if defined(CONFIG_AVM_ENHANCED) + siginfo_t info; +#endif +#if defined(CONFIG_AVM_SIMPLE_PROFILING) + avm_simple_profiling_log(avm_profile_data_type_exception_begin, regs->cp0_epc, + (regs->cp0_cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE); +#endif /*--- #if defined(CONFIG_AVM_SIMPLE_PROFILING) ---*/ + prev_state = exception_enter(); perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, regs->cp0_badvaddr); /* @@ -1559,7 +2321,11 @@ */ if (regs->cp0_badvaddr == regs->cp0_epc) goto sigbus; - +#if defined(CONFIG_AVM_ENHANCED) + if(show_unaligned_position(regs)) { + goto sigbus; + } +#endif/*--- #if defined(CONFIG_AVM_ENHANCED) ---*/ if (user_mode(regs) && !test_thread_flag(TIF_FIXADE)) goto sigbus; if (unaligned_action == UNALIGNED_ACTION_SIGNAL) @@ -1601,6 +2367,10 @@ emulate_load_store_MIPS16e(regs, (void __user *)regs->cp0_badvaddr); set_fs(seg); +#if defined(CONFIG_AVM_SIMPLE_PROFILING) + avm_simple_profiling_log(avm_profile_data_type_exception_end, regs->cp0_epc, + (regs->cp0_cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE); +#endif /*--- #if defined(CONFIG_AVM_SIMPLE_PROFILING) ---*/ return; } @@ -1618,19 +2388,34 @@ emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc); set_fs(seg); +#if defined(CONFIG_AVM_SIMPLE_PROFILING) + avm_simple_profiling_log(avm_profile_data_type_exception_end, regs->cp0_epc, + (regs->cp0_cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE); +#endif /*--- #if defined(CONFIG_AVM_SIMPLE_PROFILING) ---*/ return; sigbus: die_if_kernel("Kernel unaligned instruction access", regs); +#if defined(CONFIG_AVM_SIMPLE_PROFILING) + avm_simple_profiling_log(avm_profile_data_type_exception_end, regs->cp0_epc, + (regs->cp0_cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE); +#endif /*--- #if defined(CONFIG_AVM_SIMPLE_PROFILING) ---*/ +#if defined(CONFIG_AVM_ENHANCED) + info.si_code = BUS_ADRERR; + info.si_signo = SIGBUS; + info.si_errno = 0; + info.si_addr = (void __user *) regs->cp0_badvaddr; + force_sig_info(SIGBUS, &info, current); +#else force_sig(SIGBUS, current); - +#endif /* * XXX On return from the signal handler we should advance the epc */ + exception_exit(prev_state); } #ifdef CONFIG_DEBUG_FS -extern struct dentry *mips_debugfs_dir; static int __init debugfs_unaligned(void) { struct dentry *d; @@ -1647,5 +2432,5 @@ return -ENOMEM; return 0; } -__initcall(debugfs_unaligned); +arch_initcall(debugfs_unaligned); #endif