--- zzzz-none-000/linux-3.10.107/arch/mips/include/asm/r4kcache.h 2017-06-27 09:49:32.000000000 +0000 +++ scorpion-7490-727/linux-3.10.107/arch/mips/include/asm/r4kcache.h 2021-02-04 17:41:59.000000000 +0000 @@ -12,10 +12,18 @@ #ifndef _ASM_R4KCACHE_H #define _ASM_R4KCACHE_H +#include + #include #include +#include #include +#include #include +#include /* for segment_eq() */ + +extern void (*r4k_blast_dcache)(void); +extern void (*r4k_blast_icache)(void); /* * This macro return a properly sign-extended address suitable as base address @@ -34,88 +42,28 @@ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ - " .set mips3\n\t \n" \ + " .set "MIPS_ISA_ARCH_LEVEL" \n" \ " cache %0, %1 \n" \ " .set pop \n" \ : \ : "i" (op), "R" (*(unsigned char *)(addr))) #ifdef CONFIG_MIPS_MT -/* - * Temporary hacks for SMTC debug. Optionally force single-threaded - * execution during I-cache flushes. - */ - -#define PROTECT_CACHE_FLUSHES 1 - -#ifdef PROTECT_CACHE_FLUSHES - -extern int mt_protiflush; -extern int mt_protdflush; -extern void mt_cflush_lockdown(void); -extern void mt_cflush_release(void); - -#define BEGIN_MT_IPROT \ - unsigned long flags = 0; \ - unsigned long mtflags = 0; \ - if(mt_protiflush) { \ - local_irq_save(flags); \ - ehb(); \ - mtflags = dvpe(); \ - mt_cflush_lockdown(); \ - } - -#define END_MT_IPROT \ - if(mt_protiflush) { \ - mt_cflush_release(); \ - evpe(mtflags); \ - local_irq_restore(flags); \ - } - -#define BEGIN_MT_DPROT \ - unsigned long flags = 0; \ - unsigned long mtflags = 0; \ - if(mt_protdflush) { \ - local_irq_save(flags); \ - ehb(); \ - mtflags = dvpe(); \ - mt_cflush_lockdown(); \ - } - -#define END_MT_DPROT \ - if(mt_protdflush) { \ - mt_cflush_release(); \ - evpe(mtflags); \ - local_irq_restore(flags); \ - } - -#else - -#define BEGIN_MT_IPROT -#define BEGIN_MT_DPROT -#define END_MT_IPROT -#define END_MT_DPROT - -#endif /* PROTECT_CACHE_FLUSHES */ #define __iflush_prologue \ unsigned long redundance; \ extern int mt_n_iflushes; \ - BEGIN_MT_IPROT \ for (redundance = 0; redundance < mt_n_iflushes; redundance++) { #define __iflush_epilogue \ - END_MT_IPROT \ } #define __dflush_prologue \ unsigned long redundance; \ extern int mt_n_dflushes; \ - BEGIN_MT_DPROT \ for (redundance = 0; redundance < mt_n_dflushes; redundance++) { #define __dflush_epilogue \ - END_MT_DPROT \ } #define __inv_dflush_prologue __dflush_prologue @@ -162,7 +110,15 @@ static inline void flush_icache_line(unsigned long addr) { __iflush_prologue - cache_op(Hit_Invalidate_I, addr); + switch (boot_cpu_type()) { + case CPU_LOONGSON2: + cache_op(Hit_Invalidate_I_Loongson2, addr); + break; + + default: + cache_op(Hit_Invalidate_I, addr); + break; + } __iflush_epilogue } @@ -194,7 +150,7 @@ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ - " .set mips3 \n" \ + " .set "MIPS_ISA_ARCH_LEVEL" \n" \ "1: cache %0, (%1) \n" \ "2: .set pop \n" \ " .section __ex_table,\"a\" \n" \ @@ -203,12 +159,38 @@ : \ : "i" (op), "r" (addr)) +#define protected_cachee_op(op,addr) \ + __asm__ __volatile__( \ + " .set push \n" \ + " .set noreorder \n" \ + " .set mips0 \n" \ + " .set eva \n" \ + "1: cachee %0, (%1) \n" \ + "2: .set pop \n" \ + " .section __ex_table,\"a\" \n" \ + " "STR(PTR)" 1b, 2b \n" \ + " .previous" \ + : \ + : "i" (op), "r" (addr)) + /* * The next two are for badland addresses like signal trampolines. */ static inline void protected_flush_icache_line(unsigned long addr) { - protected_cache_op(Hit_Invalidate_I, addr); + switch (boot_cpu_type()) { + case CPU_LOONGSON2: + protected_cache_op(Hit_Invalidate_I_Loongson2, addr); + break; + + default: +#ifdef CONFIG_EVA + protected_cachee_op(Hit_Invalidate_I, addr); +#else + protected_cache_op(Hit_Invalidate_I, addr); +#endif + break; + } } /* @@ -219,7 +201,11 @@ */ static inline void protected_writeback_dcache_line(unsigned long addr) { +#ifdef CONFIG_EVA + protected_cachee_op(Hit_Writeback_Inv_D, addr); +#else protected_cache_op(Hit_Writeback_Inv_D, addr); +#endif } static inline void protected_writeback_scache_line(unsigned long addr) @@ -235,11 +221,12 @@ cache_op(Page_Invalidate_T, addr); } +#ifndef CONFIG_CPU_MIPSR6 #define cache16_unroll32(base,op) \ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ - " .set mips3 \n" \ + " .set "MIPS_ISA_ARCH_LEVEL" \n" \ " cache %1, 0x000(%0); cache %1, 0x010(%0) \n" \ " cache %1, 0x020(%0); cache %1, 0x030(%0) \n" \ " cache %1, 0x040(%0); cache %1, 0x050(%0) \n" \ @@ -265,7 +252,7 @@ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ - " .set mips3 \n" \ + " .set "MIPS_ISA_ARCH_LEVEL" \n" \ " cache %1, 0x000(%0); cache %1, 0x020(%0) \n" \ " cache %1, 0x040(%0); cache %1, 0x060(%0) \n" \ " cache %1, 0x080(%0); cache %1, 0x0a0(%0) \n" \ @@ -291,7 +278,7 @@ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ - " .set mips3 \n" \ + " .set "MIPS_ISA_ARCH_LEVEL" \n" \ " cache %1, 0x000(%0); cache %1, 0x040(%0) \n" \ " cache %1, 0x080(%0); cache %1, 0x0c0(%0) \n" \ " cache %1, 0x100(%0); cache %1, 0x140(%0) \n" \ @@ -317,7 +304,7 @@ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ - " .set mips3 \n" \ + " .set "MIPS_ISA_ARCH_LEVEL" \n" \ " cache %1, 0x000(%0); cache %1, 0x080(%0) \n" \ " cache %1, 0x100(%0); cache %1, 0x180(%0) \n" \ " cache %1, 0x200(%0); cache %1, 0x280(%0) \n" \ @@ -339,9 +326,237 @@ : "r" (base), \ "i" (op)); +#else +/* + * MIPS R6 changed the cache opcode and moved to a 8-bit offset field. + * This means we now need to increment the base register before we flush + * more cache lines + */ +#define cache16_unroll32(base,op) \ + __asm__ __volatile__( \ + " .set push\n" \ + " .set noreorder\n" \ + " .set mips64r6\n" \ + " .set noat\n" \ + " cache %1, 0x000(%0); cache %1, 0x010(%0)\n" \ + " cache %1, 0x020(%0); cache %1, 0x030(%0)\n" \ + " cache %1, 0x040(%0); cache %1, 0x050(%0)\n" \ + " cache %1, 0x060(%0); cache %1, 0x070(%0)\n" \ + " cache %1, 0x080(%0); cache %1, 0x090(%0)\n" \ + " cache %1, 0x0a0(%0); cache %1, 0x0b0(%0)\n" \ + " cache %1, 0x0c0(%0); cache %1, 0x0d0(%0)\n" \ + " cache %1, 0x0e0(%0); cache %1, 0x0f0(%0)\n" \ + " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x010($1)\n" \ + " cache %1, 0x020($1); cache %1, 0x030($1)\n" \ + " cache %1, 0x040($1); cache %1, 0x050($1)\n" \ + " cache %1, 0x060($1); cache %1, 0x070($1)\n" \ + " cache %1, 0x080($1); cache %1, 0x090($1)\n" \ + " cache %1, 0x0a0($1); cache %1, 0x0b0($1)\n" \ + " cache %1, 0x0c0($1); cache %1, 0x0d0($1)\n" \ + " cache %1, 0x0e0($1); cache %1, 0x0f0($1)\n" \ + " .set pop\n" \ + : \ + : "r" (base), \ + "i" (op)); + +#define cache32_unroll32(base,op) \ + __asm__ __volatile__( \ + " .set push\n" \ + " .set noreorder\n" \ + " .set mips64r6\n" \ + " .set noat\n" \ + " cache %1, 0x000(%0); cache %1, 0x020(%0)\n" \ + " cache %1, 0x040(%0); cache %1, 0x060(%0)\n" \ + " cache %1, 0x080(%0); cache %1, 0x0a0(%0)\n" \ + " cache %1, 0x0c0(%0); cache %1, 0x0e0(%0)\n" \ + " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x020($1)\n" \ + " cache %1, 0x040($1); cache %1, 0x060($1)\n" \ + " cache %1, 0x080($1); cache %1, 0x0a0($1)\n" \ + " cache %1, 0x0c0($1); cache %1, 0x0e0($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x020($1)\n" \ + " cache %1, 0x040($1); cache %1, 0x060($1)\n" \ + " cache %1, 0x080($1); cache %1, 0x0a0($1)\n" \ + " cache %1, 0x0c0($1); cache %1, 0x0e0($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100\n" \ + " cache %1, 0x000($1); cache %1, 0x020($1)\n" \ + " cache %1, 0x040($1); cache %1, 0x060($1)\n" \ + " cache %1, 0x080($1); cache %1, 0x0a0($1)\n" \ + " cache %1, 0x0c0($1); cache %1, 0x0e0($1)\n" \ + " .set pop\n" \ + : \ + : "r" (base), \ + "i" (op)); + +#define cache64_unroll32(base,op) \ + __asm__ __volatile__( \ + " .set push\n" \ + " .set noreorder\n" \ + " .set mips64r6\n" \ + " .set noat\n" \ + " cache %1, 0x000(%0); cache %1, 0x040(%0)\n" \ + " cache %1, 0x080(%0); cache %1, 0x0c0(%0)\n" \ + " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ + " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ + " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ + " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ + " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ + " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ + " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ + " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ + " .set pop\n" \ + : \ + : "r" (base), \ + "i" (op)); + +#define cache128_unroll32(base,op) \ + __asm__ __volatile__( \ + " .set push\n" \ + " .set noreorder\n" \ + " .set mips64r6\n" \ + " .set noat\n" \ + " cache %1, 0x000(%0); cache %1, 0x080(%0)\n" \ + " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ + " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ + " .set pop\n" \ + : \ + : "r" (base), \ + "i" (op)); +#endif /* CONFIG_CPU_MIPSR6 */ + +/* + * Perform the cache operation specified by op using a user mode virtual + * address while in kernel mode. + */ +#define cache16_unroll32_user(base,op) \ + __asm__ __volatile__( \ + " .set push \n" \ + " .set noreorder \n" \ + " .set mips0 \n" \ + " .set eva \n" \ + " cachee %1, 0x000(%0); cachee %1, 0x010(%0) \n" \ + " cachee %1, 0x020(%0); cachee %1, 0x030(%0) \n" \ + " cachee %1, 0x040(%0); cachee %1, 0x050(%0) \n" \ + " cachee %1, 0x060(%0); cachee %1, 0x070(%0) \n" \ + " cachee %1, 0x080(%0); cachee %1, 0x090(%0) \n" \ + " cachee %1, 0x0a0(%0); cachee %1, 0x0b0(%0) \n" \ + " cachee %1, 0x0c0(%0); cachee %1, 0x0d0(%0) \n" \ + " cachee %1, 0x0e0(%0); cachee %1, 0x0f0(%0) \n" \ + " cachee %1, 0x100(%0); cachee %1, 0x110(%0) \n" \ + " cachee %1, 0x120(%0); cachee %1, 0x130(%0) \n" \ + " cachee %1, 0x140(%0); cachee %1, 0x150(%0) \n" \ + " cachee %1, 0x160(%0); cachee %1, 0x170(%0) \n" \ + " cachee %1, 0x180(%0); cachee %1, 0x190(%0) \n" \ + " cachee %1, 0x1a0(%0); cachee %1, 0x1b0(%0) \n" \ + " cachee %1, 0x1c0(%0); cachee %1, 0x1d0(%0) \n" \ + " cachee %1, 0x1e0(%0); cachee %1, 0x1f0(%0) \n" \ + " .set pop \n" \ + : \ + : "r" (base), \ + "i" (op)); + +#define cache32_unroll32_user(base, op) \ + __asm__ __volatile__( \ + " .set push \n" \ + " .set noreorder \n" \ + " .set mips0 \n" \ + " .set eva \n" \ + " cachee %1, 0x000(%0); cachee %1, 0x020(%0) \n" \ + " cachee %1, 0x040(%0); cachee %1, 0x060(%0) \n" \ + " cachee %1, 0x080(%0); cachee %1, 0x0a0(%0) \n" \ + " cachee %1, 0x0c0(%0); cachee %1, 0x0e0(%0) \n" \ + " cachee %1, 0x100(%0); cachee %1, 0x120(%0) \n" \ + " cachee %1, 0x140(%0); cachee %1, 0x160(%0) \n" \ + " cachee %1, 0x180(%0); cachee %1, 0x1a0(%0) \n" \ + " cachee %1, 0x1c0(%0); cachee %1, 0x1e0(%0) \n" \ + " cachee %1, 0x200(%0); cachee %1, 0x220(%0) \n" \ + " cachee %1, 0x240(%0); cachee %1, 0x260(%0) \n" \ + " cachee %1, 0x280(%0); cachee %1, 0x2a0(%0) \n" \ + " cachee %1, 0x2c0(%0); cachee %1, 0x2e0(%0) \n" \ + " cachee %1, 0x300(%0); cachee %1, 0x320(%0) \n" \ + " cachee %1, 0x340(%0); cachee %1, 0x360(%0) \n" \ + " cachee %1, 0x380(%0); cachee %1, 0x3a0(%0) \n" \ + " cachee %1, 0x3c0(%0); cachee %1, 0x3e0(%0) \n" \ + " .set pop \n" \ + : \ + : "r" (base), \ + "i" (op)); + +#define cache64_unroll32_user(base, op) \ + __asm__ __volatile__( \ + " .set push \n" \ + " .set noreorder \n" \ + " .set mips0 \n" \ + " .set eva \n" \ + " cachee %1, 0x000(%0); cachee %1, 0x040(%0) \n" \ + " cachee %1, 0x080(%0); cachee %1, 0x0c0(%0) \n" \ + " cachee %1, 0x100(%0); cachee %1, 0x140(%0) \n" \ + " cachee %1, 0x180(%0); cachee %1, 0x1c0(%0) \n" \ + " cachee %1, 0x200(%0); cachee %1, 0x240(%0) \n" \ + " cachee %1, 0x280(%0); cachee %1, 0x2c0(%0) \n" \ + " cachee %1, 0x300(%0); cachee %1, 0x340(%0) \n" \ + " cachee %1, 0x380(%0); cachee %1, 0x3c0(%0) \n" \ + " cachee %1, 0x400(%0); cachee %1, 0x440(%0) \n" \ + " cachee %1, 0x480(%0); cachee %1, 0x4c0(%0) \n" \ + " cachee %1, 0x500(%0); cachee %1, 0x540(%0) \n" \ + " cachee %1, 0x580(%0); cachee %1, 0x5c0(%0) \n" \ + " cachee %1, 0x600(%0); cachee %1, 0x640(%0) \n" \ + " cachee %1, 0x680(%0); cachee %1, 0x6c0(%0) \n" \ + " cachee %1, 0x700(%0); cachee %1, 0x740(%0) \n" \ + " cachee %1, 0x780(%0); cachee %1, 0x7c0(%0) \n" \ + " .set pop \n" \ + : \ + : "r" (base), \ + "i" (op)); + /* build blast_xxx, blast_xxx_page, blast_xxx_page_indexed */ -#define __BUILD_BLAST_CACHE(pfx, desc, indexop, hitop, lsize) \ -static inline void blast_##pfx##cache##lsize(void) \ +#define __BUILD_BLAST_CACHE(pfx, desc, indexop, hitop, lsize, extra) \ +static inline void extra##blast_##pfx##cache##lsize(void) \ { \ unsigned long start = INDEX_BASE; \ unsigned long end = start + current_cpu_data.desc.waysize; \ @@ -359,7 +574,7 @@ __##pfx##flush_epilogue \ } \ \ -static inline void blast_##pfx##cache##lsize##_page(unsigned long page) \ +static inline void extra##blast_##pfx##cache##lsize##_page(unsigned long page) \ { \ unsigned long start = page; \ unsigned long end = page + PAGE_SIZE; \ @@ -374,7 +589,7 @@ __##pfx##flush_epilogue \ } \ \ -static inline void blast_##pfx##cache##lsize##_page_indexed(unsigned long page) \ +static inline void extra##blast_##pfx##cache##lsize##_page_indexed(unsigned long page) \ { \ unsigned long indexmask = current_cpu_data.desc.waysize - 1; \ unsigned long start = INDEX_BASE + (page & indexmask); \ @@ -393,52 +608,154 @@ __##pfx##flush_epilogue \ } -__BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 16) -__BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 16) -__BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 16) -__BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 32) -__BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 32) -__BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 32) -__BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 64) -__BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 64) -__BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 64) -__BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 128) - -__BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 16) -__BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 32) -__BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 16) -__BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 32) -__BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 64) -__BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 128) +__BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 16, ) +__BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 16, ) +__BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 16, ) +__BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 32, ) +__BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 32, ) +__BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I_Loongson2, 32, loongson2_) +__BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 32, ) +__BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 64, ) +__BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 64, ) +__BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 64, ) +__BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 128, ) +__BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 128, ) +__BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 128, ) + +__BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 16, ) +__BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 32, ) +__BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 16, ) +__BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 32, ) +__BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 64, ) +__BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 128, ) + +#define __BUILD_BLAST_USER_CACHE(pfx, desc, indexop, hitop, lsize) \ +static inline void blast_##pfx##cache##lsize##_user_page(unsigned long page) \ +{ \ + unsigned long start = page; \ + unsigned long end = page + PAGE_SIZE; \ + \ + __##pfx##flush_prologue \ + \ + do { \ + cache##lsize##_unroll32_user(start, hitop); \ + start += lsize * 32; \ + } while (start < end); \ + \ + __##pfx##flush_epilogue \ +} + +__BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, + 16) +__BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 16) +__BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, + 32) +__BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 32) +__BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, + 64) +__BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 64) /* build blast_xxx_range, protected_blast_xxx_range */ -#define __BUILD_BLAST_CACHE_RANGE(pfx, desc, hitop, prot) \ -static inline void prot##blast_##pfx##cache##_range(unsigned long start, \ +#define __BUILD_BLAST_CACHE_RANGE(pfx, desc, hitop, prot, extra) \ +static inline void prot##extra##blast_##pfx##cache##_range(unsigned long start, \ unsigned long end) \ { \ unsigned long lsize = cpu_##desc##_line_size(); \ + unsigned long lsize_2 = lsize * 2; \ + unsigned long lsize_3 = lsize * 3; \ + unsigned long lsize_4 = lsize * 4; \ + unsigned long lsize_5 = lsize * 5; \ + unsigned long lsize_6 = lsize * 6; \ + unsigned long lsize_7 = lsize * 7; \ + unsigned long lsize_8 = lsize * 8; \ unsigned long addr = start & ~(lsize - 1); \ - unsigned long aend = (end - 1) & ~(lsize - 1); \ + unsigned long aend = (end + lsize - 1) & ~(lsize - 1); \ + int lines = (aend - addr) / lsize; \ \ __##pfx##flush_prologue \ \ - while (1) { \ + while (lines >= 8) { \ + prot##cache_op(hitop, addr); \ + prot##cache_op(hitop, addr + lsize); \ + prot##cache_op(hitop, addr + lsize_2); \ + prot##cache_op(hitop, addr + lsize_3); \ + prot##cache_op(hitop, addr + lsize_4); \ + prot##cache_op(hitop, addr + lsize_5); \ + prot##cache_op(hitop, addr + lsize_6); \ + prot##cache_op(hitop, addr + lsize_7); \ + addr += lsize_8; \ + lines -= 8; \ + } \ + \ + if (lines & 0x4) { \ + prot##cache_op(hitop, addr); \ + prot##cache_op(hitop, addr + lsize); \ + prot##cache_op(hitop, addr + lsize_2); \ + prot##cache_op(hitop, addr + lsize_3); \ + addr += lsize_4; \ + } \ + \ + if (lines & 0x2) { \ + prot##cache_op(hitop, addr); \ + prot##cache_op(hitop, addr + lsize); \ + addr += lsize_2; \ + } \ + \ + if (lines & 0x1) { \ prot##cache_op(hitop, addr); \ - if (addr == aend) \ - break; \ - addr += lsize; \ } \ \ __##pfx##flush_epilogue \ } -__BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, protected_) -__BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, protected_) -__BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, protected_) -__BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, ) -__BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, ) +#ifndef CONFIG_EVA + +__BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, protected_, ) +__BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, protected_, ) + +#else + +#define __BUILD_PROT_BLAST_CACHE_RANGE(pfx, desc, hitop) \ +static inline void protected_blast_##pfx##cache##_range(unsigned long start,\ + unsigned long end) \ +{ \ + unsigned long lsize = cpu_##desc##_line_size(); \ + unsigned long addr = start & ~(lsize - 1); \ + unsigned long aend = (end - 1) & ~(lsize - 1); \ + \ + __##pfx##flush_prologue \ + \ + if (segment_eq(get_fs(), USER_DS)) { \ + while (1) { \ + protected_cachee_op(hitop, addr); \ + if (addr == aend) \ + break; \ + addr += lsize; \ + } \ + } else { \ + while (1) { \ + protected_cache_op(hitop, addr); \ + if (addr == aend) \ + break; \ + addr += lsize; \ + } \ + \ + } \ + __##pfx##flush_epilogue \ +} + +__BUILD_PROT_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D) +__BUILD_PROT_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I) + +#endif +__BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, protected_, ) +__BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I_Loongson2, \ + protected_, loongson2_) +__BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, , ) +__BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, , ) +__BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, , ) /* blast_inv_dcache_range */ -__BUILD_BLAST_CACHE_RANGE(inv_d, dcache, Hit_Invalidate_D, ) -__BUILD_BLAST_CACHE_RANGE(inv_s, scache, Hit_Invalidate_SD, ) +__BUILD_BLAST_CACHE_RANGE(inv_d, dcache, Hit_Invalidate_D, , ) +__BUILD_BLAST_CACHE_RANGE(inv_s, scache, Hit_Invalidate_SD, , ) #endif /* _ASM_R4KCACHE_H */