--- zzzz-none-000/linux-3.10.107/arch/mips/mm/c-r4k.c 2017-06-27 09:49:32.000000000 +0000 +++ vr9-7490-729/linux-3.10.107/arch/mips/mm/c-r4k.c 2021-11-10 11:53:54.000000000 +0000 @@ -6,6 +6,7 @@ * Copyright (C) 1996 David S. Miller (davem@davemloft.net) * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2012, MIPS Technology, Leonid Yegoshin (yegoshin@mips.com) */ #include #include @@ -44,6 +45,13 @@ * o collapses to normal function call on systems with a single shared * primary cache. * o doesn't disable interrupts on the local CPU + * + * Note: this function is used now for address cacheops only + * + * Note2: It is unsafe to use address cacheops via SMP call, other CPU may not + * have this process address map (ASID) loaded into EntryHI and + * it usualy requires some tricks, which are absent from this file. + * Cross-CPU address cacheops are much easy and safely. */ static inline void r4k_on_each_cpu(void (*func) (void *info), void *info) { @@ -56,13 +64,67 @@ preempt_enable(); } -#if defined(CONFIG_MIPS_CMP) +#if defined(CONFIG_MIPS_CMP) || defined(CONFIG_MIPS_CPS) #define cpu_has_safe_index_cacheops 0 #else #define cpu_has_safe_index_cacheops 1 #endif /* + * This variant of smp_call_function is used for index cacheops only. + */ +static inline void r4k_indexop_on_each_cpu(void (*func) (void *info), void *info) +{ + preempt_disable(); + +#ifdef CONFIG_SMP + if (!cpu_has_safe_index_cacheops) { + + if (smp_num_siblings > 1) { + cpumask_t tmp_mask = INIT_CPUMASK; + int cpu, this_cpu, n = 0; + + /* If processor hasn't safe index cachops (likely) + then run cache flush on other CPUs. + But I assume that siblings have common L1 cache, so - + - run cache flush only once per sibling group. LY22 */ + + this_cpu = smp_processor_id(); + for_each_online_cpu(cpu) { + + if (cpumask_test_cpu(cpu, (&per_cpu(cpu_sibling_map, this_cpu)))) + continue; + + if (cpumask_intersects(&tmp_mask, (&per_cpu(cpu_sibling_map, cpu)))) + continue; + cpu_set(cpu, tmp_mask); + n++; + } + if (n) + smp_call_function_many(&tmp_mask, func, info, 1); + } else + smp_call_function(func, info, 1); + } +#endif + func(info); + preempt_enable(); +} + +/* Define a rough size where address cacheops are still more optimal than + * index cacheops on whole cache (in D/I-cache size terms). + * Value "2" reflects an expense of smp_call_function() on top of + * whole cache flush via index cacheops. + */ +#ifndef CACHE_CPU_LATENCY +#ifdef CONFIG_SMP +#define CACHE_CPU_LATENCY (2) +#else +#define CACHE_CPU_LATENCY (1) +#endif +#endif + + +/* * Must die. */ static unsigned long icache_size __read_mostly; @@ -122,6 +184,28 @@ r4k_blast_dcache_page = r4k_blast_dcache_page_dc64; } +#ifndef CONFIG_EVA +#define r4k_blast_dcache_user_page r4k_blast_dcache_page +#else + +static void (*r4k_blast_dcache_user_page)(unsigned long addr); + +static void __cpuinit r4k_blast_dcache_user_page_setup(void) +{ + unsigned long dc_lsize = cpu_dcache_line_size(); + + if (dc_lsize == 0) + r4k_blast_dcache_user_page = (void *)cache_noop; + else if (dc_lsize == 16) + r4k_blast_dcache_user_page = blast_dcache16_user_page; + else if (dc_lsize == 32) + r4k_blast_dcache_user_page = blast_dcache32_user_page; + else if (dc_lsize == 64) + r4k_blast_dcache_user_page = blast_dcache64_user_page; +} + +#endif + static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); static void __cpuinit r4k_blast_dcache_page_indexed_setup(void) @@ -242,6 +326,27 @@ r4k_blast_icache_page = blast_icache64_page; } +#ifndef CONFIG_EVA +#define r4k_blast_icache_user_page r4k_blast_icache_page +#else + +static void (* r4k_blast_icache_user_page)(unsigned long addr); + +static void __cpuinit r4k_blast_icache_user_page_setup(void) +{ + unsigned long ic_lsize = cpu_icache_line_size(); + + if (ic_lsize == 0) + r4k_blast_icache_user_page = (void *)cache_noop; + else if (ic_lsize == 16) + r4k_blast_icache_user_page = blast_icache16_user_page; + else if (ic_lsize == 32) + r4k_blast_icache_user_page = blast_icache32_user_page; + else if (ic_lsize == 64) + r4k_blast_icache_user_page = blast_icache64_user_page; +} + +#endif static void (* r4k_blast_icache_page_indexed)(unsigned long addr); @@ -366,7 +471,7 @@ static void r4k___flush_cache_all(void) { - r4k_on_each_cpu(local_r4k___flush_cache_all, NULL); + r4k_indexop_on_each_cpu(local_r4k___flush_cache_all, NULL); } static inline int has_valid_asid(const struct mm_struct *mm) @@ -384,16 +489,73 @@ #endif } -static void r4k__flush_cache_vmap(void) + +static inline void local_r4__flush_dcache(void *args) { r4k_blast_dcache(); } -static void r4k__flush_cache_vunmap(void) +struct vmap_args { + unsigned long start; + unsigned long end; +}; + +static inline void local_r4__flush_cache_vmap(void *args) { - r4k_blast_dcache(); + blast_dcache_range(((struct vmap_args *)args)->start,((struct vmap_args *)args)->end); } +static void r4k__flush_cache_vmap(unsigned long start, unsigned long end) +{ + unsigned long size = end - start; + + if (cpu_has_safe_index_cacheops && size >= dcache_size) { + r4k_blast_dcache(); + } else { +/* Commented out until bug in free_unmap_vmap_area() is fixed - it calls + with unmapped page and address cache op does TLB refill exception + if (size >= (dcache_size * CACHE_CPU_LATENCY)) + */ + r4k_indexop_on_each_cpu(local_r4__flush_dcache, NULL); +/* Commented out until bug in free_unmap_vmap_area() is fixed - it calls + with unmapped page and address cache op does TLB refill exception + else { + struct vmap_args args; + + args.start = start; + args.end = end; + r4k_on_each_cpu(local_r4__flush_cache_vmap, (void *)&args); + } + */ + } +} + +static void r4k__flush_cache_vunmap(unsigned long start, unsigned long end) +{ + unsigned long size = end - start; + + if (cpu_has_safe_index_cacheops && size >= dcache_size) + r4k_blast_dcache(); + else { +/* Commented out until bug in free_unmap_vmap_area() is fixed - it calls + with unmapped page and address cache op does TLB refill exception + if (size >= (dcache_size * CACHE_CPU_LATENCY)) + */ + r4k_indexop_on_each_cpu(local_r4__flush_dcache, NULL); +/* Commented out until bug in free_unmap_vmap_area() is fixed - it calls + with unmapped page and address cache op does TLB refill exception + else { + struct vmap_args args; + + args.start = start; + args.end = end; + r4k_on_each_cpu(local_r4__flush_cache_vmap, (void *)&args); + } + */ + } +} + + static inline void local_r4k_flush_cache_range(void * args) { struct vm_area_struct *vma = args; @@ -403,8 +565,11 @@ return; r4k_blast_dcache(); - if (exec) + if (exec) { + if (!cpu_has_ic_fills_f_dc) + wmb(); r4k_blast_icache(); + } } static void r4k_flush_cache_range(struct vm_area_struct *vma, @@ -413,7 +578,7 @@ int exec = vma->vm_flags & VM_EXEC; if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) - r4k_on_each_cpu(local_r4k_flush_cache_range, vma); + r4k_indexop_on_each_cpu(local_r4k_flush_cache_range, vma); } static inline void local_r4k_flush_cache_mm(void * args) @@ -445,7 +610,7 @@ if (!cpu_has_dc_aliases) return; - r4k_on_each_cpu(local_r4k_flush_cache_mm, mm); + r4k_indexop_on_each_cpu(local_r4k_flush_cache_mm, mm); } struct flush_cache_page_args { @@ -468,6 +633,7 @@ pmd_t *pmdp; pte_t *ptep; void *vaddr; + int dontflash = 0; /* * If ownes no valid ASID yet, cannot possibly have gotten @@ -489,9 +655,21 @@ if (!(pte_present(*ptep))) return; - if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) - vaddr = NULL; - else { + /* accelerate it! See below, just skipping kmap_*()/kunmap_*() */ + if ((!exec) && !cpu_has_dc_aliases) + return; + + if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) { + if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { + r4k_blast_dcache_user_page(addr); + if (exec && (!cpu_has_cm2) && !cpu_has_ic_fills_f_dc) + wmb(); + if (exec && !cpu_icache_snoops_remote_store) + r4k_blast_scache_page(addr); + } + if (exec) + r4k_blast_icache_user_page(addr); + } else { /* * Use kmap_coherent or kmap_atomic to do flushes for * another ASID than the current one. @@ -503,28 +681,37 @@ else vaddr = kmap_atomic(page); addr = (unsigned long)vaddr; - } - - if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { - r4k_blast_dcache_page(addr); - if (exec && !cpu_icache_snoops_remote_store) - r4k_blast_scache_page(addr); - } - if (exec) { - if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) { - int cpu = smp_processor_id(); - if (cpu_context(cpu, mm) != 0) - drop_mmu_context(mm, cpu); - } else - r4k_blast_icache_page(addr); - } + if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { + r4k_blast_dcache_page(addr); + if (exec && (!cpu_has_cm2) && !cpu_has_ic_fills_f_dc) + wmb(); + if (exec && !cpu_icache_snoops_remote_store) + r4k_blast_scache_page(addr); + } + if (exec) { + if (cpu_has_vtag_icache && mm == current->active_mm) { + int cpu = smp_processor_id(); + + if (cpu_context(cpu, mm) != 0) + drop_mmu_context(mm, cpu); + dontflash = 1; + } else + if (map_coherent || !cpu_has_ic_aliases) + r4k_blast_icache_page(addr); + } - if (vaddr) { if (map_coherent) kunmap_coherent(); else kunmap_atomic(vaddr); + + /* in case of I-cache aliasing - blast it via coherent page */ + if (exec && cpu_has_ic_aliases && (!dontflash) && !map_coherent) { + vaddr = kmap_coherent(page, addr); + r4k_blast_icache_page((unsigned long)vaddr); + kunmap_coherent(); + } } } @@ -538,6 +725,8 @@ args.pfn = pfn; r4k_on_each_cpu(local_r4k_flush_cache_page, &args); + if (cpu_has_dc_aliases) + ClearPageDcacheDirty(pfn_to_page(pfn)); } static inline void local_r4k_flush_data_cache_page(void * addr) @@ -553,11 +742,82 @@ r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr); } + +struct mips_flush_data_cache_range_args { + struct vm_area_struct *vma; + unsigned long start; + unsigned long end; +}; + +static inline void local_r4k_mips_flush_data_cache_range(void *args) +{ + struct mips_flush_data_cache_range_args *f_args = args; + unsigned long start = f_args->start; + unsigned long end = f_args->end; + struct vm_area_struct * vma = f_args->vma; + + blast_dcache_range(start, end); + + if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc) { + if (!cpu_has_cm2) + wmb(); + + /* vma is given for exec check only, mmap is current, + so - no non-current vma page flush, just user or kernel */ + protected_blast_icache_range(start, end); + } +} + +/* flush dirty kernel data and a corresponding user instructions (if needed). + used in copy_to_user_page() */ +static void r4k_mips_flush_data_cache_range(struct vm_area_struct *vma, + struct page *page, unsigned long start, unsigned long len) +{ + struct mips_flush_data_cache_range_args args; + + args.vma = vma; + args.start = start; + args.end = start + len; + + r4k_on_each_cpu(local_r4k_mips_flush_data_cache_range, (void *)&args); +} + + struct flush_icache_range_args { unsigned long start; unsigned long end; }; +static inline void local_r4k_flush_icache(void *args) +{ + if (!cpu_has_ic_fills_f_dc) { + r4k_blast_dcache(); + + wmb(); + } + + r4k_blast_icache(); +} + +static inline void local_r4k_flush_icache_range_ipi(void *args) +{ + struct flush_icache_range_args *fir_args = args; + unsigned long start = fir_args->start; + unsigned long end = fir_args->end; + + if (!cpu_has_ic_fills_f_dc) { + R4600_HIT_CACHEOP_WAR_IMPL; + protected_blast_dcache_range(start, end); + + if (!cpu_has_cm2) + wmb(); + } + + protected_blast_icache_range(start, end); + +} + +/* This function is used only for local CPU only while boot etc */ static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end) { if (!cpu_has_ic_fills_f_dc) { @@ -565,36 +825,53 @@ r4k_blast_dcache(); } else { R4600_HIT_CACHEOP_WAR_IMPL; - protected_blast_dcache_range(start, end); + blast_dcache_range(start, end); } + + wmb(); } if (end - start > icache_size) r4k_blast_icache(); else - protected_blast_icache_range(start, end); -} - -static inline void local_r4k_flush_icache_range_ipi(void *args) -{ - struct flush_icache_range_args *fir_args = args; - unsigned long start = fir_args->start; - unsigned long end = fir_args->end; - - local_r4k_flush_icache_range(start, end); + blast_icache_range(start, end); +#ifdef CONFIG_EVA + /* This is here to smooth effect of any kind of address aliasing. + It is used only during boot, so - it doesn't create an impact on + performance. LY22 */ + bc_wback_inv(start, (end - start)); + __sync(); +#endif } +/* this function can be called for kernel OR user addresses, + * kernel is for module, *gdb*. User is for binfmt_a.out/flat + * So - take care, check get_fs() */ static void r4k_flush_icache_range(unsigned long start, unsigned long end) { struct flush_icache_range_args args; + unsigned long size = end - start; args.start = start; args.end = end; - r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args); + if (cpu_has_safe_index_cacheops && + (((size >= icache_size) && !cpu_has_ic_fills_f_dc) || + (size >= dcache_size))) + local_r4k_flush_icache((void *)&args); + else if (((size < (icache_size * CACHE_CPU_LATENCY)) && !cpu_has_ic_fills_f_dc) || + (size < (dcache_size * CACHE_CPU_LATENCY))) { + struct flush_icache_range_args args; + + args.start = start; + args.end = end; + r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, (void *)&args); + } else + r4k_indexop_on_each_cpu(local_r4k_flush_icache, NULL); instruction_hazard(); } + #ifdef CONFIG_DMA_NONCOHERENT static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) @@ -627,7 +904,8 @@ preempt_enable(); bc_wback_inv(addr, size); - __sync(); + if (!cpu_has_cm2_l2sync) + __sync(); } static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) @@ -750,7 +1028,10 @@ args.vaddr = (unsigned long) vaddr; args.size = size; - r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args); + if (cpu_has_safe_index_cacheops && size >= dcache_size) + r4k_indexop_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args); + else + r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args); } static inline void rm7k_erratum31(void) @@ -787,20 +1068,30 @@ static inline void alias_74k_erratum(struct cpuinfo_mips *c) { + unsigned int imp = c->processor_id & 0xff00; + unsigned int rev = c->processor_id & PRID_REV_MASK; + /* * Early versions of the 74K do not update the cache tags on a * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG * aliases. In this case it is better to treat the cache as always * having aliases. */ - if ((c->processor_id & 0xff) <= PRID_REV_ENCODE_332(2, 4, 0)) - c->dcache.flags |= MIPS_CACHE_VTAG; - if ((c->processor_id & 0xff) == PRID_REV_ENCODE_332(2, 4, 0)) - write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); - if (((c->processor_id & 0xff00) == PRID_IMP_1074K) && - ((c->processor_id & 0xff) <= PRID_REV_ENCODE_332(1, 1, 0))) { - c->dcache.flags |= MIPS_CACHE_VTAG; - write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); + switch (imp) { + case PRID_IMP_74K: + if (rev <= PRID_REV_ENCODE_332(2, 4, 0)) + c->dcache.flags |= MIPS_CACHE_VTAG; + if (rev == PRID_REV_ENCODE_332(2, 4, 0)) + write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); + break; + case PRID_IMP_1074K: + if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) { + c->dcache.flags |= MIPS_CACHE_VTAG; + write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); + } + break; + default: + BUG(); } } @@ -1072,9 +1363,18 @@ case CPU_34K: case CPU_74K: case CPU_1004K: - if (c->cputype == CPU_74K) + case CPU_1074K: + case CPU_PROAPTIV: + case CPU_P5600: + case CPU_INTERAPTIV: + case CPU_M5150: + if ((c->cputype == CPU_74K) || (c->cputype == CPU_1074K)) alias_74k_erratum(c); - if ((read_c0_config7() & (1 << 16))) { + if (!(read_c0_config7() & MIPS_CONF7_IAR)) { + if (c->icache.waysize > PAGE_SIZE) + c->icache.flags |= MIPS_CACHE_ALIASES; + } + if (read_c0_config7() & MIPS_CONF7_AR) { /* effectively physically indexed dcache, thus no virtual aliases. */ c->dcache.flags |= MIPS_CACHE_PINDEX; @@ -1085,6 +1385,14 @@ c->dcache.flags |= MIPS_CACHE_ALIASES; } +#ifdef CONFIG_HIGHMEM + if (((c->dcache.flags & MIPS_CACHE_ALIASES) && + ((c->dcache.waysize / PAGE_SIZE) > FIX_N_COLOURS)) || + ((c->icache.flags & MIPS_CACHE_ALIASES) && + ((c->icache.waysize / PAGE_SIZE) > FIX_N_COLOURS))) + panic("PAGE_SIZE*WAYS too small for L1 size, too many colors"); +#endif + switch (c->cputype) { case CPU_20KC: /* @@ -1107,10 +1415,12 @@ c->icache.ways = 1; #endif - printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", - icache_size >> 10, + printk("Primary instruction cache %ldkB, %s, %s, %slinesize %d bytes.\n", + icache_size >> 10, way_string[c->icache.ways], c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT", - way_string[c->icache.ways], c->icache.linesz); + (c->icache.flags & MIPS_CACHE_ALIASES) ? + "I-cache aliases, " : "", + c->icache.linesz); printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n", dcache_size >> 10, way_string[c->dcache.ways], @@ -1336,11 +1646,11 @@ NXP_BARRIER(); } -static int __cpuinitdata cca = -1; +unsigned int mips_cca = -1; static int __init cca_setup(char *str) { - get_option(&str, &cca); + get_option(&str, &mips_cca); return 0; } @@ -1349,12 +1659,12 @@ static void __cpuinit coherency_setup(void) { - if (cca < 0 || cca > 7) - cca = read_c0_config() & CONF_CM_CMASK; - _page_cachable_default = cca << _CACHE_SHIFT; + if (mips_cca < 0 || mips_cca > 7) + mips_cca = read_c0_config() & CONF_CM_CMASK; + _page_cachable_default = mips_cca << _CACHE_SHIFT; - pr_debug("Using cache attribute %d\n", cca); - change_c0_config(CONF_CM_CMASK, cca); + pr_debug("Using cache attribute %d\n", mips_cca); + change_c0_config(CONF_CM_CMASK, mips_cca); /* * c0_status.cu=0 specifies that updates by the sc instruction use @@ -1423,6 +1733,10 @@ r4k_blast_scache_page_setup(); r4k_blast_scache_page_indexed_setup(); r4k_blast_scache_setup(); +#ifdef CONFIG_EVA + r4k_blast_dcache_user_page_setup(); + r4k_blast_icache_user_page_setup(); +#endif /* * Some MIPS32 and MIPS64 processors have physically indexed caches. @@ -1451,11 +1765,12 @@ flush_icache_all = r4k_flush_icache_all; local_flush_data_cache_page = local_r4k_flush_data_cache_page; flush_data_cache_page = r4k_flush_data_cache_page; + mips_flush_data_cache_range = r4k_mips_flush_data_cache_range; flush_icache_range = r4k_flush_icache_range; local_flush_icache_range = local_r4k_flush_icache_range; #if defined(CONFIG_DMA_NONCOHERENT) - if (coherentio) { + if (coherentio > 0) { _dma_cache_wback_inv = (void *)cache_noop; _dma_cache_wback = (void *)cache_noop; _dma_cache_inv = (void *)cache_noop; @@ -1475,6 +1790,13 @@ * or not to flush caches. */ local_r4k___flush_cache_all(NULL); +#ifdef CONFIG_EVA + /* this is done just in case if some address aliasing does exist in + board like old Malta memory map. Doesn't hurt anyway. LY22 */ + smp_wmb(); + r4k_blast_scache(); + smp_wmb(); +#endif coherency_setup(); board_cache_error_setup = r4k_cache_error_setup;