--- zzzz-none-000/linux-4.19.183/drivers/irqchip/irq-gic.c 2021-03-24 10:07:39.000000000 +0000 +++ bcm63-7530ax-756/linux-4.19.183/drivers/irqchip/irq-gic.c 2023-06-28 08:54:19.000000000 +0000 @@ -20,6 +20,12 @@ * As such, the enable set/clear, pending set/clear and active bit * registers are banked per-cpu for these sources. */ + +#ifdef CONFIG_AVM_FASTIRQ +#include +#define CLIENT_FIQ_PRIO FIQ_PRIO_WATCHDOG +#endif + #include #include #include @@ -44,10 +50,20 @@ #include #include + +#if defined(CONFIG_BCM963138) +#include +#include +#endif + #include #include #include +#include +#include + + #include "irq-gic-common.h" #ifdef CONFIG_ARM64 @@ -63,6 +79,21 @@ #define gic_check_cpu_features() do { } while(0) #endif +#if defined(CONFIG_AVM_FASTIRQ) + +#include +#include +#include + +#include +#define __BUILD_AVM_CONTEXT_FUNC(func) firq_##func + +#else + +#define __BUILD_AVM_CONTEXT_FUNC(func) func + +#endif + union gic_base { void __iomem *common_base; void __percpu * __iomem *percpu_base; @@ -77,10 +108,12 @@ u32 percpu_offset; #if defined(CONFIG_CPU_PM) || defined(CONFIG_ARM_GIC_PM) u32 saved_spi_enable[DIV_ROUND_UP(1020, 32)]; + u32 saved_spi_security[DIV_ROUND_UP(1020, 32)]; u32 saved_spi_active[DIV_ROUND_UP(1020, 32)]; u32 saved_spi_conf[DIV_ROUND_UP(1020, 16)]; u32 saved_spi_target[DIV_ROUND_UP(1020, 4)]; u32 __percpu *saved_ppi_enable; + u32 __percpu *saved_ppi_security; u32 __percpu *saved_ppi_active; u32 __percpu *saved_ppi_conf; #endif @@ -93,15 +126,15 @@ #ifdef CONFIG_BL_SWITCHER -static DEFINE_RAW_SPINLOCK(cpu_map_lock); +DEFINE_RAW_SPINLOCK(cpu_map_lock); #define gic_lock_irqsave(f) \ - raw_spin_lock_irqsave(&cpu_map_lock, (f)) + __BUILD_AVM_CONTEXT_FUNC(raw_spin_lock_irqsave(&cpu_map_lock, (f))) #define gic_unlock_irqrestore(f) \ - raw_spin_unlock_irqrestore(&cpu_map_lock, (f)) + __BUILD_AVM_CONTEXT_FUNC(raw_spin_unlock_irqrestore(&cpu_map_lock, (f))) -#define gic_lock() raw_spin_lock(&cpu_map_lock) -#define gic_unlock() raw_spin_unlock(&cpu_map_lock) +#define gic_lock() __BUILD_AVM_CONTEXT_FUNC(raw_spin_lock(&cpu_map_lock)) +#define gic_unlock() __BUILD_AVM_CONTEXT_FUNC(raw_spin_unlock(&cpu_map_lock)) #else @@ -159,6 +192,40 @@ #define gic_set_base_accessor(d, f) #endif +void __iomem * get_dist_base( unsigned int gic_nr ) +{ + struct gic_chip_data *gic; + + BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR); + + gic = &gic_data[gic_nr]; + + return(gic_data_dist_base(gic)); +} + +void __iomem * get_cpu_base( unsigned int gic_nr ) +{ + struct gic_chip_data *gic; + + BUG_ON(gic_nr >= CONFIG_ARM_GIC_MAX_NR); + + gic = &gic_data[gic_nr]; + + return(gic_data_cpu_base(gic)); +} + +#ifndef CONFIG_AVM_FASTIRQ_TZ +#define SCU_PHYS_BASE 0x8001e000 +#define DIST_BASE_PHY (SCU_PHYS_BASE + CA9MP_GIC_DIST_OFF) +#define CPU_BASE_PHY (SCU_PHYS_BASE + CA9MP_GIC_CPUIF_OFF) +#endif + +struct irq_domain *get_irq_domain( void ) +{ + struct gic_chip_data *gic = &gic_data[0]; + return( gic->domain ); +} + static inline void __iomem *gic_dist_base(struct irq_data *d) { struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d); @@ -204,6 +271,9 @@ static void gic_mask_irq(struct irq_data *d) { +#if defined(CONFIG_AVM_FASTIRQ) + BUG_ON(avm_gic_fiq_is(gic_irq(d))); +#endif/*--- #if defined(CONFIG_AVM_FASTIRQ) ---*/ gic_poke_irq(d, GIC_DIST_ENABLE_CLEAR); } @@ -227,9 +297,16 @@ gic_poke_irq(d, GIC_DIST_ENABLE_SET); } +#if defined(CONFIG_AVM_FASTIRQ) && !defined(CONFIG_AVM_FASTIRQ_TZ) +unsigned int avm_secmon_gate(unsigned int wr, unsigned int addr, unsigned int val); +#endif static void gic_eoi_irq(struct irq_data *d) { +#if defined(CONFIG_AVM_FASTIRQ) && !defined(CONFIG_AVM_FASTIRQ_TZ) + (void)avm_secmon_gate(1, (CPU_BASE_PHY + GIC_CPU_EOI), gic_irq(d)); +#else writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI); +#endif } static void gic_eoimode1_eoi_irq(struct irq_data *d) @@ -342,25 +419,122 @@ } #endif +#if defined(CONFIG_AVM_FASTIRQ) && !defined(CONFIG_AVM_FASTIRQ_TZ) +/*--------------------------------------------------------------------------------*\ + * Code, um aus dem IRQ in den FIQ zu wechseln und den FIQ-Handler anzuspringen * +\*--------------------------------------------------------------------------------*/ + +static void noinline handle_FIQ(unsigned long flags) +{ + unsigned long addr; + unsigned long flags_save; + + asm volatile( \ + "mrs %1, cpsr \n" \ + "bic %1, %1, #0x1f \n" \ + "orr %1, %1, #0x11 \n" \ + "msr cpsr_fsxc, %1 \n" \ + "isb \n" \ + "msr spsr_fsxc, %2 \n" \ + "movw lr, #:lower16:2f \n" \ + "movt lr, #:upper16:2f \n" \ + "ldr %0, =avm_fiq_handler_begin \n" \ + "bx %0 \n" \ + "1: mov pc, lr \n" \ + "2: \n" \ + : "=r" (addr), "=&r" (flags_save) \ + : "r" (flags) \ + : "cc" \ + ); +} +#endif + static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) { - u32 irqstat, irqnr; + u32 irq_handled, irqstat, irqnr; struct gic_chip_data *gic = &gic_data[0]; void __iomem *cpu_base = gic_data_cpu_base(gic); +#ifndef CONFIG_AVM_FASTIRQ_TZ + unsigned long flags; +#endif + + avm_cpu_wait_end(); /*auch wenn es wait_irqoff gibt: trotzdem aufrufen, um system-load-Ausgabe zu triggern */ +#if !defined(CONFIG_AVM_FASTIRQ) + /* trace code only if no fastirq-profiler exist */ + avm_simple_profiling_enter_irq_context(regs->ARM_pc, regs->ARM_lr); +#endif + irq_handled = 0; do { + dsb(); + +#if defined(CONFIG_AVM_FASTIRQ) + + /* NON-SECURE - Zugriff zum Auslesen des Interrupts ... */ +#ifndef CONFIG_AVM_FASTIRQ_TZ + flags = avm_raw_local_fiq_and_iq_disable(); + + irqstat = avm_secmon_gate(0, (CPU_BASE_PHY + GIC_CPU_INTACK), 0); +#else irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK); +#endif irqnr = irqstat & GICC_IAR_INT_ID_MASK; +#ifndef CONFIG_AVM_FASTIRQ_TZ + if (unlikely(irqnr >= 1020))//(unlikely(avm_gic_fiq_is(irqnr))) + { + irqstat = readl_relaxed(cpu_base + OFFSET_ICCHPIR); + irqnr = irqstat & GICC_IAR_INT_ID_MASK; + + if (likely(avm_gic_fiq_is(irqnr))) + { + /* SECURE interrupt, passiert ungluecklicherweise ab und zu ... */ + handle_FIQ(flags); // avm_raw_local_fiq_and_iq_enable(flags) implizit bei Ruecksprung + // mit Modewechsel fuer flags + break; + } + else + { + avm_raw_local_fiq_and_iq_enable(flags); + + /* Interrupt Nr 1020, 1021, 1022 und Nr > 1023 fuer NON-SECURE nicht definiert, einfach Beenden ... */ + /* Spurious Interrupts zählen */ + if(!irq_handled) { + __this_cpu_write(spurious_count, (__this_cpu_read(spurious_count) + 1)); + } + + break; + } + } + + avm_raw_local_fiq_and_iq_enable(flags); +#endif + +#else + irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK); + irqnr = irqstat & GICC_IAR_INT_ID_MASK; +#endif + if (likely(irqnr > 15 && irqnr < 1020)) { if (static_branch_likely(&supports_deactivate_key)) writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI); isb(); + + /* NON-SECURE interrupt */ + avm_simple_profiling_log(avm_profile_data_type_hw_irq_begin, (unsigned long)(irq_to_desc(irqnr)), irqnr); handle_domain_irq(gic->domain, irqnr, regs); + avm_simple_profiling_log(avm_profile_data_type_hw_irq_end, (unsigned long)(irq_to_desc(irqnr)), irqnr); + irq_handled = 1; continue; } - if (irqnr < 16) { + if (likely(irqnr < 16)) { + /* NON-SECURE IPI */ +#if defined(CONFIG_AVM_FASTIRQ) && !defined(CONFIG_AVM_FASTIRQ_TZ) + (void)avm_secmon_gate(1, (CPU_BASE_PHY + GIC_CPU_EOI), irqstat); +#else writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI); +#endif + if (static_branch_likely(&supports_deactivate_key)) writel_relaxed(irqstat, cpu_base + GIC_CPU_DEACTIVATE); #ifdef CONFIG_SMP @@ -372,11 +546,22 @@ * Pairs with the write barrier in gic_raise_softirq */ smp_rmb(); + avm_simple_profiling_log(avm_profile_data_type_hw_irq_begin, (unsigned long)(irq_to_desc(irqnr)), irqnr); handle_IPI(irqnr, regs); + avm_simple_profiling_log(avm_profile_data_type_hw_irq_end, (unsigned long)(irq_to_desc(irqnr)), irqnr); #endif + irq_handled = 1; continue; } +#if defined(CONFIG_AVM_FASTIRQ) + /* Interrupt Nr 1020, 1021, 1022 und Nr > 1023 fuer NON-SECURE nicht definiert, einfach Beenden ... */ + /* Spurious Interrupts zählen */ + if(!irq_handled) { + __this_cpu_inc(spurious_count); + } +#endif break; + } while (1); } @@ -389,7 +574,11 @@ chained_irq_enter(chip, desc); +#if defined(CONFIG_AVM_FASTIRQ) && !defined(CONFIG_AVM_FASTIRQ_TZ) + status = avm_secmon_gate(0, (CPU_BASE_PHY + GIC_CPU_INTACK), 0); +#else status = readl_relaxed(gic_data_cpu_base(chip_data) + GIC_CPU_INTACK); +#endif gic_irq = (status & GICC_IAR_INT_ID_MASK); if (gic_irq == GICC_INT_SPURIOUS) @@ -589,6 +778,11 @@ readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4); for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) + gic->saved_spi_enable[i] = + gic->saved_spi_security[i] = + readl_relaxed(dist_base + GIC_DIST_IGROUP + i * 4); + + for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) gic->saved_spi_active[i] = readl_relaxed(dist_base + GIC_DIST_ACTIVE_SET + i * 4); } @@ -636,6 +830,10 @@ dist_base + GIC_DIST_ENABLE_SET + i * 4); } + for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) + writel_relaxed(gic->saved_spi_security[i], + dist_base + GIC_DIST_IGROUP + i * 4); + for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++) { writel_relaxed(GICD_INT_EN_CLR_X32, dist_base + GIC_DIST_ACTIVE_CLEAR + i * 4); @@ -670,6 +868,10 @@ for (i = 0; i < DIV_ROUND_UP(32, 32); i++) ptr[i] = readl_relaxed(dist_base + GIC_DIST_ACTIVE_SET + i * 4); + ptr = raw_cpu_ptr(gic->saved_ppi_security); + for (i = 0; i < DIV_ROUND_UP(32, 32); i++) + ptr[i] = readl_relaxed(dist_base + GIC_DIST_IGROUP + i * 4); + ptr = raw_cpu_ptr(gic->saved_ppi_conf); for (i = 0; i < DIV_ROUND_UP(32, 16); i++) ptr[i] = readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4); @@ -706,6 +908,10 @@ writel_relaxed(ptr[i], dist_base + GIC_DIST_ACTIVE_SET + i * 4); } + ptr = raw_cpu_ptr(gic->saved_ppi_security); + for (i = 0; i < DIV_ROUND_UP(32, 32); i++) + writel_relaxed(ptr[i], dist_base + GIC_DIST_IGROUP + i * 4); + ptr = raw_cpu_ptr(gic->saved_ppi_conf); for (i = 0; i < DIV_ROUND_UP(32, 16); i++) writel_relaxed(ptr[i], dist_base + GIC_DIST_CONFIG + i * 4); @@ -765,6 +971,10 @@ if (WARN_ON(!gic->saved_ppi_active)) goto free_ppi_enable; + gic->saved_ppi_security = __alloc_percpu(DIV_ROUND_UP(32, 32) * 4, + sizeof(u32)); + BUG_ON(!gic->saved_ppi_security); + gic->saved_ppi_conf = __alloc_percpu(DIV_ROUND_UP(32, 16) * 4, sizeof(u32)); if (WARN_ON(!gic->saved_ppi_conf)) @@ -815,7 +1025,11 @@ dmb(ishst); /* this always happens on GIC0 */ +#ifdef CONFIG_AVM_FASTIRQ_TZ writel_relaxed(map << 16 | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT); +#else + writel_relaxed(map << 16 | (1 << 15) | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT); +#endif gic_unlock_irqrestore(flags); }