--- zzzz-none-000/linux-3.10.107/arch/arm/include/asm/spinlock.h 2017-06-27 09:49:32.000000000 +0000 +++ scorpion-7490-727/linux-3.10.107/arch/arm/include/asm/spinlock.h 2021-02-04 17:41:59.000000000 +0000 @@ -5,21 +5,48 @@ #error SMP not supported on pre-ARMv6 CPUs #endif -#include +/*--------------------------------------------------------------------------------*\ +\*--------------------------------------------------------------------------------*/ +#if defined(CONFIG_ENABLE_SPINLOCK_PROFILING_HOOKS) +extern int avm_simple_profiling_is_enabled_func(void); +extern unsigned int avm_get_cycles_func(void); + +#define avm_simple_profiling_spin_lock(lock) ({ \ + if(avm_simple_profiling_is_enabled_func()) { \ + unsigned int avm_simple_profiling_spinlock_endtime = avm_get_cycles_func(); \ + __avm_simple_profiling_spinlock(_RET_IP_, (unsigned int)lock, avm_simple_profiling_spinlock_endtime - avm_simple_profiling_spinlock_starttime, 0); \ + } \ +}) + +#define avm_simple_profiling_spin_trylock(lock, success) ({ \ + if(avm_simple_profiling_is_enabled_func()) { \ + __avm_simple_profiling_spinlock(_RET_IP_, (unsigned int)lock, 0, success ? 1 : 2); \ + } \ +}) + +#define avm_simple_profiling_spin_unlock(lock) ({ \ + if(avm_simple_profiling_is_enabled_func()) { \ + __avm_simple_profiling_spinlock(_RET_IP_, (unsigned int)lock, 0, 3); \ + } \ +}) + +#define avm_simple_profiling_spin_gettime() unsigned int avm_simple_profiling_spinlock_starttime = avm_get_cycles_func() + +extern void __avm_simple_profiling_spinlock(unsigned int addr, unsigned int lock, unsigned int time, unsigned int id); + +#else +static inline void avm_simple_profiling_spin_lock(void *lock __attribute__ ((unused))) { }; +static inline void avm_simple_profiling_spin_trylock(void *lock __attribute__ ((unused)), unsigned int success __attribute__ ((unused))) { }; +static inline void avm_simple_profiling_spin_unlock(void *lock __attribute__ ((unused))) { }; +static inline unsigned int avm_simple_profiling_spin_gettime(void) { return 0; } +#endif +#include /* * sev and wfe are ARMv6K extensions. Uniprocessor ARMv6 may not have the K * extensions, so when running on UP, we have to patch these instructions away. */ -#define ALT_SMP(smp, up) \ - "9998: " smp "\n" \ - " .pushsection \".alt.smp.init\", \"a\"\n" \ - " .long 9998b\n" \ - " " up "\n" \ - " .popsection\n" - #ifdef CONFIG_THUMB2_KERNEL -#define SEV ALT_SMP("sev.w", "nop.w") /* * For Thumb-2, special care is needed to ensure that the conditional WFE * instruction really does assemble to exactly 4 bytes (as required by @@ -31,31 +58,23 @@ * the assembler won't change IT instructions which are explicitly present * in the input. */ -#define WFE(cond) ALT_SMP( \ +#define WFE(cond) __ALT_SMP_ASM( \ "it " cond "\n\t" \ "wfe" cond ".n", \ \ "nop.w" \ ) #else -#define SEV ALT_SMP("sev", "nop") -#define WFE(cond) ALT_SMP("wfe" cond, "nop") +#define WFE(cond) __ALT_SMP_ASM("wfe" cond, "nop") #endif +#define SEV __ALT_SMP_ASM(WASM(sev), WASM(nop)) + static inline void dsb_sev(void) { -#if __LINUX_ARM_ARCH__ >= 7 - __asm__ __volatile__ ( - "dsb\n" - SEV - ); -#else - __asm__ __volatile__ ( - "mcr p15, 0, %0, c7, c10, 4\n" - SEV - : : "r" (0) - ); -#endif + + dsb(ishst); + __asm__(SEV); } /* @@ -77,6 +96,8 @@ u32 newval; arch_spinlock_t lockval; + avm_simple_profiling_spin_gettime(); + prefetchw(&lock->slock); __asm__ __volatile__( "1: ldrex %0, [%3]\n" " add %1, %0, %4\n" @@ -93,6 +114,7 @@ } smp_mb(); + avm_simple_profiling_spin_lock(lock); } static inline int arch_spin_trylock(arch_spinlock_t *lock) @@ -100,6 +122,7 @@ unsigned long contended, res; u32 slock; + prefetchw(&lock->slock); do { __asm__ __volatile__( " ldrex %0, [%3]\n" @@ -111,7 +134,7 @@ : "r" (&lock->slock), "I" (1 << TICKET_SHIFT) : "cc"); } while (res); - + avm_simple_profiling_spin_trylock(lock, contended); if (!contended) { smp_mb(); return 1; @@ -123,19 +146,25 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) { smp_mb(); + + avm_simple_profiling_spin_unlock(lock); lock->tickets.owner++; dsb_sev(); } +static inline int arch_spin_value_unlocked(arch_spinlock_t lock) +{ + return lock.tickets.owner == lock.tickets.next; +} + static inline int arch_spin_is_locked(arch_spinlock_t *lock) { - struct __raw_tickets tickets = ACCESS_ONCE(lock->tickets); - return tickets.owner != tickets.next; + return !arch_spin_value_unlocked(READ_ONCE(*lock)); } static inline int arch_spin_is_contended(arch_spinlock_t *lock) { - struct __raw_tickets tickets = ACCESS_ONCE(lock->tickets); + struct __raw_tickets tickets = READ_ONCE(lock->tickets); return (tickets.next - tickets.owner) > 1; } #define arch_spin_is_contended arch_spin_is_contended @@ -152,6 +181,7 @@ { unsigned long tmp; + prefetchw(&rw->lock); __asm__ __volatile__( "1: ldrex %0, [%1]\n" " teq %0, #0\n" @@ -170,6 +200,7 @@ { unsigned long contended, res; + prefetchw(&rw->lock); do { __asm__ __volatile__( " ldrex %0, [%2]\n" @@ -203,7 +234,7 @@ } /* write_can_lock - would write_trylock() succeed? */ -#define arch_write_can_lock(x) ((x)->lock == 0) +#define arch_write_can_lock(x) (ACCESS_ONCE((x)->lock) == 0) /* * Read locks are a bit more hairy: @@ -221,6 +252,7 @@ { unsigned long tmp, tmp2; + prefetchw(&rw->lock); __asm__ __volatile__( "1: ldrex %0, [%2]\n" " adds %0, %0, #1\n" @@ -241,6 +273,7 @@ smp_mb(); + prefetchw(&rw->lock); __asm__ __volatile__( "1: ldrex %0, [%2]\n" " sub %0, %0, #1\n" @@ -259,6 +292,7 @@ { unsigned long contended, res; + prefetchw(&rw->lock); do { __asm__ __volatile__( " ldrex %0, [%2]\n" @@ -280,7 +314,7 @@ } /* read_can_lock - would read_trylock() succeed? */ -#define arch_read_can_lock(x) ((x)->lock < 0x80000000) +#define arch_read_can_lock(x) (ACCESS_ONCE((x)->lock) < 0x80000000) #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)