--- zzzz-none-000/linux-3.10.107/kernel/softirq.c 2017-06-27 09:49:32.000000000 +0000 +++ scorpion-7490-727/linux-3.10.107/kernel/softirq.c 2021-02-04 17:41:59.000000000 +0000 @@ -8,6 +8,8 @@ * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903) */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include #include #include @@ -23,11 +25,20 @@ #include #include #include +#include +#include + + +#ifdef CONFIG_STOPWATCH_SOFT_IRQ +#define __STOPWATCH_USE__ +#endif +#include +#include +#include #define CREATE_TRACE_POINTS #include -#include /* - No shared variables, all the data are CPU local. - If a softirq needs serialization, let it serialize itself @@ -51,11 +62,13 @@ EXPORT_SYMBOL(irq_stat); #endif +DEFINE_STOPWATCH_ARRAY(softirq, NR_SOFTIRQS); + static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp; DEFINE_PER_CPU(struct task_struct *, ksoftirqd); -char *softirq_to_name[NR_SOFTIRQS] = { +const char * const softirq_to_name[NR_SOFTIRQS] = { "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL", "TASKLET", "SCHED", "HRTIMER", "RCU" }; @@ -90,7 +103,7 @@ * where hardirqs are disabled legitimately: */ #ifdef CONFIG_TRACE_IRQFLAGS -static void __local_bh_disable(unsigned long ip, unsigned int cnt) +void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) { unsigned long flags; @@ -98,47 +111,37 @@ raw_local_irq_save(flags); /* - * The preempt tracer hooks into add_preempt_count and will break + * The preempt tracer hooks into preempt_count_add and will break * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET * is set and before current->softirq_enabled is cleared. * We must manually increment preempt_count here and manually * call the trace_preempt_off later. */ - preempt_count() += cnt; + __preempt_count_add(cnt); /* * Were softirqs turned off above: */ - if (softirq_count() == cnt) + if (softirq_count() == (cnt & SOFTIRQ_MASK)) trace_softirqs_off(ip); raw_local_irq_restore(flags); - if (preempt_count() == cnt) + if (preempt_count() == cnt) { +#ifdef CONFIG_DEBUG_PREEMPT + current->preempt_disable_ip = get_parent_ip(CALLER_ADDR1); +#endif trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); + } } -#else /* !CONFIG_TRACE_IRQFLAGS */ -static inline void __local_bh_disable(unsigned long ip, unsigned int cnt) -{ - add_preempt_count(cnt); - barrier(); -} +EXPORT_SYMBOL(__local_bh_disable_ip); #endif /* CONFIG_TRACE_IRQFLAGS */ -void local_bh_disable(void) -{ - __local_bh_disable((unsigned long)__builtin_return_address(0), - SOFTIRQ_DISABLE_OFFSET); -} - -EXPORT_SYMBOL(local_bh_disable); - static void __local_bh_enable(unsigned int cnt) { - WARN_ON_ONCE(in_irq()); WARN_ON_ONCE(!irqs_disabled()); - if (softirq_count() == cnt) - trace_softirqs_on((unsigned long)__builtin_return_address(0)); - sub_preempt_count(cnt); + if (softirq_count() == (cnt & SOFTIRQ_MASK)) + trace_softirqs_on(_RET_IP_); + preempt_count_sub(cnt); } /* @@ -148,12 +151,12 @@ */ void _local_bh_enable(void) { + WARN_ON_ONCE(in_irq()); __local_bh_enable(SOFTIRQ_DISABLE_OFFSET); } - EXPORT_SYMBOL(_local_bh_enable); -static inline void _local_bh_enable_ip(unsigned long ip) +void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) { WARN_ON_ONCE(in_irq() || irqs_disabled()); #ifdef CONFIG_TRACE_IRQFLAGS @@ -167,30 +170,55 @@ /* * Keep preemption disabled until we are done with * softirq processing: - */ - sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1); + */ + preempt_count_sub(cnt - 1); - if (unlikely(!in_interrupt() && local_softirq_pending())) + if (unlikely(!in_interrupt() && local_softirq_pending())) { + /* + * Run softirq if any pending. And do it in its own stack + * as we may be calling this deep in a task call stack already. + */ do_softirq(); + } - dec_preempt_count(); + preempt_count_dec(); #ifdef CONFIG_TRACE_IRQFLAGS local_irq_enable(); #endif preempt_check_resched(); } +EXPORT_SYMBOL(__local_bh_enable_ip); -void local_bh_enable(void) +#ifdef __STOPWATCH_USE__ +int stopwatch_softirq_show(struct seq_file *f, void *v) { - _local_bh_enable_ip((unsigned long)__builtin_return_address(0)); + int cpu; + int sirq = *((loff_t *) v); + + if (sirq == 0) + seq_printf(f, "%20s\tCPU\tmin(us)\tavg(us)\tmax(us)\n\n", ""); + + seq_printf(f, "%2d:%-18s", sirq, softirq_to_name[sirq]); + for_each_cpu(cpu, cpu_online_mask) { + if (!cpu) + seq_printf(f, "\t%d", cpu); + else + seq_printf(f, "%20s\t%d", "", cpu); + stopwatch_show(&STOPWATCH_INSTANCE_CPU(softirq[sirq], cpu), f, STOPWATCH_MICRO); + seq_printf(f, "\n"); + } + seq_printf(f, "\n"); + return 0; } -EXPORT_SYMBOL(local_bh_enable); -void local_bh_enable_ip(unsigned long ip) +static int __init softirq_stopwatch_init(void) { - _local_bh_enable_ip(ip); + INIT_STOPWATCH_ARRAY(softirq, NR_SOFTIRQS); + return stopwatch_register("softirq", NR_SOFTIRQS, stopwatch_softirq_show); } -EXPORT_SYMBOL(local_bh_enable_ip); +module_init(softirq_stopwatch_init) +#endif + /* * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times, @@ -208,14 +236,48 @@ #define MAX_SOFTIRQ_TIME msecs_to_jiffies(2) #define MAX_SOFTIRQ_RESTART 10 -asmlinkage void __do_softirq(void) +#ifdef CONFIG_TRACE_IRQFLAGS +/* + * When we run softirqs from irq_exit() and thus on the hardirq stack we need + * to keep the lockdep irq context tracking as tight as possible in order to + * not miss-qualify lock contexts and miss possible deadlocks. + */ + +static inline bool lockdep_softirq_start(void) +{ + bool in_hardirq = false; + + if (trace_hardirq_context(current)) { + in_hardirq = true; + trace_hardirq_exit(); + } + + lockdep_softirq_enter(); + + return in_hardirq; +} + +static inline void lockdep_softirq_end(bool in_hardirq) +{ + lockdep_softirq_exit(); + + if (in_hardirq) + trace_hardirq_enter(); +} +#else +static inline bool lockdep_softirq_start(void) { return false; } +static inline void lockdep_softirq_end(bool in_hardirq) { } +#endif + +asmlinkage __visible void __do_softirq(void) { - struct softirq_action *h; - __u32 pending; unsigned long end = jiffies + MAX_SOFTIRQ_TIME; - int cpu; unsigned long old_flags = current->flags; int max_restart = MAX_SOFTIRQ_RESTART; + struct softirq_action *h; + bool in_hardirq; + __u32 pending; + int softirq_bit; /* * Mask out PF_MEMALLOC s current task context is borrowed for the @@ -227,11 +289,10 @@ pending = local_softirq_pending(); account_irq_enter_time(current); - __local_bh_disable((unsigned long)__builtin_return_address(0), - SOFTIRQ_OFFSET); - lockdep_softirq_enter(); + __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET); + in_hardirq = lockdep_softirq_start(); + avm_sum_softirq_enter(); - cpu = smp_processor_id(); restart: /* Reset the pending bitmask before enabling irqs */ set_softirq_pending(0); @@ -240,31 +301,40 @@ h = softirq_vec; - do { - if (pending & 1) { - unsigned int vec_nr = h - softirq_vec; - int prev_count = preempt_count(); - - kstat_incr_softirqs_this_cpu(vec_nr); - - trace_softirq_entry(vec_nr); - h->action(h); - trace_softirq_exit(vec_nr); - if (unlikely(prev_count != preempt_count())) { - printk(KERN_ERR "huh, entered softirq %u %s %p" - "with preempt_count %08x," - " exited with %08x?\n", vec_nr, - softirq_to_name[vec_nr], h->action, - prev_count, preempt_count()); - preempt_count() = prev_count; - } - - rcu_bh_qs(cpu); + while ((softirq_bit = ffs(pending))) { + struct _runtime_stat *pts; + unsigned int vec_nr; + int prev_count; + + h += softirq_bit - 1; + + vec_nr = h - softirq_vec; + prev_count = preempt_count(); + + kstat_incr_softirqs_this_cpu(vec_nr); + + avm_simple_profiling_log(avm_profile_data_type_sw_irq_begin, + (unsigned long)(h->action), (unsigned int)h); + trace_softirq_entry(vec_nr); + pts = avm_softirq_enter(vec_nr); + STOPWATCH_START(softirq[vec_nr]); + h->action(h); + STOPWATCH_STOP(softirq[vec_nr]); + avm_softirq_leave(pts); + trace_softirq_exit(vec_nr); + avm_simple_profiling_log(avm_profile_data_type_sw_irq_end, + (unsigned long)(h->action), (unsigned int)h); + if (unlikely(prev_count != preempt_count())) { + pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", + vec_nr, softirq_to_name[vec_nr], h->action, + prev_count, preempt_count()); + preempt_count_set(prev_count); } h++; - pending >>= 1; - } while (pending); + pending >>= softirq_bit; + } + rcu_bh_qs(); local_irq_disable(); pending = local_softirq_pending(); @@ -276,16 +346,15 @@ wakeup_softirqd(); } - lockdep_softirq_exit(); - + avm_sum_softirq_leave(); + lockdep_softirq_end(in_hardirq); account_irq_exit_time(current); __local_bh_enable(SOFTIRQ_OFFSET); + WARN_ON_ONCE(in_interrupt()); tsk_restore_flags(current, old_flags, PF_MEMALLOC); } -#ifndef __ARCH_HAS_DO_SOFTIRQ - -asmlinkage void do_softirq(void) +asmlinkage __visible void do_softirq(void) { __u32 pending; unsigned long flags; @@ -298,20 +367,16 @@ pending = local_softirq_pending(); if (pending) - __do_softirq(); + do_softirq_own_stack(); local_irq_restore(flags); } -#endif - /* * Enter an interrupt context. */ void irq_enter(void) { - int cpu = smp_processor_id(); - rcu_irq_enter(); if (is_idle_task(current) && !in_interrupt()) { /* @@ -319,7 +384,7 @@ * here, as softirq will be serviced on return from interrupt. */ local_bh_disable(); - tick_check_idle(cpu); + tick_irq_enter(); _local_bh_enable(); } @@ -329,15 +394,21 @@ static inline void invoke_softirq(void) { if (!force_irqthreads) { +#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK /* * We can safely execute softirq on the current stack if * it is the irq stack, because it should be near empty - * at this stage. But we have no way to know if the arch - * calls irq_exit() on the irq stack. So call softirq - * in its own stack to prevent from any overrun on top - * of a potentially deep task stack. + * at this stage. */ - do_softirq(); + __do_softirq(); +#else + /* + * Otherwise, irq_exit() is called on the task stack that can + * be potentially deep already. So call softirq in its own stack + * to prevent from any overrun. + */ + do_softirq_own_stack(); +#endif } else { wakeup_softirqd(); } @@ -368,13 +439,13 @@ #endif account_irq_exit_time(current); - trace_hardirq_exit(); - sub_preempt_count(HARDIRQ_OFFSET); + preempt_count_sub(HARDIRQ_OFFSET); if (!in_interrupt() && local_softirq_pending()) invoke_softirq(); tick_irq_exit(); rcu_irq_exit(); + trace_hardirq_exit(); /* must be last! */ } /* @@ -420,8 +491,7 @@ /* * Tasklets */ -struct tasklet_head -{ +struct tasklet_head { struct tasklet_struct *head; struct tasklet_struct **tail; }; @@ -437,10 +507,11 @@ t->next = NULL; *__this_cpu_read(tasklet_vec.tail) = t; __this_cpu_write(tasklet_vec.tail, &(t->next)); + avm_simple_profiling_log(avm_profile_data_type_trigger_tasklet_begin, + (unsigned long)(t->func), TASKLET_SOFTIRQ); raise_softirq_irqoff(TASKLET_SOFTIRQ); local_irq_restore(flags); } - EXPORT_SYMBOL(__tasklet_schedule); void __tasklet_hi_schedule(struct tasklet_struct *t) @@ -451,10 +522,11 @@ t->next = NULL; *__this_cpu_read(tasklet_hi_vec.tail) = t; __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); + avm_simple_profiling_log(avm_profile_data_type_trigger_tasklet_begin, + (unsigned long)(t->func), HI_SOFTIRQ); raise_softirq_irqoff(HI_SOFTIRQ); local_irq_restore(flags); } - EXPORT_SYMBOL(__tasklet_hi_schedule); void __tasklet_hi_schedule_first(struct tasklet_struct *t) @@ -465,7 +537,6 @@ __this_cpu_write(tasklet_hi_vec.head, t); __raise_softirq_irqoff(HI_SOFTIRQ); } - EXPORT_SYMBOL(__tasklet_hi_schedule_first); static void tasklet_action(struct softirq_action *a) @@ -475,19 +546,32 @@ local_irq_disable(); list = __this_cpu_read(tasklet_vec.head); __this_cpu_write(tasklet_vec.head, NULL); - __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head); + __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head)); local_irq_enable(); while (list) { + struct _runtime_stat *pfunc_stat; struct tasklet_struct *t = list; list = list->next; if (tasklet_trylock(t)) { if (!atomic_read(&t->count)) { - if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) + if (!test_and_clear_bit(TASKLET_STATE_SCHED, + &t->state)) BUG(); + trace_tlet_entry(t); + avm_simple_profiling_log(avm_profile_data_type_trigger_tasklet_end, + (unsigned long)(t->func), TASKLET_SOFTIRQ); + avm_simple_profiling_log(avm_profile_data_type_tasklet_begin, + (unsigned long)(t->func), (unsigned int)(t->data)); + pfunc_stat = avm_taskletfunc_enter(t->func, TASKLET_SOFTIRQ); t->func(t->data); + avm_taskletfunc_leave(pfunc_stat); + avm_simple_profiling_log(avm_profile_data_type_tasklet_end, + (unsigned long)(t->func), (unsigned int)(t->data)); + + trace_tlet_exit(t); tasklet_unlock(t); continue; } @@ -510,19 +594,31 @@ local_irq_disable(); list = __this_cpu_read(tasklet_hi_vec.head); __this_cpu_write(tasklet_hi_vec.head, NULL); - __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head); + __this_cpu_write(tasklet_hi_vec.tail, this_cpu_ptr(&tasklet_hi_vec.head)); local_irq_enable(); while (list) { + struct _runtime_stat *pfunc_stat; struct tasklet_struct *t = list; list = list->next; if (tasklet_trylock(t)) { if (!atomic_read(&t->count)) { - if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) + if (!test_and_clear_bit(TASKLET_STATE_SCHED, + &t->state)) BUG(); + trace_tlet_entry(t); + avm_simple_profiling_log(avm_profile_data_type_trigger_tasklet_end, + (unsigned long)(t->func), HI_SOFTIRQ); + avm_simple_profiling_log(avm_profile_data_type_hi_tasklet_begin, + (unsigned long)(t->func), (unsigned int)(t->data)); + pfunc_stat = avm_taskletfunc_enter(t->func, HI_SOFTIRQ); t->func(t->data); + avm_taskletfunc_leave(pfunc_stat); + avm_simple_profiling_log(avm_profile_data_type_hi_tasklet_end, + (unsigned long)(t->func), (unsigned int)(t->data)); + trace_tlet_exit(t); tasklet_unlock(t); continue; } @@ -538,7 +634,6 @@ } } - void tasklet_init(struct tasklet_struct *t, void (*func)(unsigned long), unsigned long data) { @@ -548,13 +643,12 @@ t->func = func; t->data = data; } - EXPORT_SYMBOL(tasklet_init); void tasklet_kill(struct tasklet_struct *t) { if (in_interrupt()) - printk("Attempt to kill tasklet from interrupt\n"); + pr_notice("Attempt to kill tasklet from interrupt\n"); while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { do { @@ -564,7 +658,6 @@ tasklet_unlock_wait(t); clear_bit(TASKLET_STATE_SCHED, &t->state); } - EXPORT_SYMBOL(tasklet_kill); /* @@ -642,14 +735,13 @@ { local_irq_disable(); if (local_softirq_pending()) { + /* + * We can safely run softirq on inline stack, as we are not deep + * in the task stack here. + */ __do_softirq(); local_irq_enable(); - cond_resched(); - - preempt_disable(); - rcu_note_context_switch(cpu); - preempt_enable(); - + cond_resched_rcu_qs(); return; } local_irq_enable(); @@ -714,9 +806,8 @@ } #endif /* CONFIG_HOTPLUG_CPU */ -static int __cpuinit cpu_callback(struct notifier_block *nfb, - unsigned long action, - void *hcpu) +static int cpu_callback(struct notifier_block *nfb, unsigned long action, + void *hcpu) { switch (action) { #ifdef CONFIG_HOTPLUG_CPU @@ -729,7 +820,7 @@ return NOTIFY_OK; } -static struct notifier_block __cpuinitdata cpu_nfb = { +static struct notifier_block cpu_nfb = { .notifier_call = cpu_callback }; @@ -760,7 +851,6 @@ return 0; } -#ifdef CONFIG_GENERIC_HARDIRQS int __init __weak arch_probe_nr_irqs(void) { return NR_IRQS_LEGACY; @@ -770,4 +860,8 @@ { return 0; } -#endif + +unsigned int __weak arch_dynirq_lower_bound(unsigned int from) +{ + return from; +}