--- zzzz-none-000/linux-3.10.107/drivers/idle/intel_idle.c 2017-06-27 09:49:32.000000000 +0000 +++ scorpion-7490-727/linux-3.10.107/drivers/idle/intel_idle.c 2021-02-04 17:41:59.000000000 +0000 @@ -1,7 +1,7 @@ /* * intel_idle.c - native hardware idle loop for modern Intel processors * - * Copyright (c) 2010, Intel Corporation. + * Copyright (c) 2013, Intel Corporation. * Len Brown * * This program is free software; you can redistribute it and/or modify it @@ -55,7 +55,7 @@ #include #include -#include +#include #include #include #include @@ -65,7 +65,7 @@ #include #include -#define INTEL_IDLE_VERSION "0.4" +#define INTEL_IDLE_VERSION "0.4.1" #define PREFIX "intel_idle: " static struct cpuidle_driver intel_idle_driver = { @@ -89,6 +89,7 @@ * Indicate which enable bits to clear here. */ unsigned long auto_demotion_disable_flags; + bool byt_auto_demotion_disable_flag; bool disable_promotion_to_c1e; }; @@ -96,6 +97,8 @@ static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; static int intel_idle(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index); +static void intel_idle_freeze(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int index); static int intel_idle_cpu_init(int cpu); static struct cpuidle_state *cpuidle_state_table; @@ -123,209 +126,613 @@ * which is also the index into the MWAIT hint array. * Thus C0 is a dummy. */ -static struct cpuidle_state nehalem_cstates[CPUIDLE_STATE_MAX] = { +static struct cpuidle_state nehalem_cstates[] = { { .name = "C1-NHM", .desc = "MWAIT 0x00", - .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, + .flags = MWAIT2flg(0x00), .exit_latency = 3, .target_residency = 6, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C1E-NHM", .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, + .flags = MWAIT2flg(0x01), .exit_latency = 10, .target_residency = 20, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C3-NHM", .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 20, .target_residency = 80, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C6-NHM", .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 200, .target_residency = 800, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .enter = NULL } }; -static struct cpuidle_state snb_cstates[CPUIDLE_STATE_MAX] = { +static struct cpuidle_state snb_cstates[] = { { .name = "C1-SNB", .desc = "MWAIT 0x00", - .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, + .flags = MWAIT2flg(0x00), .exit_latency = 2, .target_residency = 2, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C1E-SNB", .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, + .flags = MWAIT2flg(0x01), .exit_latency = 10, .target_residency = 20, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C3-SNB", .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 80, .target_residency = 211, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C6-SNB", .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 104, .target_residency = 345, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C7-SNB", .desc = "MWAIT 0x30", - .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 109, .target_residency = 345, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .enter = NULL } }; -static struct cpuidle_state ivb_cstates[CPUIDLE_STATE_MAX] = { +static struct cpuidle_state byt_cstates[] = { + { + .name = "C1-BYT", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6N-BYT", + .desc = "MWAIT 0x58", + .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 300, + .target_residency = 275, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6S-BYT", + .desc = "MWAIT 0x52", + .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 500, + .target_residency = 560, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C7-BYT", + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 1200, + .target_residency = 4000, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C7S-BYT", + .desc = "MWAIT 0x64", + .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 10000, + .target_residency = 20000, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .enter = NULL } +}; + +static struct cpuidle_state cht_cstates[] = { + { + .name = "C1-CHT", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6N-CHT", + .desc = "MWAIT 0x58", + .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 80, + .target_residency = 275, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6S-CHT", + .desc = "MWAIT 0x52", + .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 200, + .target_residency = 560, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C7-CHT", + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 1200, + .target_residency = 4000, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C7S-CHT", + .desc = "MWAIT 0x64", + .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 10000, + .target_residency = 20000, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .enter = NULL } +}; + +static struct cpuidle_state ivb_cstates[] = { { .name = "C1-IVB", .desc = "MWAIT 0x00", - .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, + .flags = MWAIT2flg(0x00), .exit_latency = 1, .target_residency = 1, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C1E-IVB", .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, + .flags = MWAIT2flg(0x01), .exit_latency = 10, .target_residency = 20, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C3-IVB", .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 59, .target_residency = 156, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C6-IVB", .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 80, .target_residency = 300, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C7-IVB", .desc = "MWAIT 0x30", - .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 87, .target_residency = 300, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .enter = NULL } +}; + +static struct cpuidle_state ivt_cstates[] = { + { + .name = "C1-IVT", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C1E-IVT", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01), + .exit_latency = 10, + .target_residency = 80, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C3-IVT", + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 59, + .target_residency = 156, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6-IVT", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 82, + .target_residency = 300, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .enter = NULL } +}; + +static struct cpuidle_state ivt_cstates_4s[] = { + { + .name = "C1-IVT-4S", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C1E-IVT-4S", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01), + .exit_latency = 10, + .target_residency = 250, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C3-IVT-4S", + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 59, + .target_residency = 300, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6-IVT-4S", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 84, + .target_residency = 400, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .enter = NULL } +}; + +static struct cpuidle_state ivt_cstates_8s[] = { + { + .name = "C1-IVT-8S", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C1E-IVT-8S", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01), + .exit_latency = 10, + .target_residency = 500, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C3-IVT-8S", + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 59, + .target_residency = 600, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6-IVT-8S", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 88, + .target_residency = 700, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .enter = NULL } }; -static struct cpuidle_state hsw_cstates[CPUIDLE_STATE_MAX] = { +static struct cpuidle_state hsw_cstates[] = { { .name = "C1-HSW", .desc = "MWAIT 0x00", - .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, + .flags = MWAIT2flg(0x00), .exit_latency = 2, .target_residency = 2, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C1E-HSW", .desc = "MWAIT 0x01", - .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, + .flags = MWAIT2flg(0x01), .exit_latency = 10, .target_residency = 20, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C3-HSW", .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 33, .target_residency = 100, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C6-HSW", .desc = "MWAIT 0x20", - .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 133, .target_residency = 400, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C7s-HSW", .desc = "MWAIT 0x32", - .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 166, .target_residency = 500, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C8-HSW", .desc = "MWAIT 0x40", - .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 300, .target_residency = 900, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C9-HSW", .desc = "MWAIT 0x50", - .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 600, .target_residency = 1800, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C10-HSW", .desc = "MWAIT 0x60", - .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 2600, .target_residency = 7700, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .enter = NULL } +}; +static struct cpuidle_state bdw_cstates[] = { + { + .name = "C1-BDW", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 2, + .target_residency = 2, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C1E-BDW", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01), + .exit_latency = 10, + .target_residency = 20, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C3-BDW", + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 40, + .target_residency = 100, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6-BDW", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 133, + .target_residency = 400, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C7s-BDW", + .desc = "MWAIT 0x32", + .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 166, + .target_residency = 500, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C8-BDW", + .desc = "MWAIT 0x40", + .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 300, + .target_residency = 900, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C9-BDW", + .desc = "MWAIT 0x50", + .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 600, + .target_residency = 1800, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C10-BDW", + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 2600, + .target_residency = 7700, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .enter = NULL } +}; + +static struct cpuidle_state skl_cstates[] = { + { + .name = "C1-SKL", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 2, + .target_residency = 2, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C1E-SKL", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01), + .exit_latency = 10, + .target_residency = 20, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C3-SKL", + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 70, + .target_residency = 100, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6-SKL", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 85, + .target_residency = 200, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C7s-SKL", + .desc = "MWAIT 0x33", + .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 124, + .target_residency = 800, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C8-SKL", + .desc = "MWAIT 0x40", + .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 200, + .target_residency = 800, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C9-SKL", + .desc = "MWAIT 0x50", + .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 480, + .target_residency = 5000, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C10-SKL", + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 890, + .target_residency = 5000, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .enter = NULL } }; -static struct cpuidle_state atom_cstates[CPUIDLE_STATE_MAX] = { +static struct cpuidle_state atom_cstates[] = { { .name = "C1E-ATM", .desc = "MWAIT 0x00", - .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, + .flags = MWAIT2flg(0x00), .exit_latency = 10, .target_residency = 20, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C2-ATM", .desc = "MWAIT 0x10", - .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID, + .flags = MWAIT2flg(0x10), .exit_latency = 20, .target_residency = 80, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C4-ATM", .desc = "MWAIT 0x30", - .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 100, .target_residency = 400, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, { .name = "C6-ATM", .desc = "MWAIT 0x52", - .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, + .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, .exit_latency = 140, .target_residency = 560, - .enter = &intel_idle }, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .enter = NULL } +}; +static struct cpuidle_state avn_cstates[] = { + { + .name = "C1-AVN", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 2, + .target_residency = 2, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6-AVN", + .desc = "MWAIT 0x51", + .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 15, + .target_residency = 45, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .enter = NULL } +}; +static struct cpuidle_state knl_cstates[] = { + { + .name = "C1-KNL", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 1, + .target_residency = 2, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze }, + { + .name = "C6-KNL", + .desc = "MWAIT 0x10", + .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 120, + .target_residency = 500, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze }, { .enter = NULL } }; @@ -357,34 +764,39 @@ leave_mm(cpu); if (!(lapic_timer_reliable_states & (1 << (cstate)))) - clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu); - - if (!current_set_polling_and_test()) { + tick_broadcast_enter(); - if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR)) - clflush((void *)¤t_thread_info()->flags); - - __monitor((void *)¤t_thread_info()->flags, 0, 0); - smp_mb(); - if (!need_resched()) - __mwait(eax, ecx); - } + mwait_idle_with_hints(eax, ecx); if (!(lapic_timer_reliable_states & (1 << (cstate)))) - clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu); + tick_broadcast_exit(); return index; } -static void __setup_broadcast_timer(void *arg) +/** + * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle + * @dev: cpuidle_device + * @drv: cpuidle driver + * @index: state index + */ +static void intel_idle_freeze(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int index) { - unsigned long reason = (unsigned long)arg; - int cpu = smp_processor_id(); + unsigned long ecx = 1; /* break on interrupt flag */ + unsigned long eax = flg2MWAIT(drv->states[index].flags); - reason = reason ? - CLOCK_EVT_NOTIFY_BROADCAST_ON : CLOCK_EVT_NOTIFY_BROADCAST_OFF; + mwait_idle_with_hints(eax, ecx); +} + +static void __setup_broadcast_timer(void *arg) +{ + unsigned long on = (unsigned long)arg; - clockevents_notify(reason, &cpu); + if (on) + tick_broadcast_enable(); + else + tick_broadcast_disable(); } static int cpu_hotplug_notify(struct notifier_block *n, @@ -393,7 +805,7 @@ int hotcpu = (unsigned long)hcpu; struct cpuidle_device *dev; - switch (action & 0xf) { + switch (action & ~CPU_TASKS_FROZEN) { case CPU_ONLINE: if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) @@ -455,20 +867,57 @@ .disable_promotion_to_c1e = true, }; +static const struct idle_cpu idle_cpu_byt = { + .state_table = byt_cstates, + .disable_promotion_to_c1e = true, + .byt_auto_demotion_disable_flag = true, +}; + +static const struct idle_cpu idle_cpu_cht = { + .state_table = cht_cstates, + .disable_promotion_to_c1e = true, + .byt_auto_demotion_disable_flag = true, +}; + static const struct idle_cpu idle_cpu_ivb = { .state_table = ivb_cstates, .disable_promotion_to_c1e = true, }; +static const struct idle_cpu idle_cpu_ivt = { + .state_table = ivt_cstates, + .disable_promotion_to_c1e = true, +}; + static const struct idle_cpu idle_cpu_hsw = { .state_table = hsw_cstates, .disable_promotion_to_c1e = true, }; +static const struct idle_cpu idle_cpu_bdw = { + .state_table = bdw_cstates, + .disable_promotion_to_c1e = true, +}; + +static const struct idle_cpu idle_cpu_skl = { + .state_table = skl_cstates, + .disable_promotion_to_c1e = true, +}; + + +static const struct idle_cpu idle_cpu_avn = { + .state_table = avn_cstates, + .disable_promotion_to_c1e = true, +}; + +static const struct idle_cpu idle_cpu_knl = { + .state_table = knl_cstates, +}; + #define ICPU(model, cpu) \ { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu } -static const struct x86_cpu_id intel_idle_ids[] = { +static const struct x86_cpu_id intel_idle_ids[] __initconst = { ICPU(0x1a, idle_cpu_nehalem), ICPU(0x1e, idle_cpu_nehalem), ICPU(0x1f, idle_cpu_nehalem), @@ -480,12 +929,23 @@ ICPU(0x2f, idle_cpu_nehalem), ICPU(0x2a, idle_cpu_snb), ICPU(0x2d, idle_cpu_snb), + ICPU(0x36, idle_cpu_atom), + ICPU(0x37, idle_cpu_byt), + ICPU(0x4c, idle_cpu_cht), ICPU(0x3a, idle_cpu_ivb), - ICPU(0x3e, idle_cpu_ivb), + ICPU(0x3e, idle_cpu_ivt), ICPU(0x3c, idle_cpu_hsw), ICPU(0x3f, idle_cpu_hsw), ICPU(0x45, idle_cpu_hsw), ICPU(0x46, idle_cpu_hsw), + ICPU(0x4d, idle_cpu_avn), + ICPU(0x3d, idle_cpu_bdw), + ICPU(0x47, idle_cpu_bdw), + ICPU(0x4f, idle_cpu_bdw), + ICPU(0x56, idle_cpu_bdw), + ICPU(0x4e, idle_cpu_skl), + ICPU(0x5e, idle_cpu_skl), + ICPU(0x57, idle_cpu_knl), {} }; MODULE_DEVICE_TABLE(x86cpu, intel_idle_ids); @@ -493,7 +953,7 @@ /* * intel_idle_probe() */ -static int intel_idle_probe(void) +static int __init intel_idle_probe(void) { unsigned int eax, ebx, ecx; const struct x86_cpu_id *id; @@ -557,21 +1017,114 @@ free_percpu(intel_idle_cpuidle_devices); return; } + +/* + * ivt_idle_state_table_update(void) + * + * Tune IVT multi-socket targets + * Assumption: num_sockets == (max_package_num + 1) + */ +static void ivt_idle_state_table_update(void) +{ + /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ + int cpu, package_num, num_sockets = 1; + + for_each_online_cpu(cpu) { + package_num = topology_physical_package_id(cpu); + if (package_num + 1 > num_sockets) { + num_sockets = package_num + 1; + + if (num_sockets > 4) { + cpuidle_state_table = ivt_cstates_8s; + return; + } + } + } + + if (num_sockets > 2) + cpuidle_state_table = ivt_cstates_4s; + + /* else, 1 and 2 socket systems use default ivt_cstates */ +} +/* + * sklh_idle_state_table_update(void) + * + * On SKL-H (model 0x5e) disable C8 and C9 if: + * C10 is enabled and SGX disabled + */ +static void sklh_idle_state_table_update(void) +{ + unsigned long long msr; + unsigned int eax, ebx, ecx, edx; + + + /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ + if (max_cstate <= 7) + return; + + /* if PC10 not present in CPUID.MWAIT.EDX */ + if ((mwait_substates & (0xF << 28)) == 0) + return; + + rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr); + + /* PC10 is not enabled in PKG C-state limit */ + if ((msr & 0xF) != 8) + return; + + ecx = 0; + cpuid(7, &eax, &ebx, &ecx, &edx); + + /* if SGX is present */ + if (ebx & (1 << 2)) { + + rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); + + /* if SGX is enabled */ + if (msr & (1 << 18)) + return; + } + + skl_cstates[5].disabled = 1; /* C8-SKL */ + skl_cstates[6].disabled = 1; /* C9-SKL */ +} +/* + * intel_idle_state_table_update() + * + * Update the default state_table for this CPU-id + */ + +static void intel_idle_state_table_update(void) +{ + switch (boot_cpu_data.x86_model) { + + case 0x3e: /* IVT */ + ivt_idle_state_table_update(); + break; + case 0x5e: /* SKL-H */ + sklh_idle_state_table_update(); + break; + } +} + /* * intel_idle_cpuidle_driver_init() * allocate, initialize cpuidle_states */ -static int intel_idle_cpuidle_driver_init(void) +static int __init intel_idle_cpuidle_driver_init(void) { int cstate; struct cpuidle_driver *drv = &intel_idle_driver; + intel_idle_state_table_update(); + drv->state_count = 1; for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { - int num_substates, mwait_hint, mwait_cstate, mwait_substate; + int num_substates, mwait_hint, mwait_cstate; - if (cpuidle_state_table[cstate].enter == NULL) + if ((cpuidle_state_table[cstate].enter == NULL) && + (cpuidle_state_table[cstate].enter_freeze == NULL)) break; if (cstate + 1 > max_cstate) { @@ -582,16 +1135,23 @@ mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); - mwait_substate = MWAIT_HINT2SUBSTATE(mwait_hint); - /* does the state exist in CPUID.MWAIT? */ + /* number of sub-states for this state in CPUID.MWAIT */ num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) & MWAIT_SUBSTATE_MASK; - /* if sub-state in table is not enumerated by CPUID */ - if ((mwait_substate + 1) > num_substates) + /* if NO sub-states for this state in CPUID, skip it */ + if (num_substates == 0) continue; + /* if state marked as disabled, skip it */ + if (cpuidle_state_table[cstate].disabled != 0) { + pr_debug(PREFIX "state %s is disabled", + cpuidle_state_table[cstate].name); + continue; + } + + if (((mwait_cstate + 1) > 2) && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) mark_tsc_unstable("TSC halts in idle" @@ -606,6 +1166,11 @@ if (icpu->auto_demotion_disable_flags) on_each_cpu(auto_demotion_disable, NULL, 1); + if (icpu->byt_auto_demotion_disable_flag) { + wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); + wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); + } + if (icpu->disable_promotion_to_c1e) /* each-cpu is redundant */ on_each_cpu(c1e_promotion_disable, NULL, 1); @@ -620,39 +1185,10 @@ */ static int intel_idle_cpu_init(int cpu) { - int cstate; struct cpuidle_device *dev; dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); - dev->state_count = 1; - - for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { - int num_substates, mwait_hint, mwait_cstate, mwait_substate; - - if (cpuidle_state_table[cstate].enter == NULL) - continue; - - if (cstate + 1 > max_cstate) { - printk(PREFIX "max_cstate %d reached\n", max_cstate); - break; - } - - mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); - mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); - mwait_substate = MWAIT_HINT2SUBSTATE(mwait_hint); - - /* does the state exist in CPUID.MWAIT? */ - num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) - & MWAIT_SUBSTATE_MASK; - - /* if sub-state in table is not enumerated by CPUID */ - if ((mwait_substate + 1) > num_substates) - continue; - - dev->state_count += 1; - } - dev->cpu = cpu; if (cpuidle_register_device(dev)) { @@ -664,6 +1200,9 @@ if (icpu->auto_demotion_disable_flags) smp_call_function_single(cpu, auto_demotion_disable, NULL, 1); + if (icpu->disable_promotion_to_c1e) + smp_call_function_single(cpu, c1e_promotion_disable, NULL, 1); + return 0; } @@ -692,14 +1231,19 @@ if (intel_idle_cpuidle_devices == NULL) return -ENOMEM; + cpu_notifier_register_begin(); + for_each_online_cpu(i) { retval = intel_idle_cpu_init(i); if (retval) { + cpu_notifier_register_done(); cpuidle_unregister_driver(&intel_idle_driver); return retval; } } - register_cpu_notifier(&cpu_hotplug_notifier); + __register_cpu_notifier(&cpu_hotplug_notifier); + + cpu_notifier_register_done(); return 0; } @@ -709,10 +1253,13 @@ intel_idle_cpuidle_devices_uninit(); cpuidle_unregister_driver(&intel_idle_driver); + cpu_notifier_register_begin(); if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) on_each_cpu(__setup_broadcast_timer, (void *)false, 1); - unregister_cpu_notifier(&cpu_hotplug_notifier); + __unregister_cpu_notifier(&cpu_hotplug_notifier); + + cpu_notifier_register_done(); return; }