123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580 |
- #include <linux/cpu.h>
- #include <linux/err.h>
- #include <linux/hrtimer.h>
- #include <linux/interrupt.h>
- #include <linux/nmi.h>
- #include <linux/percpu.h>
- #include <linux/profile.h>
- #include <linux/sched.h>
- #include <linux/module.h>
- #include <trace/events/power.h>
- #include <trace/hooks/sched.h>
- #include <asm/irq_regs.h>
- #include "tick-internal.h"
- DEFINE_PER_CPU(struct tick_device, tick_cpu_device);
- ktime_t tick_next_period;
- int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT;
- #ifdef CONFIG_NO_HZ_FULL
- static int tick_do_timer_boot_cpu __read_mostly = -1;
- #endif
- struct tick_device *tick_get_device(int cpu)
- {
- return &per_cpu(tick_cpu_device, cpu);
- }
- int tick_is_oneshot_available(void)
- {
- struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
- if (!dev || !(dev->features & CLOCK_EVT_FEAT_ONESHOT))
- return 0;
- if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
- return 1;
- return tick_broadcast_oneshot_available();
- }
- static void tick_periodic(int cpu)
- {
- if (tick_do_timer_cpu == cpu) {
- raw_spin_lock(&jiffies_lock);
- write_seqcount_begin(&jiffies_seq);
-
- tick_next_period = ktime_add_ns(tick_next_period, TICK_NSEC);
- do_timer(1);
- write_seqcount_end(&jiffies_seq);
- raw_spin_unlock(&jiffies_lock);
- update_wall_time();
- trace_android_vh_jiffies_update(NULL);
- }
- update_process_times(user_mode(get_irq_regs()));
- profile_tick(CPU_PROFILING);
- }
- void tick_handle_periodic(struct clock_event_device *dev)
- {
- int cpu = smp_processor_id();
- ktime_t next = dev->next_event;
- tick_periodic(cpu);
- #if defined(CONFIG_HIGH_RES_TIMERS) || defined(CONFIG_NO_HZ_COMMON)
-
- if (dev->event_handler != tick_handle_periodic)
- return;
- #endif
- if (!clockevent_state_oneshot(dev))
- return;
- for (;;) {
-
- next = ktime_add_ns(next, TICK_NSEC);
- if (!clockevents_program_event(dev, next, false))
- return;
-
- if (timekeeping_valid_for_hres())
- tick_periodic(cpu);
- }
- }
- void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
- {
- tick_set_periodic_handler(dev, broadcast);
-
- if (!tick_device_is_functional(dev))
- return;
- if ((dev->features & CLOCK_EVT_FEAT_PERIODIC) &&
- !tick_broadcast_oneshot_active()) {
- clockevents_switch_state(dev, CLOCK_EVT_STATE_PERIODIC);
- } else {
- unsigned int seq;
- ktime_t next;
- do {
- seq = read_seqcount_begin(&jiffies_seq);
- next = tick_next_period;
- } while (read_seqcount_retry(&jiffies_seq, seq));
- clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
- for (;;) {
- if (!clockevents_program_event(dev, next, false))
- return;
- next = ktime_add_ns(next, TICK_NSEC);
- }
- }
- }
- #ifdef CONFIG_NO_HZ_FULL
- static void giveup_do_timer(void *info)
- {
- int cpu = *(unsigned int *)info;
- WARN_ON(tick_do_timer_cpu != smp_processor_id());
- tick_do_timer_cpu = cpu;
- }
- static void tick_take_do_timer_from_boot(void)
- {
- int cpu = smp_processor_id();
- int from = tick_do_timer_boot_cpu;
- if (from >= 0 && from != cpu)
- smp_call_function_single(from, giveup_do_timer, &cpu, 1);
- }
- #endif
- static void tick_setup_device(struct tick_device *td,
- struct clock_event_device *newdev, int cpu,
- const struct cpumask *cpumask)
- {
- void (*handler)(struct clock_event_device *) = NULL;
- ktime_t next_event = 0;
-
- if (!td->evtdev) {
-
- if (tick_do_timer_cpu == TICK_DO_TIMER_BOOT) {
- tick_do_timer_cpu = cpu;
- tick_next_period = ktime_get();
- #ifdef CONFIG_NO_HZ_FULL
-
- if (tick_nohz_full_cpu(cpu))
- tick_do_timer_boot_cpu = cpu;
- } else if (tick_do_timer_boot_cpu != -1 &&
- !tick_nohz_full_cpu(cpu)) {
- tick_take_do_timer_from_boot();
- tick_do_timer_boot_cpu = -1;
- WARN_ON(tick_do_timer_cpu != cpu);
- #endif
- }
-
- td->mode = TICKDEV_MODE_PERIODIC;
- } else {
- handler = td->evtdev->event_handler;
- next_event = td->evtdev->next_event;
- td->evtdev->event_handler = clockevents_handle_noop;
- }
- td->evtdev = newdev;
-
- if (!cpumask_equal(newdev->cpumask, cpumask))
- irq_set_affinity(newdev->irq, cpumask);
-
- if (tick_device_uses_broadcast(newdev, cpu))
- return;
- if (td->mode == TICKDEV_MODE_PERIODIC)
- tick_setup_periodic(newdev, 0);
- else
- tick_setup_oneshot(newdev, handler, next_event);
- }
- void tick_install_replacement(struct clock_event_device *newdev)
- {
- struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
- int cpu = smp_processor_id();
- clockevents_exchange_device(td->evtdev, newdev);
- tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
- if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
- tick_oneshot_notify();
- }
- static bool tick_check_percpu(struct clock_event_device *curdev,
- struct clock_event_device *newdev, int cpu)
- {
- if (!cpumask_test_cpu(cpu, newdev->cpumask))
- return false;
- if (cpumask_equal(newdev->cpumask, cpumask_of(cpu)))
- return true;
-
- if (newdev->irq >= 0 && !irq_can_set_affinity(newdev->irq))
- return false;
-
- if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu)))
- return false;
- return true;
- }
- static bool tick_check_preferred(struct clock_event_device *curdev,
- struct clock_event_device *newdev)
- {
-
- if (!(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) {
- if (curdev && (curdev->features & CLOCK_EVT_FEAT_ONESHOT))
- return false;
- if (tick_oneshot_mode_active())
- return false;
- }
-
- return !curdev ||
- newdev->rating > curdev->rating ||
- !cpumask_equal(curdev->cpumask, newdev->cpumask);
- }
- bool tick_check_replacement(struct clock_event_device *curdev,
- struct clock_event_device *newdev)
- {
- if (!tick_check_percpu(curdev, newdev, smp_processor_id()))
- return false;
- return tick_check_preferred(curdev, newdev);
- }
- void tick_check_new_device(struct clock_event_device *newdev)
- {
- struct clock_event_device *curdev;
- struct tick_device *td;
- int cpu;
- cpu = smp_processor_id();
- td = &per_cpu(tick_cpu_device, cpu);
- curdev = td->evtdev;
- if (!tick_check_replacement(curdev, newdev))
- goto out_bc;
- if (!try_module_get(newdev->owner))
- return;
-
- if (tick_is_broadcast_device(curdev)) {
- clockevents_shutdown(curdev);
- curdev = NULL;
- }
- clockevents_exchange_device(curdev, newdev);
- tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
- if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
- tick_oneshot_notify();
- return;
- out_bc:
-
- tick_install_broadcast_device(newdev, cpu);
- }
- int tick_broadcast_oneshot_control(enum tick_broadcast_state state)
- {
- struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
- if (!(td->evtdev->features & CLOCK_EVT_FEAT_C3STOP))
- return 0;
- return __tick_broadcast_oneshot_control(state);
- }
- EXPORT_SYMBOL_GPL(tick_broadcast_oneshot_control);
- #ifdef CONFIG_HOTPLUG_CPU
- void tick_handover_do_timer(void)
- {
- if (tick_do_timer_cpu == smp_processor_id())
- tick_do_timer_cpu = cpumask_first(cpu_online_mask);
- }
- void tick_shutdown(unsigned int cpu)
- {
- struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
- struct clock_event_device *dev = td->evtdev;
- td->mode = TICKDEV_MODE_PERIODIC;
- if (dev) {
-
- clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);
- clockevents_exchange_device(dev, NULL);
- dev->event_handler = clockevents_handle_noop;
- td->evtdev = NULL;
- }
- }
- #endif
- void tick_suspend_local(void)
- {
- struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
- clockevents_shutdown(td->evtdev);
- }
- void tick_resume_local(void)
- {
- struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
- bool broadcast = tick_resume_check_broadcast();
- clockevents_tick_resume(td->evtdev);
- if (!broadcast) {
- if (td->mode == TICKDEV_MODE_PERIODIC)
- tick_setup_periodic(td->evtdev, 0);
- else
- tick_resume_oneshot();
- }
-
- hrtimers_resume_local();
- }
- void tick_suspend(void)
- {
- tick_suspend_local();
- tick_suspend_broadcast();
- }
- void tick_resume(void)
- {
- tick_resume_broadcast();
- tick_resume_local();
- }
- #ifdef CONFIG_SUSPEND
- static DEFINE_RAW_SPINLOCK(tick_freeze_lock);
- static unsigned int tick_freeze_depth;
- void tick_freeze(void)
- {
- raw_spin_lock(&tick_freeze_lock);
- tick_freeze_depth++;
- if (tick_freeze_depth == num_online_cpus()) {
- trace_suspend_resume(TPS("timekeeping_freeze"),
- smp_processor_id(), true);
- system_state = SYSTEM_SUSPEND;
- sched_clock_suspend();
- timekeeping_suspend();
- } else {
- tick_suspend_local();
- }
- raw_spin_unlock(&tick_freeze_lock);
- }
- void tick_unfreeze(void)
- {
- raw_spin_lock(&tick_freeze_lock);
- if (tick_freeze_depth == num_online_cpus()) {
- timekeeping_resume();
- sched_clock_resume();
- system_state = SYSTEM_RUNNING;
- trace_suspend_resume(TPS("timekeeping_freeze"),
- smp_processor_id(), false);
- } else {
- touch_softlockup_watchdog();
- tick_resume_local();
- }
- tick_freeze_depth--;
- raw_spin_unlock(&tick_freeze_lock);
- }
- #endif
- void __init tick_init(void)
- {
- tick_broadcast_init();
- tick_nohz_init();
- }
|