vtime.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Virtual cpu timer based timer functions.
  4. *
  5. * Copyright IBM Corp. 2004, 2012
  6. * Author(s): Jan Glauber <[email protected]>
  7. */
  8. #include <linux/kernel_stat.h>
  9. #include <linux/sched/cputime.h>
  10. #include <linux/export.h>
  11. #include <linux/kernel.h>
  12. #include <linux/timex.h>
  13. #include <linux/types.h>
  14. #include <linux/time.h>
  15. #include <asm/alternative.h>
  16. #include <asm/vtimer.h>
  17. #include <asm/vtime.h>
  18. #include <asm/cpu_mf.h>
  19. #include <asm/smp.h>
  20. #include "entry.h"
  21. static void virt_timer_expire(void);
  22. static LIST_HEAD(virt_timer_list);
  23. static DEFINE_SPINLOCK(virt_timer_lock);
  24. static atomic64_t virt_timer_current;
  25. static atomic64_t virt_timer_elapsed;
  26. DEFINE_PER_CPU(u64, mt_cycles[8]);
  27. static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
  28. static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
  29. static DEFINE_PER_CPU(u64, mt_scaling_jiffies);
  30. static inline u64 get_vtimer(void)
  31. {
  32. u64 timer;
  33. asm volatile("stpt %0" : "=Q" (timer));
  34. return timer;
  35. }
  36. static inline void set_vtimer(u64 expires)
  37. {
  38. u64 timer;
  39. asm volatile(
  40. " stpt %0\n" /* Store current cpu timer value */
  41. " spt %1" /* Set new value imm. afterwards */
  42. : "=Q" (timer) : "Q" (expires));
  43. S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
  44. S390_lowcore.last_update_timer = expires;
  45. }
  46. static inline int virt_timer_forward(u64 elapsed)
  47. {
  48. BUG_ON(!irqs_disabled());
  49. if (list_empty(&virt_timer_list))
  50. return 0;
  51. elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
  52. return elapsed >= atomic64_read(&virt_timer_current);
  53. }
  54. static void update_mt_scaling(void)
  55. {
  56. u64 cycles_new[8], *cycles_old;
  57. u64 delta, fac, mult, div;
  58. int i;
  59. stcctm(MT_DIAG, smp_cpu_mtid + 1, cycles_new);
  60. cycles_old = this_cpu_ptr(mt_cycles);
  61. fac = 1;
  62. mult = div = 0;
  63. for (i = 0; i <= smp_cpu_mtid; i++) {
  64. delta = cycles_new[i] - cycles_old[i];
  65. div += delta;
  66. mult *= i + 1;
  67. mult += delta * fac;
  68. fac *= i + 1;
  69. }
  70. div *= fac;
  71. if (div > 0) {
  72. /* Update scaling factor */
  73. __this_cpu_write(mt_scaling_mult, mult);
  74. __this_cpu_write(mt_scaling_div, div);
  75. memcpy(cycles_old, cycles_new,
  76. sizeof(u64) * (smp_cpu_mtid + 1));
  77. }
  78. __this_cpu_write(mt_scaling_jiffies, jiffies_64);
  79. }
  80. static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
  81. {
  82. u64 delta;
  83. delta = new - *tsk_vtime;
  84. *tsk_vtime = new;
  85. return delta;
  86. }
  87. static inline u64 scale_vtime(u64 vtime)
  88. {
  89. u64 mult = __this_cpu_read(mt_scaling_mult);
  90. u64 div = __this_cpu_read(mt_scaling_div);
  91. if (smp_cpu_mtid)
  92. return vtime * mult / div;
  93. return vtime;
  94. }
  95. static void account_system_index_scaled(struct task_struct *p, u64 cputime,
  96. enum cpu_usage_stat index)
  97. {
  98. p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));
  99. account_system_index_time(p, cputime_to_nsecs(cputime), index);
  100. }
  101. /*
  102. * Update process times based on virtual cpu times stored by entry.S
  103. * to the lowcore fields user_timer, system_timer & steal_clock.
  104. */
  105. static int do_account_vtime(struct task_struct *tsk)
  106. {
  107. u64 timer, clock, user, guest, system, hardirq, softirq;
  108. timer = S390_lowcore.last_update_timer;
  109. clock = S390_lowcore.last_update_clock;
  110. asm volatile(
  111. " stpt %0\n" /* Store current cpu timer value */
  112. " stckf %1" /* Store current tod clock value */
  113. : "=Q" (S390_lowcore.last_update_timer),
  114. "=Q" (S390_lowcore.last_update_clock)
  115. : : "cc");
  116. clock = S390_lowcore.last_update_clock - clock;
  117. timer -= S390_lowcore.last_update_timer;
  118. if (hardirq_count())
  119. S390_lowcore.hardirq_timer += timer;
  120. else
  121. S390_lowcore.system_timer += timer;
  122. /* Update MT utilization calculation */
  123. if (smp_cpu_mtid &&
  124. time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
  125. update_mt_scaling();
  126. /* Calculate cputime delta */
  127. user = update_tsk_timer(&tsk->thread.user_timer,
  128. READ_ONCE(S390_lowcore.user_timer));
  129. guest = update_tsk_timer(&tsk->thread.guest_timer,
  130. READ_ONCE(S390_lowcore.guest_timer));
  131. system = update_tsk_timer(&tsk->thread.system_timer,
  132. READ_ONCE(S390_lowcore.system_timer));
  133. hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
  134. READ_ONCE(S390_lowcore.hardirq_timer));
  135. softirq = update_tsk_timer(&tsk->thread.softirq_timer,
  136. READ_ONCE(S390_lowcore.softirq_timer));
  137. S390_lowcore.steal_timer +=
  138. clock - user - guest - system - hardirq - softirq;
  139. /* Push account value */
  140. if (user) {
  141. account_user_time(tsk, cputime_to_nsecs(user));
  142. tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
  143. }
  144. if (guest) {
  145. account_guest_time(tsk, cputime_to_nsecs(guest));
  146. tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
  147. }
  148. if (system)
  149. account_system_index_scaled(tsk, system, CPUTIME_SYSTEM);
  150. if (hardirq)
  151. account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ);
  152. if (softirq)
  153. account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ);
  154. return virt_timer_forward(user + guest + system + hardirq + softirq);
  155. }
  156. void vtime_task_switch(struct task_struct *prev)
  157. {
  158. do_account_vtime(prev);
  159. prev->thread.user_timer = S390_lowcore.user_timer;
  160. prev->thread.guest_timer = S390_lowcore.guest_timer;
  161. prev->thread.system_timer = S390_lowcore.system_timer;
  162. prev->thread.hardirq_timer = S390_lowcore.hardirq_timer;
  163. prev->thread.softirq_timer = S390_lowcore.softirq_timer;
  164. S390_lowcore.user_timer = current->thread.user_timer;
  165. S390_lowcore.guest_timer = current->thread.guest_timer;
  166. S390_lowcore.system_timer = current->thread.system_timer;
  167. S390_lowcore.hardirq_timer = current->thread.hardirq_timer;
  168. S390_lowcore.softirq_timer = current->thread.softirq_timer;
  169. }
  170. /*
  171. * In s390, accounting pending user time also implies
  172. * accounting system time in order to correctly compute
  173. * the stolen time accounting.
  174. */
  175. void vtime_flush(struct task_struct *tsk)
  176. {
  177. u64 steal, avg_steal;
  178. if (do_account_vtime(tsk))
  179. virt_timer_expire();
  180. steal = S390_lowcore.steal_timer;
  181. avg_steal = S390_lowcore.avg_steal_timer / 2;
  182. if ((s64) steal > 0) {
  183. S390_lowcore.steal_timer = 0;
  184. account_steal_time(cputime_to_nsecs(steal));
  185. avg_steal += steal;
  186. }
  187. S390_lowcore.avg_steal_timer = avg_steal;
  188. }
  189. static u64 vtime_delta(void)
  190. {
  191. u64 timer = S390_lowcore.last_update_timer;
  192. S390_lowcore.last_update_timer = get_vtimer();
  193. return timer - S390_lowcore.last_update_timer;
  194. }
  195. /*
  196. * Update process times based on virtual cpu times stored by entry.S
  197. * to the lowcore fields user_timer, system_timer & steal_clock.
  198. */
  199. void vtime_account_kernel(struct task_struct *tsk)
  200. {
  201. u64 delta = vtime_delta();
  202. if (tsk->flags & PF_VCPU)
  203. S390_lowcore.guest_timer += delta;
  204. else
  205. S390_lowcore.system_timer += delta;
  206. virt_timer_forward(delta);
  207. }
  208. EXPORT_SYMBOL_GPL(vtime_account_kernel);
  209. void vtime_account_softirq(struct task_struct *tsk)
  210. {
  211. u64 delta = vtime_delta();
  212. S390_lowcore.softirq_timer += delta;
  213. virt_timer_forward(delta);
  214. }
  215. void vtime_account_hardirq(struct task_struct *tsk)
  216. {
  217. u64 delta = vtime_delta();
  218. S390_lowcore.hardirq_timer += delta;
  219. virt_timer_forward(delta);
  220. }
  221. /*
  222. * Sorted add to a list. List is linear searched until first bigger
  223. * element is found.
  224. */
  225. static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
  226. {
  227. struct vtimer_list *tmp;
  228. list_for_each_entry(tmp, head, entry) {
  229. if (tmp->expires > timer->expires) {
  230. list_add_tail(&timer->entry, &tmp->entry);
  231. return;
  232. }
  233. }
  234. list_add_tail(&timer->entry, head);
  235. }
  236. /*
  237. * Handler for expired virtual CPU timer.
  238. */
  239. static void virt_timer_expire(void)
  240. {
  241. struct vtimer_list *timer, *tmp;
  242. unsigned long elapsed;
  243. LIST_HEAD(cb_list);
  244. /* walk timer list, fire all expired timers */
  245. spin_lock(&virt_timer_lock);
  246. elapsed = atomic64_read(&virt_timer_elapsed);
  247. list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
  248. if (timer->expires < elapsed)
  249. /* move expired timer to the callback queue */
  250. list_move_tail(&timer->entry, &cb_list);
  251. else
  252. timer->expires -= elapsed;
  253. }
  254. if (!list_empty(&virt_timer_list)) {
  255. timer = list_first_entry(&virt_timer_list,
  256. struct vtimer_list, entry);
  257. atomic64_set(&virt_timer_current, timer->expires);
  258. }
  259. atomic64_sub(elapsed, &virt_timer_elapsed);
  260. spin_unlock(&virt_timer_lock);
  261. /* Do callbacks and recharge periodic timers */
  262. list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
  263. list_del_init(&timer->entry);
  264. timer->function(timer->data);
  265. if (timer->interval) {
  266. /* Recharge interval timer */
  267. timer->expires = timer->interval +
  268. atomic64_read(&virt_timer_elapsed);
  269. spin_lock(&virt_timer_lock);
  270. list_add_sorted(timer, &virt_timer_list);
  271. spin_unlock(&virt_timer_lock);
  272. }
  273. }
  274. }
  275. void init_virt_timer(struct vtimer_list *timer)
  276. {
  277. timer->function = NULL;
  278. INIT_LIST_HEAD(&timer->entry);
  279. }
  280. EXPORT_SYMBOL(init_virt_timer);
  281. static inline int vtimer_pending(struct vtimer_list *timer)
  282. {
  283. return !list_empty(&timer->entry);
  284. }
  285. static void internal_add_vtimer(struct vtimer_list *timer)
  286. {
  287. if (list_empty(&virt_timer_list)) {
  288. /* First timer, just program it. */
  289. atomic64_set(&virt_timer_current, timer->expires);
  290. atomic64_set(&virt_timer_elapsed, 0);
  291. list_add(&timer->entry, &virt_timer_list);
  292. } else {
  293. /* Update timer against current base. */
  294. timer->expires += atomic64_read(&virt_timer_elapsed);
  295. if (likely((s64) timer->expires <
  296. (s64) atomic64_read(&virt_timer_current)))
  297. /* The new timer expires before the current timer. */
  298. atomic64_set(&virt_timer_current, timer->expires);
  299. /* Insert new timer into the list. */
  300. list_add_sorted(timer, &virt_timer_list);
  301. }
  302. }
  303. static void __add_vtimer(struct vtimer_list *timer, int periodic)
  304. {
  305. unsigned long flags;
  306. timer->interval = periodic ? timer->expires : 0;
  307. spin_lock_irqsave(&virt_timer_lock, flags);
  308. internal_add_vtimer(timer);
  309. spin_unlock_irqrestore(&virt_timer_lock, flags);
  310. }
  311. /*
  312. * add_virt_timer - add a oneshot virtual CPU timer
  313. */
  314. void add_virt_timer(struct vtimer_list *timer)
  315. {
  316. __add_vtimer(timer, 0);
  317. }
  318. EXPORT_SYMBOL(add_virt_timer);
  319. /*
  320. * add_virt_timer_int - add an interval virtual CPU timer
  321. */
  322. void add_virt_timer_periodic(struct vtimer_list *timer)
  323. {
  324. __add_vtimer(timer, 1);
  325. }
  326. EXPORT_SYMBOL(add_virt_timer_periodic);
  327. static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
  328. {
  329. unsigned long flags;
  330. int rc;
  331. BUG_ON(!timer->function);
  332. if (timer->expires == expires && vtimer_pending(timer))
  333. return 1;
  334. spin_lock_irqsave(&virt_timer_lock, flags);
  335. rc = vtimer_pending(timer);
  336. if (rc)
  337. list_del_init(&timer->entry);
  338. timer->interval = periodic ? expires : 0;
  339. timer->expires = expires;
  340. internal_add_vtimer(timer);
  341. spin_unlock_irqrestore(&virt_timer_lock, flags);
  342. return rc;
  343. }
  344. /*
  345. * returns whether it has modified a pending timer (1) or not (0)
  346. */
  347. int mod_virt_timer(struct vtimer_list *timer, u64 expires)
  348. {
  349. return __mod_vtimer(timer, expires, 0);
  350. }
  351. EXPORT_SYMBOL(mod_virt_timer);
  352. /*
  353. * returns whether it has modified a pending timer (1) or not (0)
  354. */
  355. int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
  356. {
  357. return __mod_vtimer(timer, expires, 1);
  358. }
  359. EXPORT_SYMBOL(mod_virt_timer_periodic);
  360. /*
  361. * Delete a virtual timer.
  362. *
  363. * returns whether the deleted timer was pending (1) or not (0)
  364. */
  365. int del_virt_timer(struct vtimer_list *timer)
  366. {
  367. unsigned long flags;
  368. if (!vtimer_pending(timer))
  369. return 0;
  370. spin_lock_irqsave(&virt_timer_lock, flags);
  371. list_del_init(&timer->entry);
  372. spin_unlock_irqrestore(&virt_timer_lock, flags);
  373. return 1;
  374. }
  375. EXPORT_SYMBOL(del_virt_timer);
  376. /*
  377. * Start the virtual CPU timer on the current CPU.
  378. */
  379. void vtime_init(void)
  380. {
  381. /* set initial cpu timer */
  382. set_vtimer(VTIMER_MAX_SLICE);
  383. /* Setup initial MT scaling values */
  384. if (smp_cpu_mtid) {
  385. __this_cpu_write(mt_scaling_jiffies, jiffies);
  386. __this_cpu_write(mt_scaling_mult, 1);
  387. __this_cpu_write(mt_scaling_div, 1);
  388. stcctm(MT_DIAG, smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
  389. }
  390. }