blk-sec-stat-pio.c 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Samsung Block Statistics
  4. *
  5. * Copyright (C) 2021 Manjong Lee <[email protected]>
  6. * Copyright (C) 2021 Junho Kim <[email protected]>
  7. * Copyright (C) 2021 Changheun Lee <[email protected]>
  8. * Copyright (C) 2021 Seunghwan Hyun <[email protected]>
  9. * Copyright (C) 2021 Tran Xuan Nam <[email protected]>
  10. */
  11. #include <linux/sysfs.h>
  12. #include <linux/blk_types.h>
  13. #include <linux/blkdev.h>
  14. #include <linux/blk-mq.h>
  15. #include "blk-sec.h"
  16. #define MAX_PIO_NODE_NUM 10000
  17. #define SORT_PIO_NODE_NUM 100
  18. #define PIO_HASH_SIZE 100
  19. #define MAX_PIO_DURATION_MS 10000
  20. #define GET_HASH_KEY(tgid) ((unsigned int)(tgid) % PIO_HASH_SIZE)
  21. #define RESET_PIO_IO(pio) \
  22. do { \
  23. atomic_set(&(pio)->kb[REQ_OP_READ], 0); \
  24. atomic_set(&(pio)->kb[REQ_OP_WRITE], 0); \
  25. atomic_set(&(pio)->kb[REQ_OP_FLUSH], 0); \
  26. atomic_set(&(pio)->kb[REQ_OP_DISCARD], 0); \
  27. } while (0)
  28. #define GET_PIO_PRIO(pio) \
  29. (atomic_read(&(pio)->kb[REQ_OP_READ]) + \
  30. atomic_read(&(pio)->kb[REQ_OP_WRITE]) * 2)
  31. LIST_HEAD(pio_list);
  32. LIST_HEAD(inflight_pios);
  33. static DEFINE_SPINLOCK(pio_list_lock);
  34. static DEFINE_SPINLOCK(inflight_pios_lock);
  35. static int pio_cnt;
  36. static int pio_enabled;
  37. static unsigned int pio_duration_ms = 5000;
  38. static unsigned long pio_timeout;
  39. static struct kmem_cache *pio_cache;
  40. static struct pio_node *pio_hash[PIO_HASH_SIZE];
  41. static struct pio_node others;
  42. static struct pio_node *add_pio_node(struct request *rq,
  43. struct task_struct *gleader)
  44. {
  45. struct pio_node *pio = NULL;
  46. unsigned int hash_key = 0;
  47. if (pio_cnt >= MAX_PIO_NODE_NUM) {
  48. add_others:
  49. return &others;
  50. }
  51. pio = kmem_cache_alloc(pio_cache, GFP_NOWAIT);
  52. if (!pio)
  53. goto add_others;
  54. INIT_LIST_HEAD(&pio->list);
  55. pio->tgid = task_tgid_nr(gleader);
  56. strncpy(pio->name, gleader->comm, TASK_COMM_LEN - 1);
  57. pio->name[TASK_COMM_LEN - 1] = '\0';
  58. pio->start_time = gleader->start_time;
  59. RESET_PIO_IO(pio);
  60. atomic_set(&pio->ref_count, 1);
  61. hash_key = GET_HASH_KEY(pio->tgid);
  62. spin_lock(&pio_list_lock);
  63. list_add(&pio->list, &pio_list);
  64. pio->h_next = pio_hash[hash_key];
  65. pio_hash[hash_key] = pio;
  66. pio_cnt++;
  67. spin_unlock(&pio_list_lock);
  68. return pio;
  69. }
  70. static struct pio_node *find_pio_node(pid_t tgid, u64 tg_start_time)
  71. {
  72. struct pio_node *pio;
  73. for (pio = pio_hash[GET_HASH_KEY(tgid)]; pio; pio = pio->h_next) {
  74. if (pio->tgid != tgid)
  75. continue;
  76. if (pio->start_time != tg_start_time)
  77. continue;
  78. return pio;
  79. }
  80. return NULL;
  81. }
  82. static void free_pio_nodes(struct list_head *remove_list)
  83. {
  84. struct pio_node *pio;
  85. struct pio_node *pion;
  86. /* move previous inflight pios to remove_list */
  87. spin_lock(&inflight_pios_lock);
  88. list_splice_init(&inflight_pios, remove_list);
  89. spin_unlock(&inflight_pios_lock);
  90. list_for_each_entry_safe(pio, pion, remove_list, list) {
  91. list_del(&pio->list);
  92. if (atomic_read(&pio->ref_count)) {
  93. spin_lock(&inflight_pios_lock);
  94. list_add(&pio->list, &inflight_pios);
  95. spin_unlock(&inflight_pios_lock);
  96. continue;
  97. }
  98. kmem_cache_free(pio_cache, pio);
  99. }
  100. }
  101. struct pio_node *get_pio_node(struct request *rq)
  102. {
  103. struct task_struct *gleader = current->group_leader;
  104. struct pio_node *pio;
  105. if (pio_enabled == 0)
  106. return NULL;
  107. if (time_after(jiffies, pio_timeout))
  108. return NULL;
  109. if (req_op(rq) > REQ_OP_DISCARD)
  110. return NULL;
  111. spin_lock(&pio_list_lock);
  112. pio = find_pio_node(task_tgid_nr(gleader), gleader->start_time);
  113. if (pio) {
  114. atomic_inc(&pio->ref_count);
  115. spin_unlock(&pio_list_lock);
  116. return pio;
  117. }
  118. spin_unlock(&pio_list_lock);
  119. return add_pio_node(rq, gleader);
  120. }
  121. void update_pio_node(struct request *rq,
  122. unsigned int data_size, struct pio_node *pio)
  123. {
  124. if (!pio)
  125. return;
  126. /* transfer bytes to kbytes via '>> 10' */
  127. atomic_add((req_op(rq) == REQ_OP_FLUSH) ? 1 : data_size >> 10,
  128. &pio->kb[req_op(rq)]);
  129. }
  130. void put_pio_node(struct pio_node *pio)
  131. {
  132. if (!pio)
  133. return;
  134. atomic_dec(&pio->ref_count);
  135. }
  136. static void sort_pios(struct list_head *pios)
  137. {
  138. struct pio_node *max_pio = NULL;
  139. struct pio_node *pio;
  140. unsigned long long max = 0;
  141. LIST_HEAD(sorted_list);
  142. int i;
  143. for (i = 0; i < SORT_PIO_NODE_NUM; i++) {
  144. list_for_each_entry(pio, pios, list) {
  145. if (GET_PIO_PRIO(pio) > max) {
  146. max = GET_PIO_PRIO(pio);
  147. max_pio = pio;
  148. }
  149. }
  150. if (max_pio != NULL)
  151. list_move_tail(&max_pio->list, &sorted_list);
  152. max = 0;
  153. max_pio = NULL;
  154. }
  155. list_splice_init(&sorted_list, pios);
  156. }
  157. static ssize_t pio_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
  158. {
  159. LIST_HEAD(curr_pios);
  160. int curr_pio_cnt;
  161. struct pio_node curr_others;
  162. struct pio_node *pio;
  163. int len = 0;
  164. spin_lock(&pio_list_lock);
  165. list_replace_init(&pio_list, &curr_pios);
  166. curr_pio_cnt = pio_cnt;
  167. curr_others = others;
  168. memset(pio_hash, 0x0, sizeof(pio_hash));
  169. pio_cnt = 0;
  170. RESET_PIO_IO(&others);
  171. spin_unlock(&pio_list_lock);
  172. if (curr_pio_cnt > SORT_PIO_NODE_NUM)
  173. sort_pios(&curr_pios);
  174. list_for_each_entry(pio, &curr_pios, list) {
  175. if (PAGE_SIZE - len > 80) {
  176. len += scnprintf(buf + len, PAGE_SIZE - len,
  177. "%d %d %d %s\n",
  178. pio->tgid,
  179. atomic_read(&pio->kb[REQ_OP_READ]),
  180. atomic_read(&pio->kb[REQ_OP_WRITE]),
  181. (pio->name[0]) ? pio->name : "-");
  182. continue;
  183. }
  184. atomic_add(atomic_read(&pio->kb[REQ_OP_READ]),
  185. &curr_others.kb[REQ_OP_READ]);
  186. atomic_add(atomic_read(&pio->kb[REQ_OP_WRITE]),
  187. &curr_others.kb[REQ_OP_WRITE]);
  188. atomic_add(atomic_read(&pio->kb[REQ_OP_FLUSH]),
  189. &curr_others.kb[REQ_OP_FLUSH]);
  190. atomic_add(atomic_read(&pio->kb[REQ_OP_DISCARD]),
  191. &curr_others.kb[REQ_OP_DISCARD]);
  192. }
  193. if (atomic_read(&curr_others.kb[REQ_OP_READ]) +
  194. atomic_read(&curr_others.kb[REQ_OP_WRITE]))
  195. len += scnprintf(buf + len, PAGE_SIZE - len,
  196. "%d %d %d %s\n",
  197. curr_others.tgid,
  198. atomic_read(&curr_others.kb[REQ_OP_READ]),
  199. atomic_read(&curr_others.kb[REQ_OP_WRITE]),
  200. curr_others.name);
  201. free_pio_nodes(&curr_pios);
  202. pio_timeout = jiffies + msecs_to_jiffies(pio_duration_ms);
  203. return len;
  204. }
  205. static ssize_t pio_enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
  206. const char *buf, size_t count)
  207. {
  208. LIST_HEAD(curr_pios);
  209. int enable = 0;
  210. int ret;
  211. ret = kstrtoint(buf, 10, &enable);
  212. if (ret)
  213. return ret;
  214. pio_enabled = (enable >= 1) ? 1 : 0;
  215. spin_lock(&pio_list_lock);
  216. list_replace_init(&pio_list, &curr_pios);
  217. memset(pio_hash, 0x0, sizeof(pio_hash));
  218. pio_cnt = 0;
  219. RESET_PIO_IO(&others);
  220. spin_unlock(&pio_list_lock);
  221. free_pio_nodes(&curr_pios);
  222. pio_timeout = jiffies + msecs_to_jiffies(pio_duration_ms);
  223. return count;
  224. }
  225. static ssize_t pio_enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
  226. {
  227. int len = 0;
  228. len = scnprintf(buf, PAGE_SIZE, "%d\n", pio_enabled);
  229. return len;
  230. }
  231. static ssize_t pio_duration_ms_store(struct kobject *kobj, struct kobj_attribute *attr,
  232. const char *buf, size_t count)
  233. {
  234. int ret;
  235. ret = kstrtoint(buf, 10, &pio_duration_ms);
  236. if (ret)
  237. return ret;
  238. if (pio_duration_ms > MAX_PIO_DURATION_MS)
  239. pio_duration_ms = MAX_PIO_DURATION_MS;
  240. return count;
  241. }
  242. static ssize_t pio_duration_ms_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
  243. {
  244. int len = 0;
  245. len = scnprintf(buf, PAGE_SIZE, "%u\n", pio_duration_ms);
  246. return len;
  247. }
  248. static struct kobj_attribute pios_attr = __ATTR(pios, 0444, pio_show, NULL);
  249. static struct kobj_attribute pios_enable_attr = __ATTR(pios_enable, 0644,
  250. pio_enabled_show, pio_enabled_store);
  251. static struct kobj_attribute pios_duration_ms_attr = __ATTR(pios_duration_ms, 0644,
  252. pio_duration_ms_show, pio_duration_ms_store);
  253. static const struct attribute *blk_sec_stat_pio_attrs[] = {
  254. &pios_attr.attr,
  255. &pios_enable_attr.attr,
  256. &pios_duration_ms_attr.attr,
  257. NULL,
  258. };
  259. int blk_sec_stat_pio_init(struct kobject *kobj)
  260. {
  261. int retval;
  262. if (!kobj)
  263. return -EINVAL;
  264. pio_cache = kmem_cache_create("pio_node", sizeof(struct pio_node), 0, 0, NULL);
  265. if (!pio_cache)
  266. return -ENOMEM;
  267. retval = sysfs_create_files(kobj, blk_sec_stat_pio_attrs);
  268. if (retval) {
  269. kmem_cache_destroy(pio_cache);
  270. return retval;
  271. }
  272. /* init others */
  273. INIT_LIST_HEAD(&others.list);
  274. others.tgid = INT_MAX;
  275. strncpy(others.name, "others", TASK_COMM_LEN - 1);
  276. others.name[TASK_COMM_LEN - 1] = '\0';
  277. others.start_time = 0;
  278. RESET_PIO_IO(&others);
  279. atomic_set(&others.ref_count, 1);
  280. others.h_next = NULL;
  281. return 0;
  282. }
  283. void blk_sec_stat_pio_exit(struct kobject *kobj)
  284. {
  285. if (!kobj)
  286. return;
  287. sysfs_remove_files(kobj, blk_sec_stat_pio_attrs);
  288. kmem_cache_destroy(pio_cache);
  289. }