context.c 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright 2016-2021 HabanaLabs, Ltd.
  4. * All Rights Reserved.
  5. */
  6. #include "habanalabs.h"
  7. #include <linux/slab.h>
  8. void hl_encaps_handle_do_release(struct kref *ref)
  9. {
  10. struct hl_cs_encaps_sig_handle *handle =
  11. container_of(ref, struct hl_cs_encaps_sig_handle, refcount);
  12. struct hl_encaps_signals_mgr *mgr = &handle->ctx->sig_mgr;
  13. spin_lock(&mgr->lock);
  14. idr_remove(&mgr->handles, handle->id);
  15. spin_unlock(&mgr->lock);
  16. hl_ctx_put(handle->ctx);
  17. kfree(handle);
  18. }
  19. static void hl_encaps_handle_do_release_sob(struct kref *ref)
  20. {
  21. struct hl_cs_encaps_sig_handle *handle =
  22. container_of(ref, struct hl_cs_encaps_sig_handle, refcount);
  23. struct hl_encaps_signals_mgr *mgr = &handle->ctx->sig_mgr;
  24. /* if we're here, then there was a signals reservation but cs with
  25. * encaps signals wasn't submitted, so need to put refcount
  26. * to hw_sob taken at the reservation.
  27. */
  28. hw_sob_put(handle->hw_sob);
  29. spin_lock(&mgr->lock);
  30. idr_remove(&mgr->handles, handle->id);
  31. spin_unlock(&mgr->lock);
  32. hl_ctx_put(handle->ctx);
  33. kfree(handle);
  34. }
  35. static void hl_encaps_sig_mgr_init(struct hl_encaps_signals_mgr *mgr)
  36. {
  37. spin_lock_init(&mgr->lock);
  38. idr_init(&mgr->handles);
  39. }
  40. static void hl_encaps_sig_mgr_fini(struct hl_device *hdev,
  41. struct hl_encaps_signals_mgr *mgr)
  42. {
  43. struct hl_cs_encaps_sig_handle *handle;
  44. struct idr *idp;
  45. u32 id;
  46. idp = &mgr->handles;
  47. if (!idr_is_empty(idp)) {
  48. dev_warn(hdev->dev, "device released while some encaps signals handles are still allocated\n");
  49. idr_for_each_entry(idp, handle, id)
  50. kref_put(&handle->refcount,
  51. hl_encaps_handle_do_release_sob);
  52. }
  53. idr_destroy(&mgr->handles);
  54. }
  55. static void hl_ctx_fini(struct hl_ctx *ctx)
  56. {
  57. struct hl_device *hdev = ctx->hdev;
  58. int i;
  59. /* Release all allocated HW block mapped list entries and destroy
  60. * the mutex.
  61. */
  62. hl_hw_block_mem_fini(ctx);
  63. /*
  64. * If we arrived here, there are no jobs waiting for this context
  65. * on its queues so we can safely remove it.
  66. * This is because for each CS, we increment the ref count and for
  67. * every CS that was finished we decrement it and we won't arrive
  68. * to this function unless the ref count is 0
  69. */
  70. for (i = 0 ; i < hdev->asic_prop.max_pending_cs ; i++)
  71. hl_fence_put(ctx->cs_pending[i]);
  72. kfree(ctx->cs_pending);
  73. if (ctx->asid != HL_KERNEL_ASID_ID) {
  74. dev_dbg(hdev->dev, "closing user context %d\n", ctx->asid);
  75. /* The engines are stopped as there is no executing CS, but the
  76. * Coresight might be still working by accessing addresses
  77. * related to the stopped engines. Hence stop it explicitly.
  78. */
  79. if (hdev->in_debug)
  80. hl_device_set_debug_mode(hdev, ctx, false);
  81. hdev->asic_funcs->ctx_fini(ctx);
  82. hl_dec_ctx_fini(ctx);
  83. hl_cb_va_pool_fini(ctx);
  84. hl_vm_ctx_fini(ctx);
  85. hl_asid_free(hdev, ctx->asid);
  86. hl_encaps_sig_mgr_fini(hdev, &ctx->sig_mgr);
  87. } else {
  88. dev_dbg(hdev->dev, "closing kernel context\n");
  89. hdev->asic_funcs->ctx_fini(ctx);
  90. hl_vm_ctx_fini(ctx);
  91. hl_mmu_ctx_fini(ctx);
  92. }
  93. }
  94. void hl_ctx_do_release(struct kref *ref)
  95. {
  96. struct hl_ctx *ctx;
  97. ctx = container_of(ref, struct hl_ctx, refcount);
  98. hl_ctx_fini(ctx);
  99. if (ctx->hpriv) {
  100. struct hl_fpriv *hpriv = ctx->hpriv;
  101. mutex_lock(&hpriv->ctx_lock);
  102. hpriv->ctx = NULL;
  103. mutex_unlock(&hpriv->ctx_lock);
  104. hl_hpriv_put(hpriv);
  105. }
  106. kfree(ctx);
  107. }
  108. int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv)
  109. {
  110. struct hl_ctx_mgr *ctx_mgr = &hpriv->ctx_mgr;
  111. struct hl_ctx *ctx;
  112. int rc;
  113. ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
  114. if (!ctx) {
  115. rc = -ENOMEM;
  116. goto out_err;
  117. }
  118. mutex_lock(&ctx_mgr->lock);
  119. rc = idr_alloc(&ctx_mgr->handles, ctx, 1, 0, GFP_KERNEL);
  120. mutex_unlock(&ctx_mgr->lock);
  121. if (rc < 0) {
  122. dev_err(hdev->dev, "Failed to allocate IDR for a new CTX\n");
  123. goto free_ctx;
  124. }
  125. ctx->handle = rc;
  126. rc = hl_ctx_init(hdev, ctx, false);
  127. if (rc)
  128. goto remove_from_idr;
  129. hl_hpriv_get(hpriv);
  130. ctx->hpriv = hpriv;
  131. /* TODO: remove for multiple contexts per process */
  132. hpriv->ctx = ctx;
  133. /* TODO: remove the following line for multiple process support */
  134. hdev->is_compute_ctx_active = true;
  135. return 0;
  136. remove_from_idr:
  137. mutex_lock(&ctx_mgr->lock);
  138. idr_remove(&ctx_mgr->handles, ctx->handle);
  139. mutex_unlock(&ctx_mgr->lock);
  140. free_ctx:
  141. kfree(ctx);
  142. out_err:
  143. return rc;
  144. }
  145. int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
  146. {
  147. int rc = 0, i;
  148. ctx->hdev = hdev;
  149. kref_init(&ctx->refcount);
  150. ctx->cs_sequence = 1;
  151. spin_lock_init(&ctx->cs_lock);
  152. atomic_set(&ctx->thread_ctx_switch_token, 1);
  153. ctx->thread_ctx_switch_wait_token = 0;
  154. ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,
  155. sizeof(struct hl_fence *),
  156. GFP_KERNEL);
  157. if (!ctx->cs_pending)
  158. return -ENOMEM;
  159. INIT_LIST_HEAD(&ctx->outcome_store.used_list);
  160. INIT_LIST_HEAD(&ctx->outcome_store.free_list);
  161. hash_init(ctx->outcome_store.outcome_map);
  162. for (i = 0; i < ARRAY_SIZE(ctx->outcome_store.nodes_pool); ++i)
  163. list_add(&ctx->outcome_store.nodes_pool[i].list_link,
  164. &ctx->outcome_store.free_list);
  165. hl_hw_block_mem_init(ctx);
  166. if (is_kernel_ctx) {
  167. ctx->asid = HL_KERNEL_ASID_ID; /* Kernel driver gets ASID 0 */
  168. rc = hl_vm_ctx_init(ctx);
  169. if (rc) {
  170. dev_err(hdev->dev, "Failed to init mem ctx module\n");
  171. rc = -ENOMEM;
  172. goto err_hw_block_mem_fini;
  173. }
  174. rc = hdev->asic_funcs->ctx_init(ctx);
  175. if (rc) {
  176. dev_err(hdev->dev, "ctx_init failed\n");
  177. goto err_vm_ctx_fini;
  178. }
  179. } else {
  180. ctx->asid = hl_asid_alloc(hdev);
  181. if (!ctx->asid) {
  182. dev_err(hdev->dev, "No free ASID, failed to create context\n");
  183. rc = -ENOMEM;
  184. goto err_hw_block_mem_fini;
  185. }
  186. rc = hl_vm_ctx_init(ctx);
  187. if (rc) {
  188. dev_err(hdev->dev, "Failed to init mem ctx module\n");
  189. rc = -ENOMEM;
  190. goto err_asid_free;
  191. }
  192. rc = hl_cb_va_pool_init(ctx);
  193. if (rc) {
  194. dev_err(hdev->dev,
  195. "Failed to init VA pool for mapped CB\n");
  196. goto err_vm_ctx_fini;
  197. }
  198. rc = hdev->asic_funcs->ctx_init(ctx);
  199. if (rc) {
  200. dev_err(hdev->dev, "ctx_init failed\n");
  201. goto err_cb_va_pool_fini;
  202. }
  203. hl_encaps_sig_mgr_init(&ctx->sig_mgr);
  204. dev_dbg(hdev->dev, "create user context %d\n", ctx->asid);
  205. }
  206. return 0;
  207. err_cb_va_pool_fini:
  208. hl_cb_va_pool_fini(ctx);
  209. err_vm_ctx_fini:
  210. hl_vm_ctx_fini(ctx);
  211. err_asid_free:
  212. if (ctx->asid != HL_KERNEL_ASID_ID)
  213. hl_asid_free(hdev, ctx->asid);
  214. err_hw_block_mem_fini:
  215. hl_hw_block_mem_fini(ctx);
  216. kfree(ctx->cs_pending);
  217. return rc;
  218. }
  219. static int hl_ctx_get_unless_zero(struct hl_ctx *ctx)
  220. {
  221. return kref_get_unless_zero(&ctx->refcount);
  222. }
  223. void hl_ctx_get(struct hl_ctx *ctx)
  224. {
  225. kref_get(&ctx->refcount);
  226. }
  227. int hl_ctx_put(struct hl_ctx *ctx)
  228. {
  229. return kref_put(&ctx->refcount, hl_ctx_do_release);
  230. }
  231. struct hl_ctx *hl_get_compute_ctx(struct hl_device *hdev)
  232. {
  233. struct hl_ctx *ctx = NULL;
  234. struct hl_fpriv *hpriv;
  235. mutex_lock(&hdev->fpriv_list_lock);
  236. list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {
  237. mutex_lock(&hpriv->ctx_lock);
  238. ctx = hpriv->ctx;
  239. if (ctx && !hl_ctx_get_unless_zero(ctx))
  240. ctx = NULL;
  241. mutex_unlock(&hpriv->ctx_lock);
  242. /* There can only be a single user which has opened the compute device, so exit
  243. * immediately once we find its context or if we see that it has been released
  244. */
  245. break;
  246. }
  247. mutex_unlock(&hdev->fpriv_list_lock);
  248. return ctx;
  249. }
  250. /*
  251. * hl_ctx_get_fence_locked - get CS fence under CS lock
  252. *
  253. * @ctx: pointer to the context structure.
  254. * @seq: CS sequences number
  255. *
  256. * @return valid fence pointer on success, NULL if fence is gone, otherwise
  257. * error pointer.
  258. *
  259. * NOTE: this function shall be called with cs_lock locked
  260. */
  261. static struct hl_fence *hl_ctx_get_fence_locked(struct hl_ctx *ctx, u64 seq)
  262. {
  263. struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;
  264. struct hl_fence *fence;
  265. if (seq >= ctx->cs_sequence)
  266. return ERR_PTR(-EINVAL);
  267. if (seq + asic_prop->max_pending_cs < ctx->cs_sequence)
  268. return NULL;
  269. fence = ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)];
  270. hl_fence_get(fence);
  271. return fence;
  272. }
  273. struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
  274. {
  275. struct hl_fence *fence;
  276. spin_lock(&ctx->cs_lock);
  277. fence = hl_ctx_get_fence_locked(ctx, seq);
  278. spin_unlock(&ctx->cs_lock);
  279. return fence;
  280. }
  281. /*
  282. * hl_ctx_get_fences - get multiple CS fences under the same CS lock
  283. *
  284. * @ctx: pointer to the context structure.
  285. * @seq_arr: array of CS sequences to wait for
  286. * @fence: fence array to store the CS fences
  287. * @arr_len: length of seq_arr and fence_arr
  288. *
  289. * @return 0 on success, otherwise non 0 error code
  290. */
  291. int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr,
  292. struct hl_fence **fence, u32 arr_len)
  293. {
  294. struct hl_fence **fence_arr_base = fence;
  295. int i, rc = 0;
  296. spin_lock(&ctx->cs_lock);
  297. for (i = 0; i < arr_len; i++, fence++) {
  298. u64 seq = seq_arr[i];
  299. *fence = hl_ctx_get_fence_locked(ctx, seq);
  300. if (IS_ERR(*fence)) {
  301. dev_err(ctx->hdev->dev,
  302. "Failed to get fence for CS with seq 0x%llx\n",
  303. seq);
  304. rc = PTR_ERR(*fence);
  305. break;
  306. }
  307. }
  308. spin_unlock(&ctx->cs_lock);
  309. if (rc)
  310. hl_fences_put(fence_arr_base, i);
  311. return rc;
  312. }
  313. /*
  314. * hl_ctx_mgr_init - initialize the context manager
  315. *
  316. * @ctx_mgr: pointer to context manager structure
  317. *
  318. * This manager is an object inside the hpriv object of the user process.
  319. * The function is called when a user process opens the FD.
  320. */
  321. void hl_ctx_mgr_init(struct hl_ctx_mgr *ctx_mgr)
  322. {
  323. mutex_init(&ctx_mgr->lock);
  324. idr_init(&ctx_mgr->handles);
  325. }
  326. /*
  327. * hl_ctx_mgr_fini - finalize the context manager
  328. *
  329. * @hdev: pointer to device structure
  330. * @ctx_mgr: pointer to context manager structure
  331. *
  332. * This function goes over all the contexts in the manager and frees them.
  333. * It is called when a process closes the FD.
  334. */
  335. void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *ctx_mgr)
  336. {
  337. struct hl_ctx *ctx;
  338. struct idr *idp;
  339. u32 id;
  340. idp = &ctx_mgr->handles;
  341. idr_for_each_entry(idp, ctx, id)
  342. kref_put(&ctx->refcount, hl_ctx_do_release);
  343. idr_destroy(&ctx_mgr->handles);
  344. mutex_destroy(&ctx_mgr->lock);
  345. }