adsprpc_shared.h 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
  4. * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
  5. */
  6. #ifndef ADSPRPC_SHARED_H
  7. #define ADSPRPC_SHARED_H
  8. #include <linux/types.h>
  9. #include <linux/cdev.h>
  10. #ifdef CONFIG_MSM_ADSPRPC_TRUSTED
  11. #include "../include/uapi/fastrpc_shared.h"
  12. #else
  13. #include "fastrpc_shared.h"
  14. #endif
  15. #define FASTRPC_GLINK_GUID "fastrpcglink-apps-dsp"
  16. #define FASTRPC_SMD_GUID "fastrpcsmd-apps-dsp"
  17. #define DEVICE_NAME "adsprpc-smd"
  18. #define DEVICE_NAME_SECURE "adsprpc-smd-secure"
  19. /* Pre-defined parameter for print gfa structure*/
  20. #define smq_invoke_ctx_params "pid: %d, tgid: %d, handle: %d, sc: 0x%x, fl: %p, fd: %p, magic: %d\n"
  21. #define fastrpc_file_params "fl->tgid: %d, fl->cid: %d, fl->ssrcount: %p, fl->pd: %d, fl->profile: %p, fl->mode: %p, fl->tgid_open: %d, fl->num_cached_buf: %d, num_pers_hdrs: %d, fl->sessionid: %d, fl->servloc_name: %s, fl->file_close: %d, fl->dsp_proc_init: %d,fl->apps: %p, fl->qos_request: %d, fl->dev_minor: %d, fl->debug_buf: %s fl->debug_buf_alloced_attempted: %d, fl->wake_enable: %d, fl->ws_timeout: %d, fl->untrusted_process: %d\n"
  22. #define fastrpc_mmap_params "fd: %d, flags: %p, buf: %p, phys: %p, size : %d, va : %p, map->raddr: %p, len : %d, refs : %d, secure: %d\n"
  23. #define fastrpc_buf_params "buf->fl: %p, buf->phys: %p, buf->virt: %p, buf->size: %d, buf->dma_attr: %ld, buf->raddr: %p, buf->flags: %d, buf->type: %d, buf->in_use: %d\n"
  24. /* Set for buffers that have no virtual mapping in userspace */
  25. #define FASTRPC_ATTR_NOVA 0x1
  26. /* Set for buffers that are NOT dma coherent */
  27. #define FASTRPC_ATTR_NON_COHERENT 0x2
  28. /* Set for buffers that are dma coherent */
  29. #define FASTRPC_ATTR_COHERENT 0x4
  30. /* Fastrpc attribute for keeping the map persistent */
  31. #define FASTRPC_ATTR_KEEP_MAP 0x8
  32. /* Fastrpc attribute for no mapping of fd */
  33. #define FASTRPC_ATTR_NOMAP (16)
  34. /*
  35. * Fastrpc attribute to skip flush by fastrpc
  36. */
  37. #define FASTRPC_ATTR_FORCE_NOFLUSH (32)
  38. /*
  39. * Fastrpc attribute to skip invalidate by fastrpc
  40. */
  41. #define FASTRPC_ATTR_FORCE_NOINVALIDATE (64)
  42. /* Driver should operate in parallel with the co-processor */
  43. #define FASTRPC_MODE_PARALLEL 0
  44. /* Driver should operate in serial mode with the co-processor */
  45. #define FASTRPC_MODE_SERIAL 1
  46. /* Driver should operate in profile mode with the co-processor */
  47. #define FASTRPC_MODE_PROFILE 2
  48. /* Set FastRPC session ID to 1 */
  49. #define FASTRPC_MODE_SESSION 4
  50. /* Retrives number of input buffers from the scalars parameter */
  51. #define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
  52. /* Retrives number of output buffers from the scalars parameter */
  53. #define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
  54. /* Retrives number of input handles from the scalars parameter */
  55. #define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
  56. /* Retrives number of output handles from the scalars parameter */
  57. #define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
  58. /* Remote domains ID */
  59. #define ADSP_DOMAIN_ID (0)
  60. #define MDSP_DOMAIN_ID (1)
  61. #define SDSP_DOMAIN_ID (2)
  62. #define CDSP_DOMAIN_ID (3)
  63. #define CDSP1_DOMAIN_ID (4)
  64. #define NUM_CHANNELS 5 /* adsp, mdsp, slpi, cdsp, cdsp1*/
  65. #define NUM_SESSIONS 14 /* max 11 compute, 3 cpz */
  66. /* Default maximum sessions allowed per process */
  67. #define DEFAULT_MAX_SESS_PER_PROC 4
  68. #define VALID_FASTRPC_CID(cid) \
  69. (cid >= ADSP_DOMAIN_ID && cid < NUM_CHANNELS)
  70. #define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) +\
  71. REMOTE_SCALARS_OUTBUFS(sc) +\
  72. REMOTE_SCALARS_INHANDLES(sc) +\
  73. REMOTE_SCALARS_OUTHANDLES(sc))
  74. #define REMOTE_SCALARS_MAKEX(attr, method, in, out, oin, oout) \
  75. ((((uint32_t) (attr) & 0x7) << 29) | \
  76. (((uint32_t) (method) & 0x1f) << 24) | \
  77. (((uint32_t) (in) & 0xff) << 16) | \
  78. (((uint32_t) (out) & 0xff) << 8) | \
  79. (((uint32_t) (oin) & 0x0f) << 4) | \
  80. ((uint32_t) (oout) & 0x0f))
  81. #define REMOTE_SCALARS_MAKE(method, in, out) \
  82. REMOTE_SCALARS_MAKEX(0, method, in, out, 0, 0)
  83. #ifdef VERIFY_PRINT_ERROR
  84. #define VERIFY_EPRINTF(format, ...) pr_err(format, ##__VA_ARGS__)
  85. #else
  86. #define VERIFY_EPRINTF(format, args) ((void)0)
  87. #endif
  88. #ifndef VERIFY_PRINT_INFO
  89. #define VERIFY_IPRINTF(args) ((void)0)
  90. #endif
  91. #ifndef VERIFY
  92. #define __STR__(x) #x ":"
  93. #define __TOSTR__(x) __STR__(x)
  94. #define __FILE_LINE__ __FILE__ ":" __TOSTR__(__LINE__)
  95. #define __ADSPRPC_LINE__ "adsprpc:" __TOSTR__(__LINE__)
  96. #define VERIFY(err, val) \
  97. do {\
  98. VERIFY_IPRINTF(__FILE_LINE__"info: calling: " #val "\n");\
  99. if ((val) == 0) {\
  100. (err) = (err) == 0 ? -1 : (err);\
  101. VERIFY_EPRINTF(__ADSPRPC_LINE__" error: %d: "#val "\n", (err));\
  102. } else {\
  103. VERIFY_IPRINTF(__FILE_LINE__"info: passed: " #val "\n");\
  104. } \
  105. } while (0)
  106. #endif
  107. #define ADSPRPC_ERR(fmt, args...)\
  108. pr_err("Error: adsprpc (%d): %s: %s: " fmt, __LINE__,\
  109. current->comm, __func__, ##args)
  110. #define ADSPRPC_INFO(fmt, args...)\
  111. pr_info("Info: adsprpc (%d): %s: %s: " fmt, __LINE__,\
  112. current->comm, __func__, ##args)
  113. #define ADSPRPC_WARN(fmt, args...)\
  114. pr_warn("Warning: adsprpc (%d): %s: %s: " fmt, __LINE__,\
  115. current->comm, __func__, ##args)
  116. #define ADSPRPC_DEBUG(fmt, args...)\
  117. pr_debug("Debug: adsprpc (%d): %s: %s: " fmt, __LINE__,\
  118. current->comm, __func__, ##args)
  119. #define DEBUG_PRINT_SIZE_LIMIT (512*1024)
  120. #define remote_arg64_t union remote_arg64
  121. struct remote_buf64 {
  122. uint64_t pv;
  123. uint64_t len;
  124. };
  125. struct remote_dma_handle64 {
  126. int fd;
  127. uint32_t offset;
  128. uint32_t len;
  129. };
  130. union remote_arg64 {
  131. struct remote_buf64 buf;
  132. struct remote_dma_handle64 dma;
  133. uint32_t h;
  134. };
  135. struct remote_buf {
  136. void *pv; /* buffer pointer */
  137. size_t len; /* length of buffer */
  138. };
  139. /* structure to hold fd and size of buffer shared with DSP,
  140. * which contains inital debug parameters that needs to be passed
  141. * during process initialization.
  142. */
  143. struct fastrpc_proc_sharedbuf_info {
  144. int buf_fd;
  145. int buf_size;
  146. };
  147. struct remote_dma_handle {
  148. int fd;
  149. uint32_t offset;
  150. };
  151. union remote_arg {
  152. struct remote_buf buf; /* buffer info */
  153. struct remote_dma_handle dma;
  154. uint32_t h; /* remote handle */
  155. };
  156. struct fastrpc_async_job {
  157. uint32_t isasyncjob; /* flag to distinguish async job */
  158. uint64_t jobid; /* job id generated by user */
  159. uint32_t reserved; /* reserved */
  160. };
  161. /* FastRPC ioctl structure to set session related info */
  162. struct fastrpc_proc_sess_info {
  163. uint32_t domain_id; /* Set the remote subsystem, Domain ID of the session */
  164. uint32_t session_id; /* Unused, Set the Session ID on remote subsystem */
  165. uint32_t pd_type; /* Set the process type on remote subsystem */
  166. uint32_t sharedcb; /* Unused, Session can share context bank with other sessions */
  167. };
  168. /* INIT a new process or attach to guestos */
  169. enum fastrpc_init_flags {
  170. FASTRPC_INIT_NO_CREATE = -1,
  171. FASTRPC_INIT_ATTACH = 0,
  172. FASTRPC_INIT_CREATE = 1,
  173. FASTRPC_INIT_CREATE_STATIC = 2,
  174. FASTRPC_INIT_ATTACH_SENSORS = 3,
  175. };
  176. enum fastrpc_invoke2_type {
  177. FASTRPC_INVOKE2_ASYNC = 1,
  178. FASTRPC_INVOKE2_ASYNC_RESPONSE = 2,
  179. FASTRPC_INVOKE2_KERNEL_OPTIMIZATIONS,
  180. FASTRPC_INVOKE2_STATUS_NOTIF,
  181. FASTRPC_INVOKE2_PROC_SHAREDBUF_INFO,
  182. /* Set session info of remote sub system */
  183. FASTRPC_INVOKE2_SESS_INFO,
  184. };
  185. /**
  186. * Control flags for mapping memory on DSP user process
  187. */
  188. enum fastrpc_map_flags {
  189. /**
  190. * Map memory pages with RW- permission and CACHE WRITEBACK.
  191. * The driver is responsible for cache maintenance when passed
  192. * the buffer to FastRPC calls. Same virtual address will be
  193. * assigned for subsequent FastRPC calls.
  194. */
  195. FASTRPC_MAP_STATIC = 0,
  196. /* Reserved */
  197. FASTRPC_MAP_RESERVED,
  198. /**
  199. * Map memory pages with RW- permission and CACHE WRITEBACK.
  200. * Mapping tagged with a file descriptor. User is responsible for
  201. * CPU and DSP cache maintenance for the buffer. Get virtual address
  202. * of buffer on DSP using HAP_mmap_get() and HAP_mmap_put() APIs.
  203. */
  204. FASTRPC_MAP_FD = 2,
  205. /**
  206. * Mapping delayed until user call HAP_mmap() and HAP_munmap()
  207. * functions on DSP. It is useful to map a buffer with cache modes
  208. * other than default modes. User is responsible for CPU and DSP
  209. * cache maintenance for the buffer.
  210. */
  211. FASTRPC_MAP_FD_DELAYED,
  212. /**
  213. * This flag is used to skip CPU mapping,
  214. * otherwise behaves similar to FASTRPC_MAP_FD_DELAYED flag.
  215. */
  216. FASTRPC_MAP_FD_NOMAP = 16,
  217. FASTRPC_MAP_MAX,
  218. };
  219. /*
  220. * This enum is shared with DSP. So, existing values should NOT
  221. * be modified. Only new members can be added.
  222. */
  223. enum dsp_map_flags {
  224. /* Add memory to static PD pool, protection thru XPU */
  225. ADSP_MMAP_HEAP_ADDR = 4,
  226. /* MAP static DMA buffer on DSP User PD */
  227. ADSP_MMAP_DMA_BUFFER = 6,
  228. /* Add memory to static PD pool, protection thru hypervisor */
  229. ADSP_MMAP_REMOTE_HEAP_ADDR = 8,
  230. /* Add memory to userPD pool, for user heap */
  231. ADSP_MMAP_ADD_PAGES = 0x1000,
  232. /* Add memory to userPD pool, for LLC heap */
  233. ADSP_MMAP_ADD_PAGES_LLC = 0x3000,
  234. /* Map persistent header buffer on DSP */
  235. ADSP_MMAP_PERSIST_HDR = 0x4000,
  236. };
  237. enum fastrpc_control_type {
  238. FASTRPC_CONTROL_LATENCY = 1,
  239. /* Share SMMU context bank */
  240. FASTRPC_CONTROL_SMMU = 2,
  241. FASTRPC_CONTROL_KALLOC = 3,
  242. FASTRPC_CONTROL_WAKELOCK = 4,
  243. FASTRPC_CONTROL_PM = 5,
  244. /* Clean process on DSP */
  245. FASTRPC_CONTROL_DSPPROCESS_CLEAN = 6,
  246. FASTRPC_CONTROL_RPC_POLL = 7,
  247. FASTRPC_CONTROL_ASYNC_WAKE = 8,
  248. FASTRPC_CONTROL_NOTIF_WAKE = 9,
  249. };
  250. #define FASTRPC_MAX_DSP_ATTRIBUTES (256)
  251. #define FASTRPC_MAX_ATTRIBUTES (260)
  252. enum fastrpc_dsp_capability {
  253. ASYNC_FASTRPC_CAP = 9,
  254. DMA_HANDLE_REVERSE_RPC_CAP = 129,
  255. };
  256. struct smq_null_invoke {
  257. uint64_t ctx; /* invoke caller context */
  258. uint32_t handle; /* handle to invoke */
  259. uint32_t sc; /* scalars structure describing the data */
  260. };
  261. struct smq_phy_page {
  262. uint64_t addr; /* physical address */
  263. uint64_t size; /* size of contiguous region */
  264. };
  265. struct smq_invoke_buf {
  266. int num; /* number of contiguous regions */
  267. int pgidx; /* index to start of contiguous region */
  268. };
  269. struct smq_invoke {
  270. struct smq_null_invoke header;
  271. struct smq_phy_page page; /* remote arg and list of pages address */
  272. };
  273. struct smq_msg {
  274. uint32_t pid; /* process group id */
  275. uint32_t tid; /* thread id */
  276. struct smq_invoke invoke;
  277. };
  278. struct smq_invoke_rsp {
  279. uint64_t ctx; /* invoke caller context */
  280. int retval; /* invoke return value */
  281. };
  282. enum fastrpc_response_flags {
  283. NORMAL_RESPONSE = 0,
  284. EARLY_RESPONSE = 1,
  285. USER_EARLY_SIGNAL = 2,
  286. COMPLETE_SIGNAL = 3,
  287. STATUS_RESPONSE = 4,
  288. POLL_MODE = 5,
  289. };
  290. enum fastrpc_process_create_state {
  291. PROCESS_CREATE_DEFAULT = 0, /* Process is not created */
  292. PROCESS_CREATE_IS_INPROGRESS = 1, /* Process creation is in progress */
  293. PROCESS_CREATE_SUCCESS = 2, /* Process creation is successful */
  294. };
  295. struct smq_invoke_rspv2 {
  296. uint64_t ctx; /* invoke caller context */
  297. int retval; /* invoke return value */
  298. uint32_t flags; /* early response flags */
  299. uint32_t early_wake_time; /* user predicted early wakeup time in us */
  300. uint32_t version; /* Version number for validation */
  301. };
  302. enum fastrpc_status_flags {
  303. FASTRPC_USERPD_UP = 0,
  304. FASTRPC_USERPD_EXIT = 1,
  305. FASTRPC_USERPD_FORCE_KILL = 2,
  306. FASTRPC_USERPD_EXCEPTION = 3,
  307. FASTRPC_DSP_SSR = 4,
  308. };
  309. struct smq_notif_rspv3 {
  310. uint64_t ctx; /* response context */
  311. uint32_t type; /* Notification type */
  312. int pid; /* user process pid */
  313. uint32_t status; /* userpd status notification */
  314. };
  315. enum fastrpc_process_exit_states {
  316. /* Process Default State */
  317. FASTRPC_PROCESS_DEFAULT_STATE = 0,
  318. /* Process exit initiated */
  319. FASTRPC_PROCESS_EXIT_START = 1,
  320. /* Process exit issued to DSP */
  321. FASTRPC_PROCESS_DSP_EXIT_INIT = 2,
  322. /* Process exit in DSP complete */
  323. FASTRPC_PROCESS_DSP_EXIT_COMPLETE = 3,
  324. /* Process exit in DSP error */
  325. FASTRPC_PROCESS_DSP_EXIT_ERROR = 4,
  326. };
  327. /*
  328. * Process types on remote subsystem
  329. * Always add new PD types at the end, before MAX_PD_TYPE
  330. */
  331. #define DEFAULT_UNUSED 0 /* pd type not configured for context banks */
  332. #define ROOT_PD 1 /* Root PD */
  333. #define AUDIO_STATICPD 2 /* ADSP Audio Static PD */
  334. #define SENSORS_STATICPD 3 /* ADSP Sensors Static PD */
  335. #define SECURE_STATICPD 4 /* CDSP Secure Static PD */
  336. #define OIS_STATICPD 5 /* ADSP OIS Static PD */
  337. #define CPZ_USERPD 6 /* CDSP CPZ USER PD */
  338. #define USERPD 7 /* DSP User Dynamic PD */
  339. #define GUEST_OS_SHARED 8 /* Legacy Guest OS Shared */
  340. #define MAX_PD_TYPE 9 /* Max PD type */
  341. struct fastrpc_file;
  342. int fastrpc_transport_send(int cid, void *rpc_msg, uint32_t rpc_msg_size, int tvm_remote_domain);
  343. inline int fastrpc_handle_rpc_response(void *data, int len, int cid);
  344. inline int verify_transport_device(int cid, int tvm_remote_domain);
  345. int fastrpc_transport_init(void);
  346. void fastrpc_transport_deinit(void);
  347. void fastrpc_transport_session_init(int cid, char *subsys);
  348. void fastrpc_transport_session_deinit(int cid);
  349. int fastrpc_wait_for_transport_interrupt(int cid, unsigned int flags);
  350. int fastrpc_set_tvm_remote_domain(struct fastrpc_file *fl, struct fastrpc_ioctl_init *init);
  351. void fastrpc_restart_drivers(int cid);
  352. static inline struct smq_invoke_buf *smq_invoke_buf_start(remote_arg64_t *pra,
  353. uint32_t sc)
  354. {
  355. unsigned int len = REMOTE_SCALARS_LENGTH(sc);
  356. return (struct smq_invoke_buf *)(&pra[len]);
  357. }
  358. static inline struct smq_phy_page *smq_phy_page_start(uint32_t sc,
  359. struct smq_invoke_buf *buf)
  360. {
  361. unsigned int nTotal = REMOTE_SCALARS_LENGTH(sc);
  362. return (struct smq_phy_page *)(&buf[nTotal]);
  363. }
  364. /*
  365. * Fastrpc context ID bit-map:
  366. *
  367. * bits 0-3 : type of remote PD
  368. * bit 4 : type of job (sync/async)
  369. * bit 5 : reserved
  370. * bits 6-15 : index in context table
  371. * bits 16-63 : incrementing context ID
  372. */
  373. #define FASTRPC_CTX_MAX (1024)
  374. /* Length of glink transaction history to store */
  375. #define GLINK_MSG_HISTORY_LEN (128)
  376. /* Type of fastrpc DMA bufs sent to DSP */
  377. enum fastrpc_buf_type {
  378. METADATA_BUF,
  379. COPYDATA_BUF,
  380. INITMEM_BUF,
  381. USERHEAP_BUF,
  382. };
  383. /* Types of RPC calls to DSP */
  384. enum fastrpc_msg_type {
  385. /* 64 bit user application invoke message */
  386. USER_MSG = 0,
  387. /* kernel invoke message with zero pid */
  388. KERNEL_MSG_WITH_ZERO_PID,
  389. /* kernel invoke message with non zero pid to kill the PD in DSP */
  390. KERNEL_MSG_WITH_NONZERO_PID,
  391. /* 32 bit user application invoke message */
  392. COMPAT_MSG,
  393. };
  394. /* Fastrpc remote pd type */
  395. enum fastrpc_remote_pd_type {
  396. FASTRPC_ROOT_PD = 0,
  397. FASTRPC_USER_PD,
  398. FASTRPC_SENSORS_PD,
  399. };
  400. #define DSPSIGNAL_TIMEOUT_NONE 0xffffffff
  401. #define DSPSIGNAL_NUM_SIGNALS 1024
  402. // Signal state and completions are stored in groups of DSPSIGNAL_GROUP_SIZE.
  403. // Must be a power of two.
  404. #define DSPSIGNAL_GROUP_SIZE 256
  405. struct secure_vm {
  406. int *vmid;
  407. int *vmperm;
  408. int vmcount;
  409. };
  410. struct gid_list {
  411. unsigned int *gids;
  412. unsigned int gidcount;
  413. };
  414. struct fastrpc_buf {
  415. struct hlist_node hn;
  416. struct hlist_node hn_rem;
  417. struct hlist_node hn_init;
  418. struct fastrpc_file *fl;
  419. void *virt;
  420. uint64_t phys;
  421. size_t size;
  422. unsigned long dma_attr;
  423. uintptr_t raddr;
  424. uint32_t flags;
  425. int type; /* One of "fastrpc_buf_type" */
  426. bool in_use; /* Used only for persistent header buffers */
  427. struct timespec64 buf_start_time;
  428. struct timespec64 buf_end_time;
  429. };
  430. struct fastrpc_ctx_lst;
  431. struct fastrpc_tx_msg {
  432. struct smq_msg msg; /* Msg sent to remote subsystem */
  433. int transport_send_err; /* transport error */
  434. int64_t ns; /* Timestamp (in ns) of msg */
  435. uint64_t xo_time_in_us; /* XO Timestamp (in us) of sent message */
  436. uint64_t xo_time_in_us_interrupted; /* XO Timestamp (in us) of interrupted ctx */
  437. uint64_t xo_time_in_us_restored; /* XO Timestamp (in us) of restored ctx */
  438. };
  439. struct fastrpc_rx_msg {
  440. struct smq_invoke_rspv2 rsp; /* Response from remote subsystem */
  441. int64_t ns; /* Timestamp (in ns) of response */
  442. uint64_t xo_time_in_us; /* XO Timestamp (in us) of response */
  443. };
  444. struct fastrpc_transport_log {
  445. unsigned int tx_index; /* Current index of 'tx_msgs' array */
  446. unsigned int rx_index; /* Current index of 'rx_msgs' array */
  447. /* Rolling history of messages sent to remote subsystem */
  448. struct fastrpc_tx_msg tx_msgs[GLINK_MSG_HISTORY_LEN];
  449. /* Rolling history of responses from remote subsystem */
  450. struct fastrpc_rx_msg rx_msgs[GLINK_MSG_HISTORY_LEN];
  451. spinlock_t lock;
  452. };
  453. struct overlap {
  454. uintptr_t start;
  455. uintptr_t end;
  456. int raix;
  457. uintptr_t mstart;
  458. uintptr_t mend;
  459. uintptr_t offset;
  460. int do_cmo; /*used for cache maintenance of inrout buffers*/
  461. };
  462. struct fastrpc_perf {
  463. uint64_t count;
  464. uint64_t flush;
  465. uint64_t map;
  466. uint64_t copy;
  467. uint64_t link;
  468. uint64_t getargs;
  469. uint64_t putargs;
  470. uint64_t invargs;
  471. uint64_t invoke;
  472. uint64_t tid;
  473. };
  474. struct smq_notif_rsp {
  475. struct list_head notifn;
  476. int domain;
  477. int session;
  478. enum fastrpc_status_flags status;
  479. };
  480. struct smq_invoke_ctx {
  481. struct hlist_node hn;
  482. /* Async node to add to async job ctx list */
  483. struct hlist_node asyncn;
  484. struct completion work;
  485. int retval;
  486. int pid;
  487. int tgid;
  488. remote_arg_t *lpra;
  489. remote_arg64_t *rpra;
  490. remote_arg64_t *lrpra; /* Local copy of rpra for put_args */
  491. int *fds;
  492. unsigned int *attrs;
  493. struct fastrpc_mmap **maps;
  494. struct fastrpc_buf *buf;
  495. struct fastrpc_buf *copybuf; /*used to copy non-ion buffers */
  496. size_t used;
  497. struct fastrpc_file *fl;
  498. uint32_t handle;
  499. uint32_t sc;
  500. struct overlap *overs;
  501. struct overlap **overps;
  502. struct smq_msg msg;
  503. uint32_t *crc;
  504. uint64_t *perf_kernel;
  505. uint64_t *perf_dsp;
  506. unsigned int magic;
  507. uint64_t ctxid;
  508. struct fastrpc_perf *perf;
  509. /* response flags from remote processor */
  510. enum fastrpc_response_flags rsp_flags;
  511. /* user hint of completion time in us */
  512. uint32_t early_wake_time;
  513. /* work done status flag */
  514. bool is_work_done;
  515. /* Store Async job in the context*/
  516. struct fastrpc_async_job asyncjob;
  517. /* Async early flag to check the state of context */
  518. bool is_early_wakeup;
  519. uint32_t sc_interrupted;
  520. struct fastrpc_file *fl_interrupted;
  521. uint32_t handle_interrupted;
  522. uint64_t xo_time_in_us_created; /* XO Timestamp (in us) of ctx creation */
  523. uint64_t xo_time_in_us_interrupted; /* XO Timestamp (in us) of interrupted ctx */
  524. uint64_t xo_time_in_us_restored; /* XO Timestamp (in us) of restored ctx */
  525. int tx_index; /* index of current ctx in channel gmsg_log array */
  526. bool is_job_sent_to_remote_ss; /* Flag to check if job is sent to remote sub system */
  527. };
  528. struct fastrpc_ctx_lst {
  529. struct hlist_head pending;
  530. struct hlist_head interrupted;
  531. /* Number of active contexts queued to DSP */
  532. uint32_t num_active_ctxs;
  533. /* Queue which holds all async job contexts of process */
  534. struct hlist_head async_queue;
  535. /* Queue which holds all status notifications of process */
  536. struct list_head notif_queue;
  537. };
  538. struct fastrpc_smmu {
  539. struct device *dev;
  540. const char *dev_name;
  541. int cb;
  542. int enabled;
  543. int faults;
  544. int secure;
  545. int coherent;
  546. int sharedcb;
  547. int pd_type; /* Process type on remote sub system */
  548. /* gen pool for QRTR */
  549. struct gen_pool *frpc_genpool;
  550. /* fastrpc gen pool buffer */
  551. struct fastrpc_buf *frpc_genpool_buf;
  552. /* fastrpc gen pool buffer fixed IOVA */
  553. unsigned long genpool_iova;
  554. /* fastrpc gen pool buffer size */
  555. size_t genpool_size;
  556. };
  557. struct fastrpc_session_ctx {
  558. struct device *dev;
  559. struct fastrpc_smmu smmu;
  560. int used;
  561. };
  562. struct fastrpc_static_pd {
  563. char *servloc_name;
  564. char *spdname;
  565. void *pdrhandle;
  566. uint64_t pdrcount;
  567. uint64_t prevpdrcount;
  568. atomic_t ispdup;
  569. int cid;
  570. wait_queue_head_t wait_for_pdup;
  571. };
  572. struct fastrpc_dsp_capabilities {
  573. uint32_t is_cached; //! Flag if dsp attributes are cached
  574. uint32_t dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES];
  575. };
  576. struct fastrpc_channel_ctx {
  577. char *name;
  578. char *subsys;
  579. struct device *dev;
  580. struct fastrpc_session_ctx session[NUM_SESSIONS];
  581. struct fastrpc_static_pd spd[NUM_SESSIONS];
  582. struct completion work;
  583. struct completion workport;
  584. struct notifier_block nb;
  585. struct mutex smd_mutex;
  586. uint64_t sesscount;
  587. uint64_t ssrcount;
  588. int in_hib;
  589. void *handle;
  590. uint64_t prevssrcount;
  591. int subsystemstate;
  592. int vmid;
  593. struct secure_vm rhvm;
  594. void *rh_dump_dev;
  595. /* Indicates, if channel is restricted to secure node only */
  596. int secure;
  597. /* Indicates whether the channel supports unsigned PD */
  598. bool unsigned_support;
  599. struct fastrpc_dsp_capabilities dsp_cap_kernel;
  600. /* cpu capabilities shared to DSP */
  601. uint64_t cpuinfo_todsp;
  602. bool cpuinfo_status;
  603. struct smq_invoke_ctx *ctxtable[FASTRPC_CTX_MAX];
  604. spinlock_t ctxlock;
  605. struct fastrpc_transport_log gmsg_log;
  606. struct hlist_head initmems;
  607. /* Store gfa structure debug details */
  608. struct fastrpc_buf *buf;
  609. };
  610. struct fastrpc_apps {
  611. struct fastrpc_channel_ctx *channel;
  612. struct cdev cdev;
  613. struct class *class;
  614. struct smq_phy_page range;
  615. struct hlist_head maps;
  616. uint32_t staticpd_flags;
  617. dev_t dev_no;
  618. int compat;
  619. struct hlist_head drivers;
  620. spinlock_t hlock;
  621. struct device *dev;
  622. /* Indicates fastrpc device node info */
  623. struct device *dev_fastrpc;
  624. unsigned int latency;
  625. int transport_initialized;
  626. /* Flag to determine fastrpc bus registration */
  627. int fastrpc_bus_register;
  628. bool legacy_remote_heap;
  629. /* Unique job id for each message */
  630. uint64_t jobid[NUM_CHANNELS];
  631. struct gid_list gidlist;
  632. struct device *secure_dev;
  633. struct device *non_secure_dev;
  634. /* Secure subsystems like ADSP/SLPI will use secure client */
  635. struct wakeup_source *wake_source_secure;
  636. /* Non-secure subsystem like CDSP will use regular client */
  637. struct wakeup_source *wake_source;
  638. uint32_t duplicate_rsp_err_cnt;
  639. uint32_t max_size_limit;
  640. struct hlist_head frpc_devices;
  641. struct hlist_head frpc_drivers;
  642. struct mutex mut_uid;
  643. /* Indicates nsp status */
  644. int fastrpc_nsp_status;
  645. /* Indicates secure context bank to be shared */
  646. int share_securecb;
  647. /* Indicates process type is configured for SMMU context bank */
  648. bool cb_pd_type;
  649. /* Number of lowest capacity cores for given platform */
  650. unsigned int lowest_capacity_core_count;
  651. /* Flag to check if PM QoS vote needs to be done for only one core */
  652. bool single_core_latency_vote;
  653. /* Maximum sessions allowed to be created per process */
  654. uint32_t max_sess_per_proc;
  655. };
  656. struct fastrpc_mmap {
  657. struct hlist_node hn;
  658. struct fastrpc_file *fl;
  659. struct fastrpc_apps *apps;
  660. int fd;
  661. uint32_t flags;
  662. struct dma_buf *buf;
  663. struct sg_table *table;
  664. struct dma_buf_attachment *attach;
  665. struct ion_handle *handle;
  666. uint64_t phys;
  667. size_t size;
  668. uintptr_t va;
  669. size_t len;
  670. int refs;
  671. uintptr_t raddr;
  672. int secure;
  673. bool is_persistent; /* the map is persistenet across sessions */
  674. int frpc_md_index; /* Minidump unique index */
  675. uintptr_t attr;
  676. bool in_use; /* Indicates if persistent map is in use*/
  677. struct timespec64 map_start_time;
  678. struct timespec64 map_end_time;
  679. /* Mapping for fastrpc shell */
  680. bool is_filemap;
  681. bool is_dumped; /* flag to indicate map is dumped during SSR */
  682. char *servloc_name; /* Indicate which daemon mapped this */
  683. /* Indicates map is being used by a pending RPC call */
  684. unsigned int ctx_refs;
  685. /* Map in use for dma handle */
  686. unsigned int dma_handle_refs;
  687. };
  688. enum fastrpc_perfkeys {
  689. PERF_COUNT = 0,
  690. PERF_FLUSH = 1,
  691. PERF_MAP = 2,
  692. PERF_COPY = 3,
  693. PERF_LINK = 4,
  694. PERF_GETARGS = 5,
  695. PERF_PUTARGS = 6,
  696. PERF_INVARGS = 7,
  697. PERF_INVOKE = 8,
  698. PERF_TID = 9,
  699. PERF_KEY_MAX = 10,
  700. };
  701. struct fastrpc_notif_queue {
  702. /* Number of pending status notifications in queue */
  703. atomic_t notif_queue_count;
  704. /* Wait queue to synchronize notifier thread and response */
  705. wait_queue_head_t notif_wait_queue;
  706. /* IRQ safe spin lock for protecting notif queue */
  707. spinlock_t nqlock;
  708. };
  709. enum fastrpc_dspsignal_state {
  710. DSPSIGNAL_STATE_UNUSED = 0,
  711. DSPSIGNAL_STATE_PENDING,
  712. DSPSIGNAL_STATE_SIGNALED,
  713. DSPSIGNAL_STATE_CANCELED
  714. };
  715. struct fastrpc_dspsignal {
  716. struct completion comp;
  717. int state;
  718. };
  719. struct memory_snapshot {
  720. /* Total size of heap buffers allocated in userspace */
  721. size_t heap_bufs_size;
  722. /* Total size of non-heap buffers allocated in userspace */
  723. size_t nonheap_bufs_size;
  724. };
  725. struct fastrpc_file {
  726. struct hlist_node hn;
  727. spinlock_t hlock;
  728. struct hlist_head maps;
  729. struct hlist_head cached_bufs;
  730. uint32_t num_cached_buf;
  731. struct hlist_head remote_bufs;
  732. struct fastrpc_ctx_lst clst;
  733. struct fastrpc_session_ctx *sctx;
  734. struct fastrpc_buf *init_mem;
  735. struct kref refcount;
  736. /* No. of persistent headers */
  737. unsigned int num_pers_hdrs;
  738. /* Pre-allocated header buffer */
  739. struct fastrpc_buf *pers_hdr_buf;
  740. /* Pre-allocated buffer divided into N chunks */
  741. struct fastrpc_buf *hdr_bufs;
  742. /* Store snapshot of memory occupied by different buffers */
  743. struct memory_snapshot mem_snap;
  744. struct fastrpc_session_ctx *secsctx;
  745. uint32_t mode;
  746. uint32_t profile;
  747. int sessionid;
  748. int tgid_open; /* Process ID during device open */
  749. int tgid; /* Process ID that uses device for RPC calls */
  750. /* Unique HLOS process ID created by fastrpc for each client */
  751. int tgid_frpc;
  752. int cid;
  753. int tvm_remote_domain;
  754. uint64_t ssrcount;
  755. int pd;
  756. char *servloc_name;
  757. int file_close;
  758. int dsp_proc_init;
  759. int sharedcb;
  760. int pd_type; /* Process type on remote subsystem */
  761. struct fastrpc_apps *apps;
  762. struct dentry *debugfs_file;
  763. struct dev_pm_qos_request *dev_pm_qos_req;
  764. int qos_request;
  765. struct mutex map_mutex;
  766. struct mutex internal_map_mutex;
  767. /* Identifies the device (MINOR_NUM_DEV / MINOR_NUM_SECURE_DEV) */
  768. int dev_minor;
  769. char *debug_buf;
  770. /* Flag to indicate attempt has been made to allocate memory for debug_buf*/
  771. int debug_buf_alloced_attempted;
  772. /* Flag to enable PM wake/relax voting for every remote invoke */
  773. int wake_enable;
  774. struct gid_list gidlist;
  775. /* Number of jobs pending in Async Queue */
  776. atomic_t async_queue_job_count;
  777. /* Async wait queue to synchronize glink response and async thread */
  778. wait_queue_head_t async_wait_queue;
  779. /* IRQ safe spin lock for protecting async queue */
  780. spinlock_t aqlock;
  781. /* Process status notification queue */
  782. struct fastrpc_notif_queue proc_state_notif;
  783. uint32_t ws_timeout;
  784. bool untrusted_process;
  785. struct fastrpc_device *device;
  786. /* Process kill will wait on bus driver invoke thread to complete its process */
  787. struct completion dma_invoke;
  788. /* Flag to indicate invoke pending */
  789. bool is_dma_invoke_pend;
  790. /* Flag to indicate type of process (static, dynamic) */
  791. uint32_t proc_flags;
  792. /* If set, threads will poll for DSP response instead of glink wait */
  793. bool poll_mode;
  794. /* Threads poll for specified timeout and fall back to glink wait */
  795. uint32_t poll_timeout;
  796. /* Flag to indicate dynamic process creation status*/
  797. enum fastrpc_process_create_state dsp_process_state;
  798. bool is_unsigned_pd;
  799. /* Completion objects and state for dspsignals */
  800. struct fastrpc_dspsignal *signal_groups[DSPSIGNAL_NUM_SIGNALS / DSPSIGNAL_GROUP_SIZE];
  801. spinlock_t dspsignals_lock;
  802. struct mutex signal_create_mutex;
  803. struct completion shutdown;
  804. /* Flag to indicate notif thread exit requested*/
  805. bool exit_notif;
  806. /* Flag to indicate async thread exit requested*/
  807. bool exit_async;
  808. /*
  809. * structure to hold fd and size of buffer shared with DSP,
  810. * which contains initial debug configurations and other initial
  811. * config paramters.
  812. */
  813. struct fastrpc_proc_sharedbuf_info sharedbuf_info;
  814. /* Flag to indicate 4 session support available */
  815. bool multi_session_support;
  816. /* Flag to indicate session info is set */
  817. bool set_session_info;
  818. };
  819. int fastrpc_internal_invoke(struct fastrpc_file *fl, uint32_t mode,
  820. uint32_t kernel,
  821. struct fastrpc_ioctl_invoke_async *inv);
  822. int fastrpc_internal_invoke2(struct fastrpc_file *fl,
  823. struct fastrpc_ioctl_invoke2 *inv2, bool is_compat);
  824. int fastrpc_internal_munmap(struct fastrpc_file *fl,
  825. struct fastrpc_ioctl_munmap *ud);
  826. int fastrpc_internal_mem_map(struct fastrpc_file *fl,
  827. struct fastrpc_ioctl_mem_map *ud);
  828. int fastrpc_internal_mem_unmap(struct fastrpc_file *fl,
  829. struct fastrpc_ioctl_mem_unmap *ud);
  830. int fastrpc_internal_mmap(struct fastrpc_file *fl,
  831. struct fastrpc_ioctl_mmap *ud);
  832. int fastrpc_init_process(struct fastrpc_file *fl,
  833. struct fastrpc_ioctl_init_attrs *uproc);
  834. int fastrpc_get_info(struct fastrpc_file *fl, uint32_t *info);
  835. int fastrpc_internal_control(struct fastrpc_file *fl,
  836. struct fastrpc_ioctl_control *cp);
  837. int fastrpc_setmode(unsigned long ioctl_param,
  838. struct fastrpc_file *fl);
  839. int fastrpc_get_info_from_kernel(
  840. struct fastrpc_ioctl_capability *cap,
  841. struct fastrpc_file *fl);
  842. int fastrpc_dspsignal_signal(struct fastrpc_file *fl,
  843. struct fastrpc_ioctl_dspsignal_signal *sig);
  844. int fastrpc_dspsignal_wait(struct fastrpc_file *fl,
  845. struct fastrpc_ioctl_dspsignal_wait *wait);
  846. int fastrpc_dspsignal_create(struct fastrpc_file *fl,
  847. struct fastrpc_ioctl_dspsignal_create *create);
  848. int fastrpc_dspsignal_destroy(struct fastrpc_file *fl,
  849. struct fastrpc_ioctl_dspsignal_destroy *destroy);
  850. int fastrpc_dspsignal_cancel_wait(struct fastrpc_file *fl,
  851. struct fastrpc_ioctl_dspsignal_cancel_wait *cancel);
  852. void fastrpc_rproc_trace_events(const char *name, const char *event,
  853. const char *subevent);
  854. #endif