ubwcp_main.c 73 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/kernel.h>
  7. #include <linux/dma-buf.h>
  8. #include <linux/slab.h>
  9. #include <linux/cdev.h>
  10. #include <linux/hashtable.h>
  11. #include <linux/scatterlist.h>
  12. #include <linux/types.h>
  13. #include <linux/platform_device.h>
  14. #include <linux/of.h>
  15. #include <linux/of_platform.h>
  16. #include <linux/of_address.h>
  17. #include <linux/genalloc.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/regulator/consumer.h>
  20. #include <linux/numa.h>
  21. #include <linux/memory_hotplug.h>
  22. #include <asm/page.h>
  23. #include <linux/delay.h>
  24. #include <linux/ubwcp_dma_heap.h>
  25. #include <linux/debugfs.h>
  26. #include <linux/clk.h>
  27. #include <linux/iommu.h>
  28. MODULE_IMPORT_NS(DMA_BUF);
  29. #include "include/kernel/ubwcp.h"
  30. #include "ubwcp_hw.h"
  31. #include "include/uapi/ubwcp_ioctl.h"
  32. #define UBWCP_NUM_DEVICES 1
  33. #define UBWCP_DEVICE_NAME "ubwcp"
  34. #define UBWCP_BUFFER_DESC_OFFSET 64
  35. #define UBWCP_BUFFER_DESC_COUNT 256
  36. #define CACHE_ADDR(x) ((x) >> 6)
  37. #define PAGE_ADDR(x) ((x) >> 12)
  38. #define UBWCP_ALIGN(_x, _y) ((((_x) + (_y) - 1)/(_y))*(_y))
  39. //#define DBG(fmt, args...)
  40. //#define DBG_BUF_ATTR(fmt, args...)
  41. #define DBG_BUF_ATTR(fmt, args...) do { if (ubwcp_debug_trace_enable) \
  42. pr_err("ubwcp: %s(): " fmt "\n", __func__, ##args); \
  43. } while (0)
  44. #define DBG(fmt, args...) do { if (ubwcp_debug_trace_enable) \
  45. pr_err("ubwcp: %s(): " fmt "\n", __func__, ##args); \
  46. } while (0)
  47. #define ERR(fmt, args...) pr_err("ubwcp: %s(): ~~~ERROR~~~: " fmt "\n", __func__, ##args)
  48. #define FENTRY() DBG("")
  49. #define META_DATA_PITCH_ALIGN 64
  50. #define META_DATA_HEIGHT_ALIGN 16
  51. #define META_DATA_SIZE_ALIGN 4096
  52. #define PIXEL_DATA_SIZE_ALIGN 4096
  53. struct ubwcp_desc {
  54. int idx;
  55. void *ptr;
  56. };
  57. /* TBD: confirm size of width/height */
  58. struct ubwcp_dimension {
  59. u16 width;
  60. u16 height;
  61. };
  62. struct ubwcp_plane_info {
  63. u16 pixel_bytes;
  64. u16 per_pixel;
  65. struct ubwcp_dimension tilesize_p; /* pixels */
  66. struct ubwcp_dimension macrotilesize_p; /* pixels */
  67. };
  68. struct ubwcp_image_format_info {
  69. u16 planes;
  70. struct ubwcp_plane_info p_info[2];
  71. };
  72. enum ubwcp_std_image_format {
  73. RGBA = 0,
  74. NV12 = 1,
  75. NV124R = 2,
  76. P010 = 3,
  77. TP10 = 4,
  78. P016 = 5,
  79. INFO_FORMAT_LIST_SIZE,
  80. STD_IMAGE_FORMAT_INVALID = 0xFF
  81. };
  82. struct ubwcp_driver {
  83. /* cdev related */
  84. dev_t devt;
  85. struct class *dev_class; //sysfs dev class
  86. struct device *dev_sys; //sysfs dev
  87. struct cdev cdev; //char dev
  88. /* debugfs */
  89. struct dentry *debugfs_root;
  90. /* ubwcp devices */
  91. struct device *dev; //ubwcp device
  92. struct device *dev_desc_cb; //smmu dev for descriptors
  93. struct device *dev_buf_cb; //smmu dev for ubwcp buffers
  94. void __iomem *base; //ubwcp base address
  95. struct regulator *vdd;
  96. struct clk **clocks;
  97. int num_clocks;
  98. /* interrupts */
  99. int irq_range_ck_rd;
  100. int irq_range_ck_wr;
  101. int irq_encode;
  102. int irq_decode;
  103. /* ula address pool */
  104. u64 ula_pool_base;
  105. u64 ula_pool_size;
  106. struct gen_pool *ula_pool;
  107. configure_mmap mmap_config_fptr;
  108. /* HW version */
  109. u32 hw_ver_major;
  110. u32 hw_ver_minor;
  111. /* keep track of all buffers. hash table index'ed using dma_buf ptr.
  112. * 2**8 = 256 hash values
  113. */
  114. DECLARE_HASHTABLE(buf_table, 8);
  115. /* buffer descriptor */
  116. void *buffer_desc_base; /* CPU address */
  117. dma_addr_t buffer_desc_dma_handle; /* dma address */
  118. size_t buffer_desc_size;
  119. struct ubwcp_desc desc_list[UBWCP_BUFFER_DESC_COUNT];
  120. struct ubwcp_image_format_info format_info[INFO_FORMAT_LIST_SIZE];
  121. struct mutex desc_lock; /* allocate/free descriptors */
  122. spinlock_t buf_table_lock; /* add/remove dma_buf into list of managed bufffers */
  123. struct mutex mem_hotplug_lock; /* memory hotplug lock */
  124. struct mutex ula_lock; /* allocate/free ula */
  125. struct mutex ubwcp_flush_lock; /* ubwcp flush */
  126. struct mutex hw_range_ck_lock; /* range ck */
  127. struct list_head err_handler_list; /* error handler list */
  128. spinlock_t err_handler_list_lock; /* err_handler_list lock */
  129. };
  130. struct ubwcp_buf {
  131. struct hlist_node hnode;
  132. struct ubwcp_driver *ubwcp;
  133. struct ubwcp_buffer_attrs buf_attr;
  134. bool perm;
  135. struct ubwcp_desc *desc;
  136. bool buf_attr_set;
  137. bool locked;
  138. enum dma_data_direction lock_dir;
  139. int lock_count;
  140. /* dma_buf info */
  141. struct dma_buf *dma_buf;
  142. struct dma_buf_attachment *attachment;
  143. struct sg_table *sgt;
  144. /* ula info */
  145. phys_addr_t ula_pa;
  146. size_t ula_size;
  147. /* meta metadata */
  148. struct ubwcp_hw_meta_metadata mmdata;
  149. struct mutex lock;
  150. };
  151. static struct ubwcp_driver *me;
  152. static int error_print_count;
  153. u32 ubwcp_debug_trace_enable;
  154. static struct ubwcp_driver *ubwcp_get_driver(void)
  155. {
  156. if (!me)
  157. WARN(1, "ubwcp: driver ptr requested but driver not initialized");
  158. return me;
  159. }
  160. static void image_format_init(struct ubwcp_driver *ubwcp)
  161. { /* planes, bytes/p, Tp , MTp */
  162. ubwcp->format_info[RGBA] = (struct ubwcp_image_format_info)
  163. {1, {{4, 1, {16, 4}, {64, 16}}}};
  164. ubwcp->format_info[NV12] = (struct ubwcp_image_format_info)
  165. {2, {{1, 1, {32, 8}, {128, 32}},
  166. {2, 1, {16, 8}, { 64, 32}}}};
  167. ubwcp->format_info[NV124R] = (struct ubwcp_image_format_info)
  168. {2, {{1, 1, {64, 4}, {256, 16}},
  169. {2, 1, {32, 4}, {128, 16}}}};
  170. ubwcp->format_info[P010] = (struct ubwcp_image_format_info)
  171. {2, {{2, 1, {32, 4}, {128, 16}},
  172. {4, 1, {16, 4}, { 64, 16}}}};
  173. ubwcp->format_info[TP10] = (struct ubwcp_image_format_info)
  174. {2, {{4, 3, {48, 4}, {192, 16}},
  175. {8, 3, {24, 4}, { 96, 16}}}};
  176. ubwcp->format_info[P016] = (struct ubwcp_image_format_info)
  177. {2, {{2, 1, {32, 4}, {128, 16}},
  178. {4, 1, {16, 4}, { 64, 16}}}};
  179. }
  180. static void ubwcp_buf_desc_list_init(struct ubwcp_driver *ubwcp)
  181. {
  182. int idx;
  183. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  184. for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
  185. desc_list[idx].idx = -1;
  186. desc_list[idx].ptr = NULL;
  187. }
  188. }
  189. static int ubwcp_init_clocks(struct ubwcp_driver *ubwcp, struct device *dev)
  190. {
  191. const char *cname;
  192. struct property *prop;
  193. int i;
  194. ubwcp->num_clocks =
  195. of_property_count_strings(dev->of_node, "clock-names");
  196. if (ubwcp->num_clocks < 1) {
  197. ubwcp->num_clocks = 0;
  198. return 0;
  199. }
  200. ubwcp->clocks = devm_kzalloc(dev,
  201. sizeof(*ubwcp->clocks) * ubwcp->num_clocks, GFP_KERNEL);
  202. if (!ubwcp->clocks)
  203. return -ENOMEM;
  204. i = 0;
  205. of_property_for_each_string(dev->of_node, "clock-names",
  206. prop, cname) {
  207. struct clk *c = devm_clk_get(dev, cname);
  208. if (IS_ERR(c)) {
  209. ERR("Couldn't get clock: %s\n", cname);
  210. return PTR_ERR(c);
  211. }
  212. ubwcp->clocks[i] = c;
  213. ++i;
  214. }
  215. return 0;
  216. }
  217. static int ubwcp_enable_clocks(struct ubwcp_driver *ubwcp)
  218. {
  219. int i, ret = 0;
  220. for (i = 0; i < ubwcp->num_clocks; ++i) {
  221. ret = clk_prepare_enable(ubwcp->clocks[i]);
  222. if (ret) {
  223. ERR("Couldn't enable clock #%d\n", i);
  224. while (i--)
  225. clk_disable_unprepare(ubwcp->clocks[i]);
  226. break;
  227. }
  228. }
  229. return ret;
  230. }
  231. static void ubwcp_disable_clocks(struct ubwcp_driver *ubwcp)
  232. {
  233. int i;
  234. for (i = ubwcp->num_clocks; i; --i)
  235. clk_disable_unprepare(ubwcp->clocks[i - 1]);
  236. }
  237. /* UBWCP Power control */
  238. static int ubwcp_power(struct ubwcp_driver *ubwcp, bool enable)
  239. {
  240. int ret = 0;
  241. if (!ubwcp) {
  242. ERR("ubwcp ptr is NULL");
  243. return -1;
  244. }
  245. if (!ubwcp->vdd) {
  246. ERR("vdd is NULL");
  247. return -1;
  248. }
  249. if (enable) {
  250. ret = regulator_enable(ubwcp->vdd);
  251. if (ret < 0) {
  252. ERR("regulator_enable failed: %d", ret);
  253. ret = -1;
  254. } else {
  255. DBG("regulator_enable() success");
  256. }
  257. if (!ret) {
  258. ret = ubwcp_enable_clocks(ubwcp);
  259. if (ret) {
  260. ERR("enable clocks failed: %d", ret);
  261. regulator_disable(ubwcp->vdd);
  262. } else {
  263. DBG("enable clocks success");
  264. }
  265. }
  266. } else {
  267. ret = regulator_disable(ubwcp->vdd);
  268. if (ret < 0) {
  269. ERR("regulator_disable failed: %d", ret);
  270. ret = -1;
  271. } else {
  272. DBG("regulator_disable() success");
  273. }
  274. if (!ret) {
  275. ubwcp_disable_clocks(ubwcp);
  276. DBG("disable clocks success");
  277. }
  278. }
  279. return ret;
  280. }
  281. static int ubwcp_flush(struct ubwcp_driver *ubwcp)
  282. {
  283. int ret = 0;
  284. mutex_lock(&ubwcp->ubwcp_flush_lock);
  285. ret = ubwcp_hw_flush(ubwcp->base);
  286. mutex_unlock(&ubwcp->ubwcp_flush_lock);
  287. if (ret != 0)
  288. WARN(1, "ubwcp_hw_flush() failed!");
  289. return ret;
  290. }
  291. /* get dma_buf ptr for the given dma_buf fd */
  292. struct dma_buf *ubwcp_dma_buf_fd_to_dma_buf(int dma_buf_fd)
  293. {
  294. struct dma_buf *dmabuf;
  295. /* TBD: dma_buf_get() results in taking ref to buf and it won't ever get
  296. * free'ed until ref count goes to 0. So we must reduce the ref count
  297. * immediately after we find our corresponding ubwcp_buf.
  298. */
  299. dmabuf = dma_buf_get(dma_buf_fd);
  300. if (IS_ERR(dmabuf)) {
  301. ERR("dmabuf ptr not found for dma_buf_fd = %d", dma_buf_fd);
  302. return NULL;
  303. }
  304. dma_buf_put(dmabuf);
  305. return dmabuf;
  306. }
  307. EXPORT_SYMBOL(ubwcp_dma_buf_fd_to_dma_buf);
  308. /* get ubwcp_buf corresponding to the given dma_buf */
  309. static struct ubwcp_buf *dma_buf_to_ubwcp_buf(struct dma_buf *dmabuf)
  310. {
  311. struct ubwcp_buf *buf = NULL;
  312. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  313. unsigned long flags;
  314. if (!dmabuf || !ubwcp)
  315. return NULL;
  316. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  317. /* look up ubwcp_buf corresponding to this dma_buf */
  318. hash_for_each_possible(ubwcp->buf_table, buf, hnode, (u64)dmabuf) {
  319. if (buf->dma_buf == dmabuf)
  320. break;
  321. }
  322. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  323. return buf;
  324. }
  325. /* return ubwcp hardware version */
  326. int ubwcp_get_hw_version(struct ubwcp_ioctl_hw_version *ver)
  327. {
  328. struct ubwcp_driver *ubwcp;
  329. FENTRY();
  330. if (!ver) {
  331. ERR("invalid version ptr");
  332. return -EINVAL;
  333. }
  334. ubwcp = ubwcp_get_driver();
  335. if (!ubwcp)
  336. return -1;
  337. ver->major = ubwcp->hw_ver_major;
  338. ver->minor = ubwcp->hw_ver_minor;
  339. return 0;
  340. }
  341. EXPORT_SYMBOL(ubwcp_get_hw_version);
  342. /**
  343. *
  344. * Initialize ubwcp buffer for the given dma_buf. This
  345. * initializes ubwcp internal data structures and possibly hw to
  346. * use ubwcp for this buffer.
  347. *
  348. * @param dmabuf : ptr to the buffer to be configured for ubwcp
  349. *
  350. * @return int : 0 on success, otherwise error code
  351. */
  352. static int ubwcp_init_buffer(struct dma_buf *dmabuf)
  353. {
  354. int ret = 0;
  355. int nid;
  356. struct ubwcp_buf *buf;
  357. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  358. unsigned long flags;
  359. bool table_empty;
  360. FENTRY();
  361. if (!ubwcp)
  362. return -1;
  363. if (!dmabuf) {
  364. ERR("NULL dmabuf input ptr");
  365. return -EINVAL;
  366. }
  367. if (dma_buf_to_ubwcp_buf(dmabuf)) {
  368. ERR("dma_buf already initialized for ubwcp");
  369. return -EEXIST;
  370. }
  371. buf = kzalloc(sizeof(*buf), GFP_KERNEL);
  372. if (!buf) {
  373. ERR("failed to alloc for new ubwcp_buf");
  374. return -ENOMEM;
  375. }
  376. mutex_init(&buf->lock);
  377. buf->dma_buf = dmabuf;
  378. buf->ubwcp = ubwcp;
  379. mutex_lock(&ubwcp->mem_hotplug_lock);
  380. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  381. table_empty = hash_empty(ubwcp->buf_table);
  382. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  383. if (table_empty) {
  384. ret = ubwcp_power(ubwcp, true);
  385. if (ret)
  386. goto err_power_on;
  387. nid = memory_add_physaddr_to_nid(ubwcp->ula_pool_base);
  388. DBG("calling add_memory()...");
  389. ret = add_memory(nid, ubwcp->ula_pool_base, ubwcp->ula_pool_size, MHP_NONE);
  390. if (ret) {
  391. ERR("add_memory() failed st:0x%lx sz:0x%lx err: %d",
  392. ubwcp->ula_pool_base,
  393. ubwcp->ula_pool_size,
  394. ret);
  395. goto err_add_memory;
  396. } else {
  397. DBG("add_memory() ula_pool_base:0x%llx, size:0x%zx, kernel addr:0x%p",
  398. ubwcp->ula_pool_base,
  399. ubwcp->ula_pool_size,
  400. page_to_virt(pfn_to_page(PFN_DOWN(ubwcp->ula_pool_base))));
  401. }
  402. }
  403. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  404. hash_add(ubwcp->buf_table, &buf->hnode, (u64)buf->dma_buf);
  405. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  406. mutex_unlock(&ubwcp->mem_hotplug_lock);
  407. return ret;
  408. err_add_memory:
  409. ubwcp_power(ubwcp, false);
  410. err_power_on:
  411. mutex_unlock(&ubwcp->mem_hotplug_lock);
  412. kfree(buf);
  413. if (!ret)
  414. ret = -1;
  415. return ret;
  416. }
  417. static void dump_attributes(struct ubwcp_buffer_attrs *attr)
  418. {
  419. DBG_BUF_ATTR("");
  420. DBG_BUF_ATTR("image_format: %d", attr->image_format);
  421. DBG_BUF_ATTR("major_ubwc_ver: %d", attr->major_ubwc_ver);
  422. DBG_BUF_ATTR("minor_ubwc_ver: %d", attr->minor_ubwc_ver);
  423. DBG_BUF_ATTR("compression_type: %d", attr->compression_type);
  424. DBG_BUF_ATTR("lossy_params: %llu", attr->lossy_params);
  425. DBG_BUF_ATTR("width: %d", attr->width);
  426. DBG_BUF_ATTR("height: %d", attr->height);
  427. DBG_BUF_ATTR("stride: %d", attr->stride);
  428. DBG_BUF_ATTR("scanlines: %d", attr->scanlines);
  429. DBG_BUF_ATTR("planar_padding: %d", attr->planar_padding);
  430. DBG_BUF_ATTR("subsample: %d", attr->subsample);
  431. DBG_BUF_ATTR("sub_system_target: %d", attr->sub_system_target);
  432. DBG_BUF_ATTR("y_offset: %d", attr->y_offset);
  433. DBG_BUF_ATTR("batch_size: %d", attr->batch_size);
  434. DBG_BUF_ATTR("");
  435. }
  436. /* validate buffer attributes */
  437. static bool ubwcp_buf_attrs_valid(struct ubwcp_buffer_attrs *attr)
  438. {
  439. bool valid_format;
  440. switch (attr->image_format) {
  441. case UBWCP_LINEAR:
  442. case UBWCP_RGBA8888:
  443. case UBWCP_NV12:
  444. case UBWCP_NV12_Y:
  445. case UBWCP_NV12_UV:
  446. case UBWCP_NV124R:
  447. case UBWCP_NV124R_Y:
  448. case UBWCP_NV124R_UV:
  449. case UBWCP_TP10:
  450. case UBWCP_TP10_Y:
  451. case UBWCP_TP10_UV:
  452. case UBWCP_P010:
  453. case UBWCP_P010_Y:
  454. case UBWCP_P010_UV:
  455. case UBWCP_P016:
  456. case UBWCP_P016_Y:
  457. case UBWCP_P016_UV:
  458. valid_format = true;
  459. break;
  460. default:
  461. valid_format = false;
  462. }
  463. if (!valid_format) {
  464. ERR("invalid image format: %d", attr->image_format);
  465. goto err;
  466. }
  467. if (attr->major_ubwc_ver || attr->minor_ubwc_ver) {
  468. ERR("major/minor ubwc ver must be 0. major: %d minor: %d",
  469. attr->major_ubwc_ver, attr->minor_ubwc_ver);
  470. goto err;
  471. }
  472. if (attr->compression_type != UBWCP_COMPRESSION_LOSSLESS) {
  473. ERR("compression_type is not valid: %d",
  474. attr->compression_type);
  475. goto err;
  476. }
  477. if (attr->lossy_params != 0) {
  478. ERR("lossy_params is not valid: %d", attr->lossy_params);
  479. goto err;
  480. }
  481. //TBD: some upper limit for width?
  482. if (attr->width > 10*1024) {
  483. ERR("width is invalid (above upper limit): %d", attr->width);
  484. goto err;
  485. }
  486. //TBD: some upper limit for height?
  487. if (attr->height > 10*1024) {
  488. ERR("height is invalid (above upper limit): %d", attr->height);
  489. goto err;
  490. }
  491. /* TBD: what's the upper limit for stride? 8K is likely too high. */
  492. if (!IS_ALIGNED(attr->stride, 64) ||
  493. (attr->stride < attr->width) ||
  494. (attr->stride > 4*8192)) {
  495. ERR("stride is not valid (aligned to 64 and <= 8192): %d",
  496. attr->stride);
  497. goto err;
  498. }
  499. /* TBD: currently assume height + 10. Replace 10 with right num from camera. */
  500. if ((attr->scanlines < attr->height) ||
  501. (attr->scanlines > attr->height + 10)) {
  502. ERR("scanlines is not valid - height: %d scanlines: %d",
  503. attr->height, attr->scanlines);
  504. goto err;
  505. }
  506. if (attr->planar_padding > 4096) {
  507. ERR("planar_padding is not valid. (<= 4096): %d",
  508. attr->planar_padding);
  509. goto err;
  510. }
  511. if (attr->subsample != UBWCP_SUBSAMPLE_4_2_0) {
  512. ERR("subsample is not valid: %d", attr->subsample);
  513. goto err;
  514. }
  515. if (attr->sub_system_target & ~UBWCP_SUBSYSTEM_TARGET_CPU) {
  516. ERR("sub_system_target other that CPU is not supported: %d",
  517. attr->sub_system_target);
  518. goto err;
  519. }
  520. if (!(attr->sub_system_target & UBWCP_SUBSYSTEM_TARGET_CPU)) {
  521. ERR("sub_system_target is not set to CPU: %d",
  522. attr->sub_system_target);
  523. goto err;
  524. }
  525. if (attr->y_offset != 0) {
  526. ERR("y_offset is not valid: %d", attr->y_offset);
  527. goto err;
  528. }
  529. if (attr->batch_size != 1) {
  530. ERR("batch_size is not valid: %d", attr->batch_size);
  531. goto err;
  532. }
  533. dump_attributes(attr);
  534. return true;
  535. err:
  536. dump_attributes(attr);
  537. return false;
  538. }
  539. /* return true if image format has only Y plane*/
  540. bool ubwcp_image_y_only(u16 format)
  541. {
  542. switch (format) {
  543. case UBWCP_NV12_Y:
  544. case UBWCP_NV124R_Y:
  545. case UBWCP_TP10_Y:
  546. case UBWCP_P010_Y:
  547. case UBWCP_P016_Y:
  548. return true;
  549. default:
  550. return false;
  551. }
  552. }
  553. /* return true if image format has only UV plane*/
  554. bool ubwcp_image_uv_only(u16 format)
  555. {
  556. switch (format) {
  557. case UBWCP_NV12_UV:
  558. case UBWCP_NV124R_UV:
  559. case UBWCP_TP10_UV:
  560. case UBWCP_P010_UV:
  561. case UBWCP_P016_UV:
  562. return true;
  563. default:
  564. return false;
  565. }
  566. }
  567. /* calculate and return metadata buffer size for a given plane
  568. * and buffer attributes
  569. * NOTE: in this function, we will only pass in NV12 format.
  570. * NOT NV12_Y or NV12_UV etc.
  571. * the Y or UV information is in the "plane"
  572. * "format" here purely means "encoding format" and no information
  573. * if some plane data is missing.
  574. */
  575. static size_t metadata_buf_sz(struct ubwcp_driver *ubwcp,
  576. enum ubwcp_std_image_format format,
  577. u32 width, u32 height, u8 plane)
  578. {
  579. size_t size;
  580. u64 pitch;
  581. u64 lines;
  582. u64 tile_width;
  583. u32 tile_height;
  584. struct ubwcp_image_format_info f_info;
  585. struct ubwcp_plane_info p_info;
  586. f_info = ubwcp->format_info[format];
  587. DBG_BUF_ATTR("");
  588. DBG_BUF_ATTR("");
  589. DBG_BUF_ATTR("Calculating metadata buffer size: format = %d, plane = %d", format, plane);
  590. if (plane >= f_info.planes) {
  591. ERR("Format does not have requested plane info: format: %d, plane: %d",
  592. format, plane);
  593. WARN(1, "Fix this!!!!!");
  594. return 0;
  595. }
  596. p_info = f_info.p_info[plane];
  597. /* UV plane */
  598. if (plane == 1) {
  599. width = width/2;
  600. height = height/2;
  601. }
  602. tile_width = p_info.tilesize_p.width;
  603. tile_height = p_info.tilesize_p.height;
  604. /* pitch: # of tiles in a row
  605. * lines: # of tile rows
  606. */
  607. pitch = UBWCP_ALIGN((width + tile_width - 1)/tile_width, META_DATA_PITCH_ALIGN);
  608. lines = UBWCP_ALIGN((height + tile_height - 1)/tile_height, META_DATA_HEIGHT_ALIGN);
  609. DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
  610. DBG_BUF_ATTR("tile params : %d x %d (pixels)", tile_width, tile_height);
  611. DBG_BUF_ATTR("pitch : %d (%d)", pitch, width/tile_width);
  612. DBG_BUF_ATTR("lines : %d (%d)", lines, height);
  613. DBG_BUF_ATTR("size (p*l*bytes) : %d", pitch*lines*1);
  614. /* x1 below is only to clarify that we are multiplying by 1 bytes/tile */
  615. size = UBWCP_ALIGN(pitch*lines*1, META_DATA_SIZE_ALIGN);
  616. DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", size, size);
  617. return size;
  618. }
  619. /* calculate and return size of pixel data buffer for a given plane
  620. * and buffer attributes
  621. */
  622. static size_t pixeldata_buf_sz(struct ubwcp_driver *ubwcp,
  623. u16 format, u32 width,
  624. u32 height, u8 plane)
  625. {
  626. size_t size;
  627. u64 pitch;
  628. u64 lines;
  629. u16 pixel_bytes;
  630. u16 per_pixel;
  631. u64 macro_tile_width_p;
  632. u64 macro_tile_height_p;
  633. struct ubwcp_image_format_info f_info;
  634. struct ubwcp_plane_info p_info;
  635. f_info = ubwcp->format_info[format];
  636. DBG_BUF_ATTR("");
  637. DBG_BUF_ATTR("");
  638. DBG_BUF_ATTR("Calculating Pixeldata buffer size: format = %d, plane = %d", format, plane);
  639. if (plane >= f_info.planes) {
  640. ERR("Format does not have requested plane info: format: %d, plane: %d",
  641. format, plane);
  642. WARN(1, "Fix this!!!!!");
  643. return 0;
  644. }
  645. p_info = f_info.p_info[plane];
  646. pixel_bytes = p_info.pixel_bytes;
  647. per_pixel = p_info.per_pixel;
  648. /* UV plane */
  649. if (plane == 1) {
  650. width = width/2;
  651. height = height/2;
  652. }
  653. macro_tile_width_p = p_info.macrotilesize_p.width;
  654. macro_tile_height_p = p_info.macrotilesize_p.height;
  655. /* align pixel width and height macro tile width and height */
  656. pitch = UBWCP_ALIGN(width, macro_tile_width_p);
  657. lines = UBWCP_ALIGN(height, macro_tile_height_p);
  658. DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
  659. DBG_BUF_ATTR("macro tile params: %d x %d (pixels)", macro_tile_width_p,
  660. macro_tile_height_p);
  661. DBG_BUF_ATTR("bytes_per_pixel : %d/%d", pixel_bytes, per_pixel);
  662. DBG_BUF_ATTR("pitch : %d", pitch);
  663. DBG_BUF_ATTR("lines : %d", lines);
  664. DBG_BUF_ATTR("size (p*l*bytes) : %d", (pitch*lines*pixel_bytes)/per_pixel);
  665. size = UBWCP_ALIGN((pitch*lines*pixel_bytes)/per_pixel, PIXEL_DATA_SIZE_ALIGN);
  666. DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", size, size);
  667. return size;
  668. }
  669. static int get_tile_height(struct ubwcp_driver *ubwcp, enum ubwcp_std_image_format format,
  670. u8 plane)
  671. {
  672. struct ubwcp_image_format_info f_info;
  673. struct ubwcp_plane_info p_info;
  674. f_info = ubwcp->format_info[format];
  675. p_info = f_info.p_info[plane];
  676. return p_info.tilesize_p.height;
  677. }
  678. /*
  679. * plane: must be 0 or 1 (1st plane == 0, 2nd plane == 1)
  680. */
  681. static size_t ubwcp_ula_size(struct ubwcp_driver *ubwcp, u16 format,
  682. u32 stride_b, u32 scanlines, u8 plane,
  683. bool add_tile_pad)
  684. {
  685. size_t size;
  686. DBG_BUF_ATTR("%s(format = %d, plane = %d)", __func__, format, plane);
  687. /* UV plane */
  688. if (plane == 1)
  689. scanlines = scanlines/2;
  690. if (add_tile_pad) {
  691. int tile_height = get_tile_height(ubwcp, format, plane);
  692. /* Align plane size to plane tile height */
  693. scanlines = ((scanlines + tile_height - 1) / tile_height) * tile_height;
  694. }
  695. size = stride_b*scanlines;
  696. DBG_BUF_ATTR("Size of plane-%u: (%u * %u) = %zu (0x%zx)",
  697. plane, stride_b, scanlines, size, size);
  698. return size;
  699. }
  700. int missing_plane_from_format(u16 ioctl_image_format)
  701. {
  702. int missing_plane;
  703. switch (ioctl_image_format) {
  704. case UBWCP_NV12_Y:
  705. missing_plane = 2;
  706. break;
  707. case UBWCP_NV12_UV:
  708. missing_plane = 1;
  709. break;
  710. case UBWCP_NV124R_Y:
  711. missing_plane = 2;
  712. break;
  713. case UBWCP_NV124R_UV:
  714. missing_plane = 1;
  715. break;
  716. case UBWCP_TP10_Y:
  717. missing_plane = 2;
  718. break;
  719. case UBWCP_TP10_UV:
  720. missing_plane = 1;
  721. break;
  722. case UBWCP_P010_Y:
  723. missing_plane = 2;
  724. break;
  725. case UBWCP_P010_UV:
  726. missing_plane = 1;
  727. break;
  728. case UBWCP_P016_Y:
  729. missing_plane = 2;
  730. break;
  731. case UBWCP_P016_UV:
  732. missing_plane = 1;
  733. break;
  734. default:
  735. missing_plane = 0;
  736. }
  737. return missing_plane;
  738. }
  739. int planes_in_format(enum ubwcp_std_image_format format)
  740. {
  741. if (format == RGBA)
  742. return 1;
  743. else
  744. return 2;
  745. }
  746. enum ubwcp_std_image_format to_std_format(u16 ioctl_image_format)
  747. {
  748. switch (ioctl_image_format) {
  749. case UBWCP_RGBA8888:
  750. return RGBA;
  751. case UBWCP_NV12:
  752. case UBWCP_NV12_Y:
  753. case UBWCP_NV12_UV:
  754. return NV12;
  755. case UBWCP_NV124R:
  756. case UBWCP_NV124R_Y:
  757. case UBWCP_NV124R_UV:
  758. return NV124R;
  759. case UBWCP_TP10:
  760. case UBWCP_TP10_Y:
  761. case UBWCP_TP10_UV:
  762. return TP10;
  763. case UBWCP_P010:
  764. case UBWCP_P010_Y:
  765. case UBWCP_P010_UV:
  766. return P010;
  767. case UBWCP_P016:
  768. case UBWCP_P016_Y:
  769. case UBWCP_P016_UV:
  770. return P016;
  771. default:
  772. WARN(1, "Fix this!!!");
  773. return STD_IMAGE_FORMAT_INVALID;
  774. }
  775. }
  776. unsigned int ubwcp_get_hw_image_format_value(u16 ioctl_image_format)
  777. {
  778. enum ubwcp_std_image_format format;
  779. format = to_std_format(ioctl_image_format);
  780. switch (format) {
  781. case RGBA:
  782. return HW_BUFFER_FORMAT_RGBA;
  783. case NV12:
  784. return HW_BUFFER_FORMAT_NV12;
  785. case NV124R:
  786. return HW_BUFFER_FORMAT_NV124R;
  787. case P010:
  788. return HW_BUFFER_FORMAT_P010;
  789. case TP10:
  790. return HW_BUFFER_FORMAT_TP10;
  791. case P016:
  792. return HW_BUFFER_FORMAT_P016;
  793. default:
  794. WARN(1, "Fix this!!!!!");
  795. return 0;
  796. }
  797. }
  798. static int ubwcp_validate_uv_align(struct ubwcp_driver *ubwcp,
  799. struct ubwcp_buffer_attrs *attr,
  800. size_t ula_y_plane_size,
  801. size_t uv_start_offset)
  802. {
  803. int ret = 0;
  804. size_t ula_y_plane_size_align;
  805. size_t y_tile_align_bytes;
  806. int y_tile_height;
  807. int planes;
  808. /* Only validate UV align if there is both a Y and UV plane */
  809. planes = planes_in_format(to_std_format(attr->image_format));
  810. if (planes != 2)
  811. return 0;
  812. /* Check it is cache line size aligned */
  813. if ((uv_start_offset % 64) != 0) {
  814. ret = -EINVAL;
  815. ERR("uv_start_offset %zu not cache line aligned",
  816. uv_start_offset);
  817. goto err;
  818. }
  819. /*
  820. * Check that UV plane does not overlap with any of the Y plane’s tiles
  821. */
  822. y_tile_height = get_tile_height(ubwcp, to_std_format(attr->image_format), 0);
  823. y_tile_align_bytes = y_tile_height * attr->stride;
  824. ula_y_plane_size_align = ((ula_y_plane_size + y_tile_align_bytes - 1) /
  825. y_tile_align_bytes) * y_tile_align_bytes;
  826. if (uv_start_offset < ula_y_plane_size_align) {
  827. ret = -EINVAL;
  828. ERR("uv offset %zu less than y plane align %zu for y plane size %zu",
  829. uv_start_offset, ula_y_plane_size_align,
  830. ula_y_plane_size);
  831. goto err;
  832. }
  833. return 0;
  834. err:
  835. return ret;
  836. }
  837. /* calculate ULA buffer parms
  838. * TBD: how do we make sure uv_start address (not the offset)
  839. * is aligned per requirement: cache line
  840. */
  841. static int ubwcp_calc_ula_params(struct ubwcp_driver *ubwcp,
  842. struct ubwcp_buffer_attrs *attr,
  843. size_t *ula_size,
  844. size_t *ula_y_plane_size,
  845. size_t *uv_start_offset)
  846. {
  847. size_t size;
  848. enum ubwcp_std_image_format format;
  849. int planes;
  850. int missing_plane;
  851. u32 stride;
  852. u32 scanlines;
  853. u32 planar_padding;
  854. stride = attr->stride;
  855. scanlines = attr->scanlines;
  856. planar_padding = attr->planar_padding;
  857. /* convert ioctl image format to standard image format */
  858. format = to_std_format(attr->image_format);
  859. /* Number of "expected" planes in "the standard defined" image format */
  860. planes = planes_in_format(format);
  861. /* any plane missing?
  862. * valid missing_plane values:
  863. * 0 == no plane missing
  864. * 1 == 1st plane missing
  865. * 2 == 2nd plane missing
  866. */
  867. missing_plane = missing_plane_from_format(attr->image_format);
  868. DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
  869. DBG_BUF_ATTR("planes_in_format : %d", planes);
  870. DBG_BUF_ATTR("missing_plane : %d", missing_plane);
  871. DBG_BUF_ATTR("Planar Padding : %d", planar_padding);
  872. if (planes == 1) {
  873. /* uv_start beyond ULA range */
  874. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
  875. *uv_start_offset = size;
  876. *ula_y_plane_size = size;
  877. } else {
  878. if (!missing_plane) {
  879. /* size for both planes and padding */
  880. /* Don't pad out Y plane as client would not expect this padding */
  881. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, false);
  882. *ula_y_plane_size = size;
  883. size += planar_padding;
  884. *uv_start_offset = size;
  885. size += ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
  886. } else {
  887. if (missing_plane == 2) {
  888. /* Y-only image, set uv_start beyond ULA range */
  889. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
  890. *uv_start_offset = size;
  891. *ula_y_plane_size = size;
  892. } else {
  893. /* first plane data is not there */
  894. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
  895. *uv_start_offset = 0; /* uv data is at the beginning */
  896. *ula_y_plane_size = 0;
  897. }
  898. }
  899. }
  900. //TBD: cleanup
  901. *ula_size = size;
  902. DBG_BUF_ATTR("Before page align: Total ULA_Size: %d (0x%x) (planes + planar padding)",
  903. *ula_size, *ula_size);
  904. *ula_size = UBWCP_ALIGN(size, 4096);
  905. DBG_BUF_ATTR("After page align : Total ULA_Size: %d (0x%x) (planes + planar padding)",
  906. *ula_size, *ula_size);
  907. return 0;
  908. }
  909. /* calculate UBWCP buffer parms */
  910. static int ubwcp_calc_ubwcp_buf_params(struct ubwcp_driver *ubwcp,
  911. struct ubwcp_buffer_attrs *attr,
  912. size_t *md_p0, size_t *pd_p0,
  913. size_t *md_p1, size_t *pd_p1,
  914. size_t *stride_tp10_b)
  915. {
  916. int planes;
  917. int missing_plane;
  918. enum ubwcp_std_image_format format;
  919. size_t stride_tp10_p;
  920. FENTRY();
  921. /* convert ioctl image format to standard image format */
  922. format = to_std_format(attr->image_format);
  923. missing_plane = missing_plane_from_format(attr->image_format);
  924. planes = planes_in_format(format); //pass in 0 (RGB) should return 1
  925. DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
  926. DBG_BUF_ATTR("planes_in_format : %d", planes);
  927. DBG_BUF_ATTR("missing_plane : %d", missing_plane);
  928. if (!missing_plane) {
  929. *md_p0 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  930. *pd_p0 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  931. if (planes == 2) {
  932. *md_p1 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  933. *pd_p1 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  934. }
  935. } else {
  936. if (missing_plane == 1) {
  937. *md_p0 = 0;
  938. *pd_p0 = 0;
  939. *md_p1 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  940. *pd_p1 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  941. } else {
  942. *md_p0 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  943. *pd_p0 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  944. *md_p1 = 0;
  945. *pd_p1 = 0;
  946. }
  947. }
  948. if (format == TP10) {
  949. stride_tp10_p = UBWCP_ALIGN(attr->width, 192);
  950. *stride_tp10_b = (stride_tp10_p/3) + stride_tp10_p;
  951. } else {
  952. *stride_tp10_b = 0;
  953. }
  954. return 0;
  955. }
  956. /* reserve ULA address space of the given size */
  957. static phys_addr_t ubwcp_ula_alloc(struct ubwcp_driver *ubwcp, size_t size)
  958. {
  959. phys_addr_t pa;
  960. mutex_lock(&ubwcp->ula_lock);
  961. pa = gen_pool_alloc(ubwcp->ula_pool, size);
  962. DBG("addr: %p, size: %zx", pa, size);
  963. mutex_unlock(&ubwcp->ula_lock);
  964. return pa;
  965. }
  966. /* free ULA address space of the given address and size */
  967. static void ubwcp_ula_free(struct ubwcp_driver *ubwcp, phys_addr_t pa, size_t size)
  968. {
  969. mutex_lock(&ubwcp->ula_lock);
  970. if (!gen_pool_has_addr(ubwcp->ula_pool, pa, size)) {
  971. ERR("Attempt to free mem not from gen_pool: pa: %p, size: %zx", pa, size);
  972. goto err;
  973. }
  974. DBG("addr: %p, size: %zx", pa, size);
  975. gen_pool_free(ubwcp->ula_pool, pa, size);
  976. mutex_unlock(&ubwcp->ula_lock);
  977. return;
  978. err:
  979. mutex_unlock(&ubwcp->ula_lock);
  980. }
  981. /* free up or expand current_pa and return the new pa */
  982. static phys_addr_t ubwcp_ula_realloc(struct ubwcp_driver *ubwcp,
  983. phys_addr_t pa,
  984. size_t size,
  985. size_t new_size)
  986. {
  987. if (size == new_size)
  988. return pa;
  989. if (pa)
  990. ubwcp_ula_free(ubwcp, pa, size);
  991. return ubwcp_ula_alloc(ubwcp, new_size);
  992. }
  993. /* unmap dma buf */
  994. static void ubwcp_dma_unmap(struct ubwcp_buf *buf)
  995. {
  996. FENTRY();
  997. if (buf->dma_buf && buf->attachment) {
  998. DBG("Calling dma_buf_unmap_attachment()");
  999. dma_buf_unmap_attachment(buf->attachment, buf->sgt, DMA_BIDIRECTIONAL);
  1000. buf->sgt = NULL;
  1001. dma_buf_detach(buf->dma_buf, buf->attachment);
  1002. buf->attachment = NULL;
  1003. }
  1004. }
  1005. /* dma map ubwcp buffer */
  1006. static int ubwcp_dma_map(struct ubwcp_buf *buf,
  1007. struct device *dev,
  1008. size_t iova_min_size,
  1009. dma_addr_t *iova)
  1010. {
  1011. int ret = 0;
  1012. struct dma_buf *dma_buf = buf->dma_buf;
  1013. struct dma_buf_attachment *attachment;
  1014. struct sg_table *sgt;
  1015. size_t dma_len;
  1016. /* Map buffer to SMMU and get IOVA */
  1017. attachment = dma_buf_attach(dma_buf, dev);
  1018. if (IS_ERR(attachment)) {
  1019. ret = PTR_ERR(attachment);
  1020. ERR("dma_buf_attach() failed: %d", ret);
  1021. goto err;
  1022. }
  1023. dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
  1024. dma_set_seg_boundary(dev, (unsigned long)DMA_BIT_MASK(64));
  1025. sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
  1026. if (IS_ERR_OR_NULL(sgt)) {
  1027. ret = PTR_ERR(sgt);
  1028. ERR("dma_buf_map_attachment() failed: %d", ret);
  1029. goto err_detach;
  1030. }
  1031. if (sgt->nents != 1) {
  1032. ERR("nents = %d", sgt->nents);
  1033. goto err_unmap;
  1034. }
  1035. /* ensure that dma_buf is big enough for the new attrs */
  1036. dma_len = sg_dma_len(sgt->sgl);
  1037. if (dma_len < iova_min_size) {
  1038. ERR("dma len: %d is less than min ubwcp buffer size: %d",
  1039. dma_len, iova_min_size);
  1040. goto err_unmap;
  1041. }
  1042. *iova = sg_dma_address(sgt->sgl);
  1043. buf->attachment = attachment;
  1044. buf->sgt = sgt;
  1045. return ret;
  1046. err_unmap:
  1047. dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
  1048. err_detach:
  1049. dma_buf_detach(dma_buf, attachment);
  1050. err:
  1051. if (!ret)
  1052. ret = -1;
  1053. return ret;
  1054. }
  1055. static void
  1056. ubwcp_pixel_to_bytes(struct ubwcp_driver *ubwcp,
  1057. enum ubwcp_std_image_format format,
  1058. u32 width_p, u32 height_p,
  1059. u32 *width_b, u32 *height_b)
  1060. {
  1061. u16 pixel_bytes;
  1062. u16 per_pixel;
  1063. struct ubwcp_image_format_info f_info;
  1064. struct ubwcp_plane_info p_info;
  1065. f_info = ubwcp->format_info[format];
  1066. p_info = f_info.p_info[0];
  1067. pixel_bytes = p_info.pixel_bytes;
  1068. per_pixel = p_info.per_pixel;
  1069. *width_b = (width_p*pixel_bytes)/per_pixel;
  1070. *height_b = (height_p*pixel_bytes)/per_pixel;
  1071. }
  1072. static void reset_buf_attrs(struct ubwcp_buf *buf)
  1073. {
  1074. struct ubwcp_hw_meta_metadata *mmdata;
  1075. struct ubwcp_driver *ubwcp;
  1076. ubwcp = buf->ubwcp;
  1077. mmdata = &buf->mmdata;
  1078. ubwcp_dma_unmap(buf);
  1079. /* reset ula params */
  1080. if (buf->ula_size) {
  1081. ubwcp_ula_free(ubwcp, buf->ula_pa, buf->ula_size);
  1082. buf->ula_size = 0;
  1083. buf->ula_pa = 0;
  1084. }
  1085. /* reset ubwcp params */
  1086. memset(mmdata, 0, sizeof(*mmdata));
  1087. buf->buf_attr_set = false;
  1088. }
  1089. static void print_mmdata_desc(struct ubwcp_hw_meta_metadata *mmdata)
  1090. {
  1091. DBG_BUF_ATTR("");
  1092. DBG_BUF_ATTR("--------MM_DATA DESC ---------");
  1093. DBG_BUF_ATTR("uv_start_addr : 0x%08llx (cache addr) (actual: 0x%llx)",
  1094. mmdata->uv_start_addr, mmdata->uv_start_addr << 6);
  1095. DBG_BUF_ATTR("format : 0x%08x", mmdata->format);
  1096. DBG_BUF_ATTR("stride : 0x%08x (cache addr) (actual: 0x%x)",
  1097. mmdata->stride, mmdata->stride << 6);
  1098. DBG_BUF_ATTR("stride_ubwcp : 0x%08x (cache addr) (actual: 0x%zx)",
  1099. mmdata->stride_ubwcp, mmdata->stride_ubwcp << 6);
  1100. DBG_BUF_ATTR("metadata_base_y : 0x%08x (page addr) (actual: 0x%llx)",
  1101. mmdata->metadata_base_y, mmdata->metadata_base_y << 12);
  1102. DBG_BUF_ATTR("metadata_base_uv: 0x%08x (page addr) (actual: 0x%zx)",
  1103. mmdata->metadata_base_uv, mmdata->metadata_base_uv << 12);
  1104. DBG_BUF_ATTR("buffer_y_offset : 0x%08x (page addr) (actual: 0x%zx)",
  1105. mmdata->buffer_y_offset, mmdata->buffer_y_offset << 12);
  1106. DBG_BUF_ATTR("buffer_uv_offset: 0x%08x (page addr) (actual: 0x%zx)",
  1107. mmdata->buffer_uv_offset, mmdata->buffer_uv_offset << 12);
  1108. DBG_BUF_ATTR("width_height : 0x%08x (width: 0x%x height: 0x%x)",
  1109. mmdata->width_height, mmdata->width_height >> 16, mmdata->width_height & 0xFFFF);
  1110. DBG_BUF_ATTR("");
  1111. }
  1112. /* set buffer attributes:
  1113. * Failure:
  1114. * If a call to ubwcp_set_buf_attrs() fails, any attributes set from a previously
  1115. * successful ubwcp_set_buf_attrs() will be also removed. Thus,
  1116. * ubwcp_set_buf_attrs() implicitly does "unset previous attributes" and
  1117. * then "try to set these new attributes".
  1118. *
  1119. * The result of a failed call to ubwcp_set_buf_attrs() will leave the buffer
  1120. * in a linear mode, NOT with attributes from earlier successful call.
  1121. */
  1122. int ubwcp_set_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
  1123. {
  1124. int ret = 0;
  1125. size_t ula_size = 0;
  1126. size_t uv_start_offset = 0;
  1127. size_t ula_y_plane_size = 0;
  1128. phys_addr_t ula_pa = 0x0;
  1129. struct ubwcp_buf *buf;
  1130. struct ubwcp_driver *ubwcp;
  1131. size_t metadata_p0;
  1132. size_t pixeldata_p0;
  1133. size_t metadata_p1;
  1134. size_t pixeldata_p1;
  1135. size_t iova_min_size;
  1136. size_t stride_tp10_b;
  1137. dma_addr_t iova_base;
  1138. struct ubwcp_hw_meta_metadata *mmdata;
  1139. u64 uv_start;
  1140. u32 stride_b;
  1141. u32 width_b;
  1142. u32 height_b;
  1143. enum ubwcp_std_image_format std_image_format;
  1144. FENTRY();
  1145. if (!dmabuf) {
  1146. ERR("NULL dmabuf input ptr");
  1147. return -EINVAL;
  1148. }
  1149. if (!attr) {
  1150. ERR("NULL attr ptr");
  1151. return -EINVAL;
  1152. }
  1153. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1154. if (!buf) {
  1155. ERR("No corresponding ubwcp_buf for the passed in dma_buf");
  1156. return -EINVAL;
  1157. }
  1158. mutex_lock(&buf->lock);
  1159. if (buf->locked) {
  1160. ERR("Cannot set attr when buffer is locked");
  1161. ret = -EBUSY;
  1162. goto err;
  1163. }
  1164. ubwcp = buf->ubwcp;
  1165. mmdata = &buf->mmdata;
  1166. //TBD: now that we have single exit point for all errors,
  1167. //we can limit this call to error only?
  1168. //also see if this can be part of reset_buf_attrs()
  1169. DBG_BUF_ATTR("resetting mmap to linear");
  1170. /* remove any earlier dma buf mmap configuration */
  1171. ret = ubwcp->mmap_config_fptr(buf->dma_buf, true, 0, 0);
  1172. if (ret) {
  1173. ERR("dma_buf_mmap_config() failed: %d", ret);
  1174. goto err;
  1175. }
  1176. if (!ubwcp_buf_attrs_valid(attr)) {
  1177. ERR("Invalid buf attrs");
  1178. goto err;
  1179. }
  1180. DBG_BUF_ATTR("valid buf attrs");
  1181. if (attr->image_format == UBWCP_LINEAR) {
  1182. DBG_BUF_ATTR("Linear format requested");
  1183. /* linear format request with permanent range xlation doesn't
  1184. * make sense. need to define behavior if this happens.
  1185. * note: with perm set, desc is allocated to this buffer.
  1186. */
  1187. //TBD: UBWCP_ASSERT(!buf->perm);
  1188. if (buf->buf_attr_set)
  1189. reset_buf_attrs(buf);
  1190. mutex_unlock(&buf->lock);
  1191. return 0;
  1192. }
  1193. std_image_format = to_std_format(attr->image_format);
  1194. if (std_image_format == STD_IMAGE_FORMAT_INVALID) {
  1195. ERR("Unable to map ioctl image format to std image format");
  1196. goto err;
  1197. }
  1198. /* Calculate uncompressed-buffer size. */
  1199. DBG_BUF_ATTR("");
  1200. DBG_BUF_ATTR("");
  1201. DBG_BUF_ATTR("Calculating ula params -->");
  1202. ret = ubwcp_calc_ula_params(ubwcp, attr, &ula_size, &ula_y_plane_size, &uv_start_offset);
  1203. if (ret) {
  1204. ERR("ubwcp_calc_ula_params() failed: %d", ret);
  1205. goto err;
  1206. }
  1207. ret = ubwcp_validate_uv_align(ubwcp, attr, ula_y_plane_size, uv_start_offset);
  1208. if (ret) {
  1209. ERR("ubwcp_validate_uv_align() failed: %d", ret);
  1210. goto err;
  1211. }
  1212. DBG_BUF_ATTR("");
  1213. DBG_BUF_ATTR("");
  1214. DBG_BUF_ATTR("Calculating ubwcp params -->");
  1215. ret = ubwcp_calc_ubwcp_buf_params(ubwcp, attr,
  1216. &metadata_p0, &pixeldata_p0,
  1217. &metadata_p1, &pixeldata_p1,
  1218. &stride_tp10_b);
  1219. if (ret) {
  1220. ERR("ubwcp_calc_buf_params() failed: %d", ret);
  1221. goto err;
  1222. }
  1223. iova_min_size = metadata_p0 + pixeldata_p0 + metadata_p1 + pixeldata_p1;
  1224. DBG_BUF_ATTR("");
  1225. DBG_BUF_ATTR("");
  1226. DBG_BUF_ATTR("------Summary ULA Calculated Params ------");
  1227. DBG_BUF_ATTR("ULA Size : %8zu (0x%8zx)", ula_size, ula_size);
  1228. DBG_BUF_ATTR("UV Start Offset : %8zu (0x%8zx)", uv_start_offset, uv_start_offset);
  1229. DBG_BUF_ATTR("------Summary UBCP Calculated Params ------");
  1230. DBG_BUF_ATTR("metadata_p0 : %8d (0x%8zx)", metadata_p0, metadata_p0);
  1231. DBG_BUF_ATTR("pixeldata_p0 : %8d (0x%8zx)", pixeldata_p0, pixeldata_p0);
  1232. DBG_BUF_ATTR("metadata_p1 : %8d (0x%8zx)", metadata_p1, metadata_p1);
  1233. DBG_BUF_ATTR("pixeldata_p1 : %8d (0x%8zx)", pixeldata_p1, pixeldata_p1);
  1234. DBG_BUF_ATTR("stride_tp10 : %8d (0x%8zx)", stride_tp10_b, stride_tp10_b);
  1235. DBG_BUF_ATTR("iova_min_size : %8d (0x%8zx)", iova_min_size, iova_min_size);
  1236. DBG_BUF_ATTR("");
  1237. if (buf->buf_attr_set) {
  1238. /* if buf attr were previously set, these must not be 0 */
  1239. /* TBD: do we need this check in production code? */
  1240. if (!buf->ula_pa) {
  1241. WARN(1, "ula_pa cannot be 0 if buf_attr_set is true!!!");
  1242. goto err;
  1243. }
  1244. if (!buf->ula_size) {
  1245. WARN(1, "ula_size cannot be 0 if buf_attr_set is true!!!");
  1246. goto err;
  1247. }
  1248. }
  1249. /* assign ULA PA with uncompressed-size range */
  1250. ula_pa = ubwcp_ula_realloc(ubwcp, buf->ula_pa, buf->ula_size, ula_size);
  1251. if (!ula_pa) {
  1252. ERR("ubwcp_ula_alloc/realloc() failed. running out of ULA PA space?");
  1253. goto err;
  1254. }
  1255. buf->ula_size = ula_size;
  1256. buf->ula_pa = ula_pa;
  1257. DBG_BUF_ATTR("Allocated ULA_PA: 0x%p of size: 0x%zx", ula_pa, ula_size);
  1258. DBG_BUF_ATTR("");
  1259. /* inform ULA-PA to dma-heap: needed for dma-heap to do CMOs later on */
  1260. DBG_BUF_ATTR("Calling mmap_config(): ULA_PA: 0x%p size: 0x%zx", ula_pa, ula_size);
  1261. ret = ubwcp->mmap_config_fptr(buf->dma_buf, false, buf->ula_pa,
  1262. buf->ula_size);
  1263. if (ret) {
  1264. ERR("dma_buf_mmap_config() failed: %d", ret);
  1265. goto err;
  1266. }
  1267. /* dma map only the first time attribute is set */
  1268. if (!buf->buf_attr_set) {
  1269. /* linear -> ubwcp. map ubwcp buffer */
  1270. ret = ubwcp_dma_map(buf, ubwcp->dev_buf_cb, iova_min_size, &iova_base);
  1271. if (ret) {
  1272. ERR("ubwcp_dma_map() failed: %d", ret);
  1273. goto err;
  1274. }
  1275. DBG_BUF_ATTR("dma_buf IOVA range: 0x%llx + min_size (0x%zx): 0x%llx",
  1276. iova_base, iova_min_size, iova_base + iova_min_size);
  1277. }
  1278. uv_start = ula_pa + uv_start_offset;
  1279. if (!IS_ALIGNED(uv_start, 64)) {
  1280. ERR("ERROR: uv_start is NOT aligned to cache line");
  1281. goto err;
  1282. }
  1283. /* Convert height and width to bytes for writing to mmdata */
  1284. if (std_image_format != TP10) {
  1285. ubwcp_pixel_to_bytes(ubwcp, std_image_format, attr->width,
  1286. attr->height, &width_b, &height_b);
  1287. } else {
  1288. /* for tp10 image compression, we need to program p010 width/height */
  1289. ubwcp_pixel_to_bytes(ubwcp, P010, attr->width,
  1290. attr->height, &width_b, &height_b);
  1291. }
  1292. stride_b = attr->stride;
  1293. /* create the mmdata descriptor */
  1294. memset(mmdata, 0, sizeof(*mmdata));
  1295. mmdata->uv_start_addr = CACHE_ADDR(uv_start);
  1296. mmdata->format = ubwcp_get_hw_image_format_value(attr->image_format);
  1297. if (std_image_format != TP10) {
  1298. mmdata->stride = CACHE_ADDR(stride_b); /* uncompressed stride */
  1299. } else {
  1300. mmdata->stride = CACHE_ADDR(stride_tp10_b); /* compressed stride */
  1301. mmdata->stride_ubwcp = CACHE_ADDR(stride_b); /* uncompressed stride */
  1302. }
  1303. mmdata->metadata_base_y = PAGE_ADDR(iova_base);
  1304. mmdata->metadata_base_uv = PAGE_ADDR(iova_base + metadata_p0 + pixeldata_p0);
  1305. mmdata->buffer_y_offset = PAGE_ADDR(metadata_p0);
  1306. mmdata->buffer_uv_offset = PAGE_ADDR(metadata_p1);
  1307. /* NOTE: For version 1.1, both width & height needs to be in bytes.
  1308. * For other versions, width in bytes & height in pixels.
  1309. */
  1310. if ((ubwcp->hw_ver_major == 1) && (ubwcp->hw_ver_minor == 1))
  1311. mmdata->width_height = width_b << 16 | height_b;
  1312. else
  1313. mmdata->width_height = width_b << 16 | attr->height;
  1314. print_mmdata_desc(mmdata);
  1315. buf->buf_attr = *attr;
  1316. buf->buf_attr_set = true;
  1317. //TBD: UBWCP_ASSERT(!buf->perm);
  1318. mutex_unlock(&buf->lock);
  1319. return 0;
  1320. err:
  1321. reset_buf_attrs(buf);
  1322. mutex_unlock(&buf->lock);
  1323. if (!ret)
  1324. ret = -1;
  1325. return ret;
  1326. }
  1327. EXPORT_SYMBOL(ubwcp_set_buf_attrs);
  1328. /* Set buffer attributes ioctl */
  1329. static int ubwcp_set_buf_attrs_ioctl(struct ubwcp_ioctl_buffer_attrs *attr_ioctl)
  1330. {
  1331. struct dma_buf *dmabuf;
  1332. dmabuf = ubwcp_dma_buf_fd_to_dma_buf(attr_ioctl->fd);
  1333. return ubwcp_set_buf_attrs(dmabuf, &attr_ioctl->attr);
  1334. }
  1335. /* Free up the buffer descriptor */
  1336. static void ubwcp_buf_desc_free(struct ubwcp_driver *ubwcp, struct ubwcp_desc *desc)
  1337. {
  1338. int idx = desc->idx;
  1339. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  1340. mutex_lock(&ubwcp->desc_lock);
  1341. desc_list[idx].idx = -1;
  1342. desc_list[idx].ptr = NULL;
  1343. DBG("freed descriptor_id: %d", idx);
  1344. mutex_unlock(&ubwcp->desc_lock);
  1345. }
  1346. /* Allocate next available buffer descriptor. */
  1347. static struct ubwcp_desc *ubwcp_buf_desc_allocate(struct ubwcp_driver *ubwcp)
  1348. {
  1349. int idx;
  1350. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  1351. mutex_lock(&ubwcp->desc_lock);
  1352. for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
  1353. if (desc_list[idx].idx == -1) {
  1354. desc_list[idx].idx = idx;
  1355. desc_list[idx].ptr = ubwcp->buffer_desc_base +
  1356. idx*UBWCP_BUFFER_DESC_OFFSET;
  1357. DBG("allocated descriptor_id: %d", idx);
  1358. mutex_unlock(&ubwcp->desc_lock);
  1359. return &desc_list[idx];
  1360. }
  1361. }
  1362. mutex_unlock(&ubwcp->desc_lock);
  1363. return NULL;
  1364. }
  1365. /**
  1366. * Lock buffer for CPU access. This prepares ubwcp hw to allow
  1367. * CPU access to the compressed buffer. It will perform
  1368. * necessary address translation configuration and cache maintenance ops
  1369. * so that CPU can safely access ubwcp buffer, if this call is
  1370. * successful.
  1371. * Allocate descriptor if not already,
  1372. * perform CMO and then enable range check
  1373. *
  1374. * @param dmabuf : ptr to the dma buf
  1375. * @param direction : direction of access
  1376. *
  1377. * @return int : 0 on success, otherwise error code
  1378. */
  1379. static int ubwcp_lock(struct dma_buf *dmabuf, enum dma_data_direction dir)
  1380. {
  1381. int ret = 0;
  1382. struct ubwcp_buf *buf;
  1383. struct ubwcp_driver *ubwcp;
  1384. FENTRY();
  1385. if (!dmabuf) {
  1386. ERR("NULL dmabuf input ptr");
  1387. return -EINVAL;
  1388. }
  1389. if (!valid_dma_direction(dir)) {
  1390. ERR("invalid direction: %d", dir);
  1391. return -EINVAL;
  1392. }
  1393. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1394. if (!buf) {
  1395. ERR("ubwcp_buf ptr not found");
  1396. return -1;
  1397. }
  1398. mutex_lock(&buf->lock);
  1399. if (!buf->buf_attr_set) {
  1400. ERR("lock() called on buffer, but attr not set");
  1401. goto err;
  1402. }
  1403. if (buf->buf_attr.image_format == UBWCP_LINEAR) {
  1404. ERR("lock() called on linear buffer");
  1405. goto err;
  1406. }
  1407. if (!buf->locked) {
  1408. DBG("first lock on buffer");
  1409. ubwcp = buf->ubwcp;
  1410. /* buf->desc could already be allocated because of perm range xlation */
  1411. if (!buf->desc) {
  1412. /* allocate a buffer descriptor */
  1413. buf->desc = ubwcp_buf_desc_allocate(buf->ubwcp);
  1414. if (!buf->desc) {
  1415. ERR("ubwcp_allocate_buf_desc() failed");
  1416. goto err;
  1417. }
  1418. memcpy(buf->desc->ptr, &buf->mmdata, sizeof(buf->mmdata));
  1419. /* Flushing of updated mmdata:
  1420. * mmdata is iocoherent and ubwcp will get it from CPU cache -
  1421. * *as long as* it has not cached that itself during previous
  1422. * access to the same descriptor.
  1423. *
  1424. * During unlock of previous use of this descriptor,
  1425. * we do hw flush, which will get rid of this mmdata from
  1426. * ubwcp cache.
  1427. *
  1428. * In addition, we also do a hw flush after enable_range_ck().
  1429. * That will also get rid of any speculative fetch of mmdata
  1430. * by the ubwcp hw. At this time, the assumption is that ubwcp
  1431. * will cache mmdata only for active descriptor. But if ubwcp
  1432. * is speculatively fetching mmdata for all descriptors
  1433. * (irrespetive of enabled or not), the flush during lock
  1434. * will be necessary to make sure ubwcp sees updated mmdata
  1435. * that we just updated
  1436. */
  1437. /* program ULA range for this buffer */
  1438. DBG("setting range check: descriptor_id: %d, addr: %p, size: %zx",
  1439. buf->desc->idx, buf->ula_pa, buf->ula_size);
  1440. ubwcp_hw_set_range_check(ubwcp->base, buf->desc->idx, buf->ula_pa,
  1441. buf->ula_size);
  1442. }
  1443. /* enable range check */
  1444. DBG("enabling range check, descriptor_id: %d", buf->desc->idx);
  1445. mutex_lock(&ubwcp->hw_range_ck_lock);
  1446. ubwcp_hw_enable_range_check(ubwcp->base, buf->desc->idx);
  1447. mutex_unlock(&ubwcp->hw_range_ck_lock);
  1448. /* Flush/invalidate UBWCP caches */
  1449. /* Why: cpu could have done a speculative fetch before
  1450. * enable_range_ck() and ubwcp in process of returning "default" data
  1451. * we don't want that stashing of default data pending.
  1452. * we force completion of that and then we also cpu invalidate which
  1453. * will get rid of that line.
  1454. */
  1455. ubwcp_flush(ubwcp);
  1456. /* Flush/invalidate ULA PA from CPU caches
  1457. * TBD: if (dir == READ or BIDIRECTION) //NOT for write
  1458. * -- Confirm with Chris if this can be skipped for write
  1459. */
  1460. dma_sync_single_for_cpu(ubwcp->dev, buf->ula_pa, buf->ula_size, dir);
  1461. buf->lock_dir = dir;
  1462. buf->locked = true;
  1463. } else {
  1464. DBG("buf already locked");
  1465. /* TBD: what if new buffer direction is not same as previous?
  1466. * must update the dir.
  1467. */
  1468. }
  1469. buf->lock_count++;
  1470. DBG("new lock_count: %d", buf->lock_count);
  1471. mutex_unlock(&buf->lock);
  1472. return ret;
  1473. err:
  1474. mutex_unlock(&buf->lock);
  1475. if (!ret)
  1476. ret = -1;
  1477. return ret;
  1478. }
  1479. /* This can be called as a result of external unlock() call or
  1480. * internally if free() is called without unlock().
  1481. */
  1482. static int unlock_internal(struct ubwcp_buf *buf, enum dma_data_direction dir, bool free_buffer)
  1483. {
  1484. int ret = 0;
  1485. struct ubwcp_driver *ubwcp;
  1486. DBG("current lock_count: %d", buf->lock_count);
  1487. if (free_buffer) {
  1488. buf->lock_count = 0;
  1489. DBG("Forced lock_count: %d", buf->lock_count);
  1490. } else {
  1491. buf->lock_count--;
  1492. DBG("new lock_count: %d", buf->lock_count);
  1493. if (buf->lock_count) {
  1494. DBG("more than 1 lock on buffer. waiting until last unlock");
  1495. return 0;
  1496. }
  1497. }
  1498. ubwcp = buf->ubwcp;
  1499. /* Flush/invalidate ULA PA from CPU caches */
  1500. //TBD: if (dir == WRITE or BIDIRECTION)
  1501. dma_sync_single_for_device(ubwcp->dev, buf->ula_pa, buf->ula_size, dir);
  1502. /* disable range check with ubwcp flush */
  1503. DBG("disabling range check");
  1504. //TBD: could combine these 2 locks into a single lock to make it simpler
  1505. mutex_lock(&ubwcp->ubwcp_flush_lock);
  1506. mutex_lock(&ubwcp->hw_range_ck_lock);
  1507. ret = ubwcp_hw_disable_range_check_with_flush(ubwcp->base, buf->desc->idx);
  1508. if (ret)
  1509. ERR("disable_range_check_with_flush() failed: %d", ret);
  1510. mutex_unlock(&ubwcp->hw_range_ck_lock);
  1511. mutex_unlock(&ubwcp->ubwcp_flush_lock);
  1512. /* release descriptor if perm range xlation is not set */
  1513. if (!buf->perm) {
  1514. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1515. buf->desc = NULL;
  1516. }
  1517. buf->locked = false;
  1518. return ret;
  1519. }
  1520. /**
  1521. * Unlock buffer from CPU access. This prepares ubwcp hw to
  1522. * safely allow for device access to the compressed buffer including any
  1523. * necessary cache maintenance ops. It may also free up certain ubwcp
  1524. * resources that could result in error when accessed by CPU in
  1525. * unlocked state.
  1526. *
  1527. * @param dmabuf : ptr to the dma buf
  1528. * @param direction : direction of access
  1529. *
  1530. * @return int : 0 on success, otherwise error code
  1531. */
  1532. static int ubwcp_unlock(struct dma_buf *dmabuf, enum dma_data_direction dir)
  1533. {
  1534. struct ubwcp_buf *buf;
  1535. int ret;
  1536. FENTRY();
  1537. if (!dmabuf) {
  1538. ERR("NULL dmabuf input ptr");
  1539. return -EINVAL;
  1540. }
  1541. if (!valid_dma_direction(dir)) {
  1542. ERR("invalid direction: %d", dir);
  1543. return -EINVAL;
  1544. }
  1545. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1546. if (!buf) {
  1547. ERR("ubwcp_buf not found");
  1548. return -1;
  1549. }
  1550. if (!buf->locked) {
  1551. ERR("unlock() called on buffer which not in locked state");
  1552. return -1;
  1553. }
  1554. error_print_count = 0;
  1555. mutex_lock(&buf->lock);
  1556. ret = unlock_internal(buf, dir, false);
  1557. mutex_unlock(&buf->lock);
  1558. return ret;
  1559. }
  1560. /* Return buffer attributes for the given buffer */
  1561. int ubwcp_get_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
  1562. {
  1563. int ret = 0;
  1564. struct ubwcp_buf *buf;
  1565. FENTRY();
  1566. if (!dmabuf) {
  1567. ERR("NULL dmabuf input ptr");
  1568. return -EINVAL;
  1569. }
  1570. if (!attr) {
  1571. ERR("NULL attr ptr");
  1572. return -EINVAL;
  1573. }
  1574. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1575. if (!buf) {
  1576. ERR("ubwcp_buf ptr not found");
  1577. return -1;
  1578. }
  1579. mutex_lock(&buf->lock);
  1580. if (!buf->buf_attr_set) {
  1581. ERR("buffer attributes not set");
  1582. mutex_unlock(&buf->lock);
  1583. return -1;
  1584. }
  1585. *attr = buf->buf_attr;
  1586. mutex_unlock(&buf->lock);
  1587. return ret;
  1588. }
  1589. EXPORT_SYMBOL(ubwcp_get_buf_attrs);
  1590. /* Set permanent range translation.
  1591. * enable: Descriptor will be reserved for this buffer until disabled,
  1592. * making lock/unlock quicker.
  1593. * disable: Descriptor will not be reserved for this buffer. Instead,
  1594. * descriptor will be allocated and released for each lock/unlock.
  1595. * If currently allocated but not being used, descriptor will be
  1596. * released.
  1597. */
  1598. int ubwcp_set_perm_range_translation(struct dma_buf *dmabuf, bool enable)
  1599. {
  1600. int ret = 0;
  1601. struct ubwcp_buf *buf;
  1602. FENTRY();
  1603. if (!dmabuf) {
  1604. ERR("NULL dmabuf input ptr");
  1605. return -EINVAL;
  1606. }
  1607. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1608. if (!buf) {
  1609. ERR("ubwcp_buf not found");
  1610. return -1;
  1611. }
  1612. /* not implemented */
  1613. if (1) {
  1614. ERR("API not implemented yet");
  1615. return -1;
  1616. }
  1617. /* TBD: make sure we acquire buf lock while setting this so there is
  1618. * no race condition with attr_set/lock/unlock
  1619. */
  1620. buf->perm = enable;
  1621. /* if "disable" and we have allocated a desc and it is not being
  1622. * used currently, release it
  1623. */
  1624. if (!enable && buf->desc && !buf->locked) {
  1625. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1626. buf->desc = NULL;
  1627. /* Flush/invalidate UBWCP caches */
  1628. //TBD: need to do anything?
  1629. }
  1630. return ret;
  1631. }
  1632. EXPORT_SYMBOL(ubwcp_set_perm_range_translation);
  1633. /**
  1634. * Free up ubwcp resources for this buffer.
  1635. *
  1636. * @param dmabuf : ptr to the dma buf
  1637. *
  1638. * @return int : 0 on success, otherwise error code
  1639. */
  1640. static int ubwcp_free_buffer(struct dma_buf *dmabuf)
  1641. {
  1642. int ret = 0;
  1643. struct ubwcp_buf *buf;
  1644. struct ubwcp_driver *ubwcp;
  1645. bool table_empty;
  1646. unsigned long flags;
  1647. FENTRY();
  1648. if (!dmabuf) {
  1649. ERR("NULL dmabuf input ptr");
  1650. return -EINVAL;
  1651. }
  1652. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1653. if (!buf) {
  1654. ERR("ubwcp_buf ptr not found");
  1655. return -1;
  1656. }
  1657. mutex_lock(&buf->lock);
  1658. ubwcp = buf->ubwcp;
  1659. if (buf->locked) {
  1660. DBG("free() called without unlock. unlock()'ing first...");
  1661. ret = unlock_internal(buf, buf->lock_dir, true);
  1662. if (ret)
  1663. ERR("unlock_internal(): failed : %d, but continuing free()", ret);
  1664. }
  1665. /* if we are still holding a desc, release it. this can happen only if perm == true */
  1666. if (buf->desc) {
  1667. WARN_ON(!buf->perm); /* TBD: change to BUG() later...*/
  1668. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1669. buf->desc = NULL;
  1670. }
  1671. if (buf->buf_attr_set)
  1672. reset_buf_attrs(buf);
  1673. mutex_lock(&ubwcp->mem_hotplug_lock);
  1674. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  1675. hash_del(&buf->hnode);
  1676. table_empty = hash_empty(ubwcp->buf_table);
  1677. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  1678. kfree(buf);
  1679. /* If this is the last buffer being freed, power off ubwcp */
  1680. if (table_empty) {
  1681. DBG("last buffer: ~~~~~~~~~~~");
  1682. /* TBD: If everything is working fine, ubwcp_flush() should not
  1683. * be needed here. Each buffer free logic should be taking
  1684. * care of flush. Just a note for now. Might need to add the
  1685. * flush here for debug purpose.
  1686. */
  1687. DBG("Calling offline_and_remove_memory() for ULA PA pool");
  1688. ret = offline_and_remove_memory(ubwcp->ula_pool_base,
  1689. ubwcp->ula_pool_size);
  1690. if (ret) {
  1691. ERR("offline_and_remove_memory failed st:0x%lx sz:0x%lx err: %d",
  1692. ubwcp->ula_pool_base,
  1693. ubwcp->ula_pool_size, ret);
  1694. goto err_remove_mem;
  1695. } else {
  1696. DBG("DONE: calling offline_and_remove_memory() for ULA PA pool");
  1697. }
  1698. DBG("Don't Call power OFF ...");
  1699. }
  1700. mutex_unlock(&ubwcp->mem_hotplug_lock);
  1701. return ret;
  1702. err_remove_mem:
  1703. mutex_unlock(&ubwcp->mem_hotplug_lock);
  1704. if (!ret)
  1705. ret = -1;
  1706. DBG("returning error: %d", ret);
  1707. return ret;
  1708. }
  1709. /* file open: TBD: increment ref count? */
  1710. static int ubwcp_open(struct inode *i, struct file *f)
  1711. {
  1712. return 0;
  1713. }
  1714. /* file open: TBD: decrement ref count? */
  1715. static int ubwcp_close(struct inode *i, struct file *f)
  1716. {
  1717. return 0;
  1718. }
  1719. /* handle IOCTLs */
  1720. static long ubwcp_ioctl(struct file *file, unsigned int ioctl_num, unsigned long ioctl_param)
  1721. {
  1722. struct ubwcp_ioctl_buffer_attrs buf_attr_ioctl;
  1723. struct ubwcp_ioctl_hw_version hw_ver;
  1724. switch (ioctl_num) {
  1725. case UBWCP_IOCTL_SET_BUF_ATTR:
  1726. if (copy_from_user(&buf_attr_ioctl, (const void __user *) ioctl_param,
  1727. sizeof(buf_attr_ioctl))) {
  1728. ERR("ERROR: copy_from_user() failed");
  1729. return -EFAULT;
  1730. }
  1731. DBG("IOCTL : SET_BUF_ATTR: fd = %d", buf_attr_ioctl.fd);
  1732. return ubwcp_set_buf_attrs_ioctl(&buf_attr_ioctl);
  1733. case UBWCP_IOCTL_GET_HW_VER:
  1734. DBG("IOCTL : GET_HW_VER");
  1735. ubwcp_get_hw_version(&hw_ver);
  1736. if (copy_to_user((void __user *)ioctl_param, &hw_ver, sizeof(hw_ver))) {
  1737. ERR("ERROR: copy_to_user() failed");
  1738. return -EFAULT;
  1739. }
  1740. break;
  1741. default:
  1742. ERR("Invalid ioctl_num = %d", ioctl_num);
  1743. return -EINVAL;
  1744. }
  1745. return 0;
  1746. }
  1747. static const struct file_operations ubwcp_fops = {
  1748. .owner = THIS_MODULE,
  1749. .open = ubwcp_open,
  1750. .release = ubwcp_close,
  1751. .unlocked_ioctl = ubwcp_ioctl,
  1752. };
  1753. static int ubwcp_debugfs_init(struct ubwcp_driver *ubwcp)
  1754. {
  1755. struct dentry *debugfs_root;
  1756. debugfs_root = debugfs_create_dir("ubwcp", NULL);
  1757. if (!debugfs_root) {
  1758. pr_warn("Failed to create debugfs for ubwcp\n");
  1759. return -1;
  1760. }
  1761. debugfs_create_u32("debug_trace_enable", 0644, debugfs_root, &ubwcp_debug_trace_enable);
  1762. ubwcp->debugfs_root = debugfs_root;
  1763. return 0;
  1764. }
  1765. static void ubwcp_debugfs_deinit(struct ubwcp_driver *ubwcp)
  1766. {
  1767. debugfs_remove_recursive(ubwcp->debugfs_root);
  1768. }
  1769. /* ubwcp char device initialization */
  1770. static int ubwcp_cdev_init(struct ubwcp_driver *ubwcp)
  1771. {
  1772. int ret;
  1773. dev_t devt;
  1774. struct class *dev_class;
  1775. struct device *dev_sys;
  1776. /* allocate major device number (/proc/devices -> major_num ubwcp) */
  1777. ret = alloc_chrdev_region(&devt, 0, UBWCP_NUM_DEVICES, UBWCP_DEVICE_NAME);
  1778. if (ret) {
  1779. ERR("alloc_chrdev_region() failed: %d", ret);
  1780. return ret;
  1781. }
  1782. /* create device class (/sys/class/ubwcp_class) */
  1783. dev_class = class_create(THIS_MODULE, "ubwcp_class");
  1784. if (IS_ERR(dev_class)) {
  1785. ERR("class_create() failed");
  1786. return -1;
  1787. }
  1788. /* Create device and register with sysfs
  1789. * (/sys/class/ubwcp_class/ubwcp/... -> dev/power/subsystem/uevent)
  1790. */
  1791. dev_sys = device_create(dev_class, NULL, devt, NULL,
  1792. UBWCP_DEVICE_NAME);
  1793. if (IS_ERR(dev_sys)) {
  1794. ERR("device_create() failed");
  1795. return -1;
  1796. }
  1797. /* register file operations and get cdev */
  1798. cdev_init(&ubwcp->cdev, &ubwcp_fops);
  1799. /* associate cdev and device major/minor with file system
  1800. * can do file ops on /dev/ubwcp after this
  1801. */
  1802. ret = cdev_add(&ubwcp->cdev, devt, 1);
  1803. if (ret) {
  1804. ERR("cdev_add() failed");
  1805. return -1;
  1806. }
  1807. ubwcp->devt = devt;
  1808. ubwcp->dev_class = dev_class;
  1809. ubwcp->dev_sys = dev_sys;
  1810. return 0;
  1811. }
  1812. static void ubwcp_cdev_deinit(struct ubwcp_driver *ubwcp)
  1813. {
  1814. device_destroy(ubwcp->dev_class, ubwcp->devt);
  1815. class_destroy(ubwcp->dev_class);
  1816. cdev_del(&ubwcp->cdev);
  1817. unregister_chrdev_region(ubwcp->devt, UBWCP_NUM_DEVICES);
  1818. }
  1819. struct handler_node {
  1820. struct list_head list;
  1821. u32 client_id;
  1822. ubwcp_error_handler_t handler;
  1823. void *data;
  1824. };
  1825. int ubwcp_register_error_handler(u32 client_id, ubwcp_error_handler_t handler,
  1826. void *data)
  1827. {
  1828. struct handler_node *node;
  1829. unsigned long flags;
  1830. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1831. if (!ubwcp)
  1832. return -EINVAL;
  1833. if (client_id != -1)
  1834. return -EINVAL;
  1835. if (!handler)
  1836. return -EINVAL;
  1837. node = kzalloc(sizeof(*node), GFP_KERNEL);
  1838. if (!node)
  1839. return -ENOMEM;
  1840. node->client_id = client_id;
  1841. node->handler = handler;
  1842. node->data = data;
  1843. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  1844. list_add_tail(&node->list, &ubwcp->err_handler_list);
  1845. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  1846. return 0;
  1847. }
  1848. EXPORT_SYMBOL(ubwcp_register_error_handler);
  1849. static void ubwcp_notify_error_handlers(struct unwcp_err_info *err)
  1850. {
  1851. struct handler_node *node;
  1852. unsigned long flags;
  1853. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1854. if (!ubwcp)
  1855. return;
  1856. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  1857. list_for_each_entry(node, &ubwcp->err_handler_list, list)
  1858. node->handler(err, node->data);
  1859. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  1860. }
  1861. int ubwcp_unregister_error_handler(u32 client_id)
  1862. {
  1863. int ret = -EINVAL;
  1864. struct handler_node *node;
  1865. unsigned long flags;
  1866. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1867. if (!ubwcp)
  1868. return -EINVAL;
  1869. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  1870. list_for_each_entry(node, &ubwcp->err_handler_list, list)
  1871. if (node->client_id == client_id) {
  1872. list_del(&node->list);
  1873. kfree(node);
  1874. ret = 0;
  1875. break;
  1876. }
  1877. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  1878. return ret;
  1879. }
  1880. EXPORT_SYMBOL(ubwcp_unregister_error_handler);
  1881. /* get ubwcp_buf corresponding to the ULA PA*/
  1882. static struct dma_buf *get_dma_buf_from_ulapa(phys_addr_t addr)
  1883. {
  1884. struct ubwcp_buf *buf = NULL;
  1885. struct dma_buf *ret_buf = NULL;
  1886. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1887. unsigned long flags;
  1888. u32 i;
  1889. if (!ubwcp)
  1890. return NULL;
  1891. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  1892. hash_for_each(ubwcp->buf_table, i, buf, hnode) {
  1893. if (buf->ula_pa <= addr && addr < buf->ula_pa + buf->ula_size) {
  1894. ret_buf = buf->dma_buf;
  1895. break;
  1896. }
  1897. }
  1898. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  1899. return ret_buf;
  1900. }
  1901. /* get ubwcp_buf corresponding to the IOVA*/
  1902. static struct dma_buf *get_dma_buf_from_iova(unsigned long addr)
  1903. {
  1904. struct ubwcp_buf *buf = NULL;
  1905. struct dma_buf *ret_buf = NULL;
  1906. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1907. unsigned long flags;
  1908. u32 i;
  1909. if (!ubwcp)
  1910. return NULL;
  1911. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  1912. hash_for_each(ubwcp->buf_table, i, buf, hnode) {
  1913. unsigned long iova_base = sg_dma_address(buf->sgt->sgl);
  1914. unsigned int iova_size = sg_dma_len(buf->sgt->sgl);
  1915. if (iova_base <= addr && addr < iova_base + iova_size) {
  1916. ret_buf = buf->dma_buf;
  1917. break;
  1918. }
  1919. }
  1920. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  1921. return ret_buf;
  1922. }
  1923. #define ERR_PRINT_COUNT_MAX 21
  1924. /* TBD: use proper rate limit for debug prints */
  1925. int ubwcp_iommu_fault_handler(struct iommu_domain *domain, struct device *dev,
  1926. unsigned long iova, int flags, void *data)
  1927. {
  1928. int ret = 0;
  1929. struct unwcp_err_info err;
  1930. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1931. struct device *cb_dev = (struct device *)data;
  1932. if (!ubwcp) {
  1933. ret = -EINVAL;
  1934. goto err;
  1935. }
  1936. error_print_count++;
  1937. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  1938. err.err_code = UBWCP_SMMU_FAULT;
  1939. if (cb_dev == ubwcp->dev_desc_cb)
  1940. err.smmu_err.iommu_dev_id = UBWCP_DESC_CB_ID;
  1941. else if (cb_dev == ubwcp->dev_buf_cb)
  1942. err.smmu_err.iommu_dev_id = UBWCP_BUF_CB_ID;
  1943. else
  1944. err.smmu_err.iommu_dev_id = UBWCP_UNKNOWN_CB_ID;
  1945. ERR("smmu fault error: iommu_dev_id:%d iova 0x%llx flags:0x%x",
  1946. err.smmu_err.iommu_dev_id, iova, flags);
  1947. err.smmu_err.dmabuf = get_dma_buf_from_iova(iova);
  1948. err.smmu_err.iova = iova;
  1949. err.smmu_err.iommu_fault_flags = flags;
  1950. ubwcp_notify_error_handlers(&err);
  1951. }
  1952. err:
  1953. return ret;
  1954. }
  1955. irqreturn_t ubwcp_irq_handler(int irq, void *ptr)
  1956. {
  1957. struct ubwcp_driver *ubwcp;
  1958. void __iomem *base;
  1959. u64 src;
  1960. phys_addr_t addr;
  1961. struct unwcp_err_info err;
  1962. error_print_count++;
  1963. ubwcp = (struct ubwcp_driver *) ptr;
  1964. base = ubwcp->base;
  1965. if (irq == ubwcp->irq_range_ck_rd) {
  1966. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  1967. src = ubwcp_hw_interrupt_src_address(base, 0);
  1968. addr = src << 6;
  1969. ERR("check range read error: src: 0x%llx", addr);
  1970. err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
  1971. err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
  1972. err.translation_err.ula_pa = addr;
  1973. err.translation_err.read = true;
  1974. ubwcp_notify_error_handlers(&err);
  1975. }
  1976. ubwcp_hw_interrupt_clear(ubwcp->base, 0);
  1977. } else if (irq == ubwcp->irq_range_ck_wr) {
  1978. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  1979. src = ubwcp_hw_interrupt_src_address(base, 1);
  1980. addr = src << 6;
  1981. ERR("check range write error: src: 0x%llx", addr);
  1982. err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
  1983. err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
  1984. err.translation_err.ula_pa = addr;
  1985. err.translation_err.read = false;
  1986. ubwcp_notify_error_handlers(&err);
  1987. }
  1988. ubwcp_hw_interrupt_clear(ubwcp->base, 1);
  1989. } else if (irq == ubwcp->irq_encode) {
  1990. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  1991. src = ubwcp_hw_interrupt_src_address(base, 3);
  1992. addr = src << 6;
  1993. ERR("encode error: src: 0x%llx", addr);
  1994. err.err_code = UBWCP_ENCODE_ERROR;
  1995. err.enc_err.dmabuf = get_dma_buf_from_ulapa(addr);
  1996. err.enc_err.ula_pa = addr;
  1997. ubwcp_notify_error_handlers(&err);
  1998. }
  1999. ubwcp_hw_interrupt_clear(ubwcp->base, 3); //TBD: encode is bit-3 instead of bit-2
  2000. } else if (irq == ubwcp->irq_decode) {
  2001. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  2002. src = ubwcp_hw_interrupt_src_address(base, 2);
  2003. addr = src << 6;
  2004. ERR("decode error: src: 0x%llx", addr);
  2005. err.err_code = UBWCP_DECODE_ERROR;
  2006. err.dec_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2007. err.dec_err.ula_pa = addr;
  2008. ubwcp_notify_error_handlers(&err);
  2009. }
  2010. ubwcp_hw_interrupt_clear(ubwcp->base, 2); //TBD: decode is bit-2 instead of bit-3
  2011. } else {
  2012. ERR("unknown irq: %d", irq);
  2013. return IRQ_NONE;
  2014. }
  2015. return IRQ_HANDLED;
  2016. }
  2017. static int ubwcp_interrupt_register(struct platform_device *pdev, struct ubwcp_driver *ubwcp)
  2018. {
  2019. int ret = 0;
  2020. struct device *dev = &pdev->dev;
  2021. FENTRY();
  2022. ubwcp->irq_range_ck_rd = platform_get_irq(pdev, 0);
  2023. if (ubwcp->irq_range_ck_rd < 0)
  2024. return ubwcp->irq_range_ck_rd;
  2025. ubwcp->irq_range_ck_wr = platform_get_irq(pdev, 1);
  2026. if (ubwcp->irq_range_ck_wr < 0)
  2027. return ubwcp->irq_range_ck_wr;
  2028. ubwcp->irq_encode = platform_get_irq(pdev, 2);
  2029. if (ubwcp->irq_encode < 0)
  2030. return ubwcp->irq_encode;
  2031. ubwcp->irq_decode = platform_get_irq(pdev, 3);
  2032. if (ubwcp->irq_decode < 0)
  2033. return ubwcp->irq_decode;
  2034. DBG("got irqs: %d %d %d %d", ubwcp->irq_range_ck_rd,
  2035. ubwcp->irq_range_ck_wr,
  2036. ubwcp->irq_encode,
  2037. ubwcp->irq_decode);
  2038. ret = devm_request_irq(dev, ubwcp->irq_range_ck_rd, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2039. if (ret) {
  2040. ERR("request_irq() failed. irq: %d ret: %d",
  2041. ubwcp->irq_range_ck_rd, ret);
  2042. return ret;
  2043. }
  2044. ret = devm_request_irq(dev, ubwcp->irq_range_ck_wr, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2045. if (ret) {
  2046. ERR("request_irq() failed. irq: %d ret: %d",
  2047. ubwcp->irq_range_ck_wr, ret);
  2048. return ret;
  2049. }
  2050. ret = devm_request_irq(dev, ubwcp->irq_encode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2051. if (ret) {
  2052. ERR("request_irq() failed. irq: %d ret: %d",
  2053. ubwcp->irq_encode, ret);
  2054. return ret;
  2055. }
  2056. ret = devm_request_irq(dev, ubwcp->irq_decode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2057. if (ret) {
  2058. ERR("request_irq() failed. irq: %d ret: %d",
  2059. ubwcp->irq_decode, ret);
  2060. return ret;
  2061. }
  2062. return ret;
  2063. }
  2064. /* ubwcp device probe */
  2065. static int qcom_ubwcp_probe(struct platform_device *pdev)
  2066. {
  2067. int ret = 0;
  2068. struct ubwcp_driver *ubwcp;
  2069. struct device *ubwcp_dev = &pdev->dev;
  2070. FENTRY();
  2071. ubwcp = devm_kzalloc(ubwcp_dev, sizeof(*ubwcp), GFP_KERNEL);
  2072. if (!ubwcp) {
  2073. ERR("devm_kzalloc() failed");
  2074. return -ENOMEM;
  2075. }
  2076. ubwcp->dev = &pdev->dev;
  2077. ret = dma_set_mask_and_coherent(ubwcp->dev, DMA_BIT_MASK(64));
  2078. #ifdef UBWCP_USE_SMC
  2079. {
  2080. struct resource res;
  2081. of_address_to_resource(ubwcp_dev->of_node, 0, &res);
  2082. ubwcp->base = (void __iomem *) res.start;
  2083. DBG("Using SMC calls. base: %p", ubwcp->base);
  2084. }
  2085. #else
  2086. ubwcp->base = devm_platform_ioremap_resource(pdev, 0);
  2087. if (IS_ERR(ubwcp->base)) {
  2088. ERR("devm ioremap() failed: %d", PTR_ERR(ubwcp->base));
  2089. return PTR_ERR(ubwcp->base);
  2090. }
  2091. DBG("ubwcp->base: %p", ubwcp->base);
  2092. #endif
  2093. ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 0, &ubwcp->ula_pool_base);
  2094. if (ret) {
  2095. ERR("failed reading ula_range (base): %d", ret);
  2096. return ret;
  2097. }
  2098. DBG("ubwcp: ula_range: base = 0x%lx", ubwcp->ula_pool_base);
  2099. ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 1, &ubwcp->ula_pool_size);
  2100. if (ret) {
  2101. ERR("failed reading ula_range (size): %d", ret);
  2102. return ret;
  2103. }
  2104. DBG("ubwcp: ula_range: size = 0x%lx", ubwcp->ula_pool_size);
  2105. /*TBD: remove later. reducing size for quick testing...*/
  2106. ubwcp->ula_pool_size = 0x20000000; //500MB instead of 8GB
  2107. INIT_LIST_HEAD(&ubwcp->err_handler_list);
  2108. mutex_init(&ubwcp->desc_lock);
  2109. spin_lock_init(&ubwcp->buf_table_lock);
  2110. mutex_init(&ubwcp->mem_hotplug_lock);
  2111. mutex_init(&ubwcp->ula_lock);
  2112. mutex_init(&ubwcp->ubwcp_flush_lock);
  2113. mutex_init(&ubwcp->hw_range_ck_lock);
  2114. spin_lock_init(&ubwcp->err_handler_list_lock);
  2115. if (ubwcp_interrupt_register(pdev, ubwcp))
  2116. return -1;
  2117. /* Regulator */
  2118. ubwcp->vdd = devm_regulator_get(ubwcp_dev, "vdd");
  2119. if (IS_ERR_OR_NULL(ubwcp->vdd)) {
  2120. ret = PTR_ERR(ubwcp->vdd);
  2121. ERR("devm_regulator_get() failed: %d", ret);
  2122. return -1;
  2123. }
  2124. ret = ubwcp_init_clocks(ubwcp, ubwcp_dev);
  2125. if (ret) {
  2126. ERR("failed to initialize ubwcp clocks err: %d", ret);
  2127. return ret;
  2128. }
  2129. if (ubwcp_power(ubwcp, true))
  2130. return -1;
  2131. if (ubwcp_cdev_init(ubwcp))
  2132. return -1;
  2133. if (ubwcp_debugfs_init(ubwcp))
  2134. return -1;
  2135. /* create ULA pool */
  2136. ubwcp->ula_pool = gen_pool_create(12, -1);
  2137. if (!ubwcp->ula_pool) {
  2138. ERR("failed gen_pool_create()");
  2139. ret = -1;
  2140. goto err_pool_create;
  2141. }
  2142. ret = gen_pool_add(ubwcp->ula_pool, ubwcp->ula_pool_base, ubwcp->ula_pool_size, -1);
  2143. if (ret) {
  2144. ERR("failed gen_pool_add(): %d", ret);
  2145. ret = -1;
  2146. goto err_pool_add;
  2147. }
  2148. /* register the default config mmap function. */
  2149. ubwcp->mmap_config_fptr = msm_ubwcp_dma_buf_configure_mmap;
  2150. hash_init(ubwcp->buf_table);
  2151. ubwcp_buf_desc_list_init(ubwcp);
  2152. image_format_init(ubwcp);
  2153. /* one time hw init */
  2154. ubwcp_hw_one_time_init(ubwcp->base);
  2155. ubwcp_hw_version(ubwcp->base, &ubwcp->hw_ver_major, &ubwcp->hw_ver_minor);
  2156. pr_err("ubwcp: hw version: major %d, minor %d\n", ubwcp->hw_ver_major, ubwcp->hw_ver_minor);
  2157. if (ubwcp->hw_ver_major == 0) {
  2158. ERR("Failed to read HW version");
  2159. ret = -1;
  2160. goto err_pool_add;
  2161. }
  2162. /* set pdev->dev->driver_data = ubwcp */
  2163. platform_set_drvdata(pdev, ubwcp);
  2164. /* enable all 4 interrupts */
  2165. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, true);
  2166. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, true);
  2167. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, true);
  2168. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, true);
  2169. /* Turn OFF until buffers are allocated */
  2170. if (ubwcp_power(ubwcp, false)) {
  2171. ret = -1;
  2172. goto err_power_off;
  2173. }
  2174. ret = msm_ubwcp_set_ops(ubwcp_init_buffer, ubwcp_free_buffer, ubwcp_lock, ubwcp_unlock);
  2175. if (ret) {
  2176. ERR("msm_ubwcp_set_ops() failed: %d, but IGNORED", ret);
  2177. /* TBD: ignore return error during testing phase.
  2178. * This allows us to rmmod/insmod for faster dev cycle.
  2179. * In final version: return error and de-register driver if set_ops fails.
  2180. */
  2181. ret = 0;
  2182. //goto err_power_off;
  2183. } else {
  2184. DBG("msm_ubwcp_set_ops(): success"); }
  2185. me = ubwcp;
  2186. return ret;
  2187. err_power_off:
  2188. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2189. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2190. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2191. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2192. err_pool_add:
  2193. gen_pool_destroy(ubwcp->ula_pool);
  2194. err_pool_create:
  2195. ubwcp_cdev_deinit(ubwcp);
  2196. return ret;
  2197. }
  2198. /* buffer context bank device probe */
  2199. static int ubwcp_probe_cb_buf(struct platform_device *pdev)
  2200. {
  2201. struct ubwcp_driver *ubwcp;
  2202. struct iommu_domain *domain = NULL;
  2203. FENTRY();
  2204. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2205. if (!ubwcp) {
  2206. ERR("failed to get ubwcp ptr");
  2207. return -EINVAL;
  2208. }
  2209. /* save the buffer cb device */
  2210. ubwcp->dev_buf_cb = &pdev->dev;
  2211. domain = iommu_get_domain_for_dev(ubwcp->dev_buf_cb);
  2212. if (domain)
  2213. iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_buf_cb);
  2214. return 0;
  2215. }
  2216. /* descriptor context bank device probe */
  2217. static int ubwcp_probe_cb_desc(struct platform_device *pdev)
  2218. {
  2219. int ret = 0;
  2220. struct ubwcp_driver *ubwcp;
  2221. struct iommu_domain *domain = NULL;
  2222. FENTRY();
  2223. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2224. if (!ubwcp) {
  2225. ERR("failed to get ubwcp ptr");
  2226. return -EINVAL;
  2227. }
  2228. ubwcp->buffer_desc_size = UBWCP_BUFFER_DESC_OFFSET *
  2229. UBWCP_BUFFER_DESC_COUNT;
  2230. ubwcp->dev_desc_cb = &pdev->dev;
  2231. dma_set_max_seg_size(ubwcp->dev_desc_cb, DMA_BIT_MASK(32));
  2232. dma_set_seg_boundary(ubwcp->dev_desc_cb, (unsigned long)DMA_BIT_MASK(64));
  2233. /* Allocate buffer descriptors. UBWCP is iocoherent device.
  2234. * Thus we don't need to flush after updates to buffer descriptors.
  2235. */
  2236. ubwcp->buffer_desc_base = dma_alloc_coherent(ubwcp->dev_desc_cb,
  2237. ubwcp->buffer_desc_size,
  2238. &ubwcp->buffer_desc_dma_handle,
  2239. GFP_KERNEL);
  2240. if (!ubwcp->buffer_desc_base) {
  2241. ERR("failed to allocate desc buffer");
  2242. return -ENOMEM;
  2243. }
  2244. DBG("desc_base = %p size = %zu", ubwcp->buffer_desc_base,
  2245. ubwcp->buffer_desc_size);
  2246. ret = ubwcp_power(ubwcp, true);
  2247. if (ret) {
  2248. ERR("failed to power on");
  2249. goto err;
  2250. }
  2251. ubwcp_hw_set_buf_desc(ubwcp->base, (u64) ubwcp->buffer_desc_dma_handle,
  2252. UBWCP_BUFFER_DESC_OFFSET);
  2253. ret = ubwcp_power(ubwcp, false);
  2254. if (ret) {
  2255. ERR("failed to power off");
  2256. goto err;
  2257. }
  2258. domain = iommu_get_domain_for_dev(ubwcp->dev_desc_cb);
  2259. if (domain)
  2260. iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_desc_cb);
  2261. return ret;
  2262. err:
  2263. dma_free_coherent(ubwcp->dev_desc_cb,
  2264. ubwcp->buffer_desc_size,
  2265. ubwcp->buffer_desc_base,
  2266. ubwcp->buffer_desc_dma_handle);
  2267. ubwcp->buffer_desc_base = NULL;
  2268. ubwcp->buffer_desc_dma_handle = 0;
  2269. ubwcp->dev_desc_cb = NULL;
  2270. return -1;
  2271. }
  2272. /* buffer context bank device remove */
  2273. static int ubwcp_remove_cb_buf(struct platform_device *pdev)
  2274. {
  2275. struct ubwcp_driver *ubwcp;
  2276. FENTRY();
  2277. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2278. if (!ubwcp) {
  2279. ERR("failed to get ubwcp ptr");
  2280. return -EINVAL;
  2281. }
  2282. /* remove buf_cb reference */
  2283. ubwcp->dev_buf_cb = NULL;
  2284. return 0;
  2285. }
  2286. /* descriptor context bank device remove */
  2287. static int ubwcp_remove_cb_desc(struct platform_device *pdev)
  2288. {
  2289. struct ubwcp_driver *ubwcp;
  2290. FENTRY();
  2291. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2292. if (!ubwcp) {
  2293. ERR("failed to get ubwcp ptr");
  2294. return -EINVAL;
  2295. }
  2296. if (!ubwcp->dev_desc_cb) {
  2297. ERR("ubwcp->dev_desc_cb == NULL");
  2298. return -1;
  2299. }
  2300. ubwcp_power(ubwcp, true);
  2301. ubwcp_hw_set_buf_desc(ubwcp->base, 0x0, 0x0);
  2302. ubwcp_power(ubwcp, false);
  2303. dma_free_coherent(ubwcp->dev_desc_cb,
  2304. ubwcp->buffer_desc_size,
  2305. ubwcp->buffer_desc_base,
  2306. ubwcp->buffer_desc_dma_handle);
  2307. ubwcp->buffer_desc_base = NULL;
  2308. ubwcp->buffer_desc_dma_handle = 0;
  2309. return 0;
  2310. }
  2311. /* ubwcp device remove */
  2312. static int qcom_ubwcp_remove(struct platform_device *pdev)
  2313. {
  2314. size_t avail;
  2315. size_t psize;
  2316. struct ubwcp_driver *ubwcp;
  2317. FENTRY();
  2318. /* get pdev->dev->driver_data = ubwcp */
  2319. ubwcp = platform_get_drvdata(pdev);
  2320. if (!ubwcp) {
  2321. ERR("ubwcp == NULL");
  2322. return -1;
  2323. }
  2324. ubwcp_power(ubwcp, true);
  2325. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2326. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2327. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2328. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2329. ubwcp_power(ubwcp, false);
  2330. /* before destroying, make sure pool is empty. otherwise pool_destroy() panics.
  2331. * TBD: remove this check for production code and let it panic
  2332. */
  2333. avail = gen_pool_avail(ubwcp->ula_pool);
  2334. psize = gen_pool_size(ubwcp->ula_pool);
  2335. if (psize != avail) {
  2336. ERR("gen_pool is not empty! avail: %zx size: %zx", avail, psize);
  2337. ERR("skipping pool destroy....cause it will PANIC. Fix this!!!!");
  2338. WARN(1, "Fix this!");
  2339. } else {
  2340. gen_pool_destroy(ubwcp->ula_pool);
  2341. }
  2342. ubwcp_debugfs_deinit(ubwcp);
  2343. ubwcp_cdev_deinit(ubwcp);
  2344. return 0;
  2345. }
  2346. /* top level ubwcp device probe function */
  2347. static int ubwcp_probe(struct platform_device *pdev)
  2348. {
  2349. const char *compatible = "";
  2350. FENTRY();
  2351. if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
  2352. return qcom_ubwcp_probe(pdev);
  2353. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
  2354. return ubwcp_probe_cb_desc(pdev);
  2355. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
  2356. return ubwcp_probe_cb_buf(pdev);
  2357. of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
  2358. ERR("unknown device: %s", compatible);
  2359. WARN_ON(1);
  2360. return -EINVAL;
  2361. }
  2362. /* top level ubwcp device remove function */
  2363. static int ubwcp_remove(struct platform_device *pdev)
  2364. {
  2365. const char *compatible = "";
  2366. FENTRY();
  2367. /* TBD: what if buffers are still allocated? locked? etc.
  2368. * also should turn off power?
  2369. */
  2370. if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
  2371. return qcom_ubwcp_remove(pdev);
  2372. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
  2373. return ubwcp_remove_cb_desc(pdev);
  2374. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
  2375. return ubwcp_remove_cb_buf(pdev);
  2376. of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
  2377. ERR("unknown device: %s", compatible);
  2378. WARN_ON(1);
  2379. return -EINVAL;
  2380. }
  2381. static const struct of_device_id ubwcp_dt_match[] = {
  2382. {.compatible = "qcom,ubwcp"},
  2383. {.compatible = "qcom,ubwcp-context-bank-desc"},
  2384. {.compatible = "qcom,ubwcp-context-bank-buf"},
  2385. {}
  2386. };
  2387. struct platform_driver ubwcp_platform_driver = {
  2388. .probe = ubwcp_probe,
  2389. .remove = ubwcp_remove,
  2390. .driver = {
  2391. .name = "qcom,ubwcp",
  2392. .of_match_table = ubwcp_dt_match,
  2393. },
  2394. };
  2395. int ubwcp_init(void)
  2396. {
  2397. int ret = 0;
  2398. DBG("+++++++++++");
  2399. ret = platform_driver_register(&ubwcp_platform_driver);
  2400. if (ret)
  2401. ERR("platform_driver_register() failed: %d", ret);
  2402. return ret;
  2403. }
  2404. void ubwcp_exit(void)
  2405. {
  2406. platform_driver_unregister(&ubwcp_platform_driver);
  2407. DBG("-----------");
  2408. }
  2409. module_init(ubwcp_init);
  2410. module_exit(ubwcp_exit);
  2411. MODULE_LICENSE("GPL");