ubwcp_main.c 78 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/kernel.h>
  7. #include <linux/dma-buf.h>
  8. #include <linux/slab.h>
  9. #include <linux/cdev.h>
  10. #include <linux/hashtable.h>
  11. #include <linux/scatterlist.h>
  12. #include <linux/types.h>
  13. #include <linux/platform_device.h>
  14. #include <linux/of.h>
  15. #include <linux/of_platform.h>
  16. #include <linux/of_address.h>
  17. #include <linux/genalloc.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/regulator/consumer.h>
  20. #include <linux/numa.h>
  21. #include <linux/memory_hotplug.h>
  22. #include <asm/page.h>
  23. #include <linux/delay.h>
  24. #include <linux/ubwcp_dma_heap.h>
  25. #include <linux/debugfs.h>
  26. #include <linux/clk.h>
  27. #include <linux/iommu.h>
  28. #include <linux/set_memory.h>
  29. MODULE_IMPORT_NS(DMA_BUF);
  30. #include "include/kernel/ubwcp.h"
  31. #include "ubwcp_hw.h"
  32. #include "include/uapi/ubwcp_ioctl.h"
  33. #define CREATE_TRACE_POINTS
  34. #include "ubwcp_trace.h"
  35. #define UBWCP_NUM_DEVICES 1
  36. #define UBWCP_DEVICE_NAME "ubwcp"
  37. #define UBWCP_BUFFER_DESC_OFFSET 64
  38. #define UBWCP_BUFFER_DESC_COUNT 256
  39. #define CACHE_ADDR(x) ((x) >> 6)
  40. #define PAGE_ADDR(x) ((x) >> 12)
  41. #define UBWCP_ALIGN(_x, _y) ((((_x) + (_y) - 1)/(_y))*(_y))
  42. //#define DBG(fmt, args...)
  43. //#define DBG_BUF_ATTR(fmt, args...)
  44. #define DBG_BUF_ATTR(fmt, args...) do { if (ubwcp_debug_trace_enable) \
  45. pr_err("ubwcp: %s(): " fmt "\n", __func__, ##args); \
  46. } while (0)
  47. #define DBG(fmt, args...) do { if (ubwcp_debug_trace_enable) \
  48. pr_err("ubwcp: %s(): " fmt "\n", __func__, ##args); \
  49. } while (0)
  50. #define ERR(fmt, args...) pr_err("ubwcp: %s(): ~~~ERROR~~~: " fmt "\n", __func__, ##args)
  51. #define FENTRY() DBG("")
  52. #define META_DATA_PITCH_ALIGN 64
  53. #define META_DATA_HEIGHT_ALIGN 16
  54. #define META_DATA_SIZE_ALIGN 4096
  55. #define PIXEL_DATA_SIZE_ALIGN 4096
  56. #define UBWCP_SYNC_GRANULE 0x4000000L /* 64 MB */
  57. struct ubwcp_desc {
  58. int idx;
  59. void *ptr;
  60. };
  61. /* TBD: confirm size of width/height */
  62. struct ubwcp_dimension {
  63. u16 width;
  64. u16 height;
  65. };
  66. struct ubwcp_plane_info {
  67. u16 pixel_bytes;
  68. u16 per_pixel;
  69. struct ubwcp_dimension tilesize_p; /* pixels */
  70. struct ubwcp_dimension macrotilesize_p; /* pixels */
  71. };
  72. struct ubwcp_image_format_info {
  73. u16 planes;
  74. struct ubwcp_plane_info p_info[2];
  75. };
  76. enum ubwcp_std_image_format {
  77. RGBA = 0,
  78. NV12 = 1,
  79. NV124R = 2,
  80. P010 = 3,
  81. TP10 = 4,
  82. P016 = 5,
  83. INFO_FORMAT_LIST_SIZE,
  84. STD_IMAGE_FORMAT_INVALID = 0xFF
  85. };
  86. struct ubwcp_driver {
  87. /* cdev related */
  88. dev_t devt;
  89. struct class *dev_class; //sysfs dev class
  90. struct device *dev_sys; //sysfs dev
  91. struct cdev cdev; //char dev
  92. /* debugfs */
  93. struct dentry *debugfs_root;
  94. /* ubwcp devices */
  95. struct device *dev; //ubwcp device
  96. struct device *dev_desc_cb; //smmu dev for descriptors
  97. struct device *dev_buf_cb; //smmu dev for ubwcp buffers
  98. void __iomem *base; //ubwcp base address
  99. struct regulator *vdd;
  100. struct clk **clocks;
  101. int num_clocks;
  102. /* interrupts */
  103. int irq_range_ck_rd;
  104. int irq_range_ck_wr;
  105. int irq_encode;
  106. int irq_decode;
  107. /* ula address pool */
  108. u64 ula_pool_base;
  109. u64 ula_pool_size;
  110. struct gen_pool *ula_pool;
  111. configure_mmap mmap_config_fptr;
  112. /* HW version */
  113. u32 hw_ver_major;
  114. u32 hw_ver_minor;
  115. /* keep track of all potential buffers.
  116. * hash table index'ed using dma_buf ptr.
  117. * 2**13 = 8192 hash values
  118. */
  119. DECLARE_HASHTABLE(buf_table, 13);
  120. /* buffer descriptor */
  121. void *buffer_desc_base; /* CPU address */
  122. dma_addr_t buffer_desc_dma_handle; /* dma address */
  123. size_t buffer_desc_size;
  124. struct ubwcp_desc desc_list[UBWCP_BUFFER_DESC_COUNT];
  125. struct ubwcp_image_format_info format_info[INFO_FORMAT_LIST_SIZE];
  126. atomic_t num_non_lin_buffers;
  127. bool mem_online;
  128. struct mutex desc_lock; /* allocate/free descriptors */
  129. spinlock_t buf_table_lock; /* add/remove dma_buf into list of managed bufffers */
  130. struct mutex mem_hotplug_lock; /* memory hotplug lock */
  131. struct mutex ula_lock; /* allocate/free ula */
  132. struct mutex ubwcp_flush_lock; /* ubwcp flush */
  133. struct mutex hw_range_ck_lock; /* range ck */
  134. struct list_head err_handler_list; /* error handler list */
  135. spinlock_t err_handler_list_lock; /* err_handler_list lock */
  136. };
  137. struct ubwcp_buf {
  138. struct hlist_node hnode;
  139. struct ubwcp_driver *ubwcp;
  140. struct ubwcp_buffer_attrs buf_attr;
  141. bool perm;
  142. struct ubwcp_desc *desc;
  143. bool buf_attr_set;
  144. bool locked;
  145. enum dma_data_direction lock_dir;
  146. int lock_count;
  147. /* dma_buf info */
  148. struct dma_buf *dma_buf;
  149. struct dma_buf_attachment *attachment;
  150. struct sg_table *sgt;
  151. /* ula info */
  152. phys_addr_t ula_pa;
  153. size_t ula_size;
  154. /* meta metadata */
  155. struct ubwcp_hw_meta_metadata mmdata;
  156. struct mutex lock;
  157. };
  158. static struct ubwcp_driver *me;
  159. static int error_print_count;
  160. u32 ubwcp_debug_trace_enable;
  161. static struct ubwcp_driver *ubwcp_get_driver(void)
  162. {
  163. if (!me)
  164. WARN(1, "ubwcp: driver ptr requested but driver not initialized");
  165. return me;
  166. }
  167. static void image_format_init(struct ubwcp_driver *ubwcp)
  168. { /* planes, bytes/p, Tp , MTp */
  169. ubwcp->format_info[RGBA] = (struct ubwcp_image_format_info)
  170. {1, {{4, 1, {16, 4}, {64, 16}}}};
  171. ubwcp->format_info[NV12] = (struct ubwcp_image_format_info)
  172. {2, {{1, 1, {32, 8}, {128, 32}},
  173. {2, 1, {16, 8}, { 64, 32}}}};
  174. ubwcp->format_info[NV124R] = (struct ubwcp_image_format_info)
  175. {2, {{1, 1, {64, 4}, {256, 16}},
  176. {2, 1, {32, 4}, {128, 16}}}};
  177. ubwcp->format_info[P010] = (struct ubwcp_image_format_info)
  178. {2, {{2, 1, {32, 4}, {128, 16}},
  179. {4, 1, {16, 4}, { 64, 16}}}};
  180. ubwcp->format_info[TP10] = (struct ubwcp_image_format_info)
  181. {2, {{4, 3, {48, 4}, {192, 16}},
  182. {8, 3, {24, 4}, { 96, 16}}}};
  183. ubwcp->format_info[P016] = (struct ubwcp_image_format_info)
  184. {2, {{2, 1, {32, 4}, {128, 16}},
  185. {4, 1, {16, 4}, { 64, 16}}}};
  186. }
  187. static void ubwcp_buf_desc_list_init(struct ubwcp_driver *ubwcp)
  188. {
  189. int idx;
  190. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  191. for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
  192. desc_list[idx].idx = -1;
  193. desc_list[idx].ptr = NULL;
  194. }
  195. }
  196. static int ubwcp_init_clocks(struct ubwcp_driver *ubwcp, struct device *dev)
  197. {
  198. const char *cname;
  199. struct property *prop;
  200. int i;
  201. ubwcp->num_clocks =
  202. of_property_count_strings(dev->of_node, "clock-names");
  203. if (ubwcp->num_clocks < 1) {
  204. ubwcp->num_clocks = 0;
  205. return 0;
  206. }
  207. ubwcp->clocks = devm_kzalloc(dev,
  208. sizeof(*ubwcp->clocks) * ubwcp->num_clocks, GFP_KERNEL);
  209. if (!ubwcp->clocks)
  210. return -ENOMEM;
  211. i = 0;
  212. of_property_for_each_string(dev->of_node, "clock-names",
  213. prop, cname) {
  214. struct clk *c = devm_clk_get(dev, cname);
  215. if (IS_ERR(c)) {
  216. ERR("Couldn't get clock: %s\n", cname);
  217. return PTR_ERR(c);
  218. }
  219. ubwcp->clocks[i] = c;
  220. ++i;
  221. }
  222. return 0;
  223. }
  224. static int ubwcp_enable_clocks(struct ubwcp_driver *ubwcp)
  225. {
  226. int i, ret = 0;
  227. for (i = 0; i < ubwcp->num_clocks; ++i) {
  228. ret = clk_prepare_enable(ubwcp->clocks[i]);
  229. if (ret) {
  230. ERR("Couldn't enable clock #%d\n", i);
  231. while (i--)
  232. clk_disable_unprepare(ubwcp->clocks[i]);
  233. break;
  234. }
  235. }
  236. return ret;
  237. }
  238. static void ubwcp_disable_clocks(struct ubwcp_driver *ubwcp)
  239. {
  240. int i;
  241. for (i = ubwcp->num_clocks; i; --i)
  242. clk_disable_unprepare(ubwcp->clocks[i - 1]);
  243. }
  244. /* UBWCP Power control */
  245. static int ubwcp_power(struct ubwcp_driver *ubwcp, bool enable)
  246. {
  247. int ret = 0;
  248. if (!ubwcp) {
  249. ERR("ubwcp ptr is NULL");
  250. return -1;
  251. }
  252. if (!ubwcp->vdd) {
  253. ERR("vdd is NULL");
  254. return -1;
  255. }
  256. if (enable) {
  257. ret = regulator_enable(ubwcp->vdd);
  258. if (ret < 0) {
  259. ERR("regulator_enable failed: %d", ret);
  260. ret = -1;
  261. } else {
  262. DBG("regulator_enable() success");
  263. }
  264. if (!ret) {
  265. ret = ubwcp_enable_clocks(ubwcp);
  266. if (ret) {
  267. ERR("enable clocks failed: %d", ret);
  268. regulator_disable(ubwcp->vdd);
  269. } else {
  270. DBG("enable clocks success");
  271. }
  272. }
  273. } else {
  274. ret = regulator_disable(ubwcp->vdd);
  275. if (ret < 0) {
  276. ERR("regulator_disable failed: %d", ret);
  277. ret = -1;
  278. } else {
  279. DBG("regulator_disable() success");
  280. }
  281. if (!ret) {
  282. ubwcp_disable_clocks(ubwcp);
  283. DBG("disable clocks success");
  284. }
  285. }
  286. return ret;
  287. }
  288. static int ubwcp_flush(struct ubwcp_driver *ubwcp)
  289. {
  290. int ret = 0;
  291. mutex_lock(&ubwcp->ubwcp_flush_lock);
  292. ret = ubwcp_hw_flush(ubwcp->base);
  293. mutex_unlock(&ubwcp->ubwcp_flush_lock);
  294. if (ret != 0)
  295. WARN(1, "ubwcp_hw_flush() failed!");
  296. return ret;
  297. }
  298. /* get dma_buf ptr for the given dma_buf fd */
  299. struct dma_buf *ubwcp_dma_buf_fd_to_dma_buf(int dma_buf_fd)
  300. {
  301. struct dma_buf *dmabuf;
  302. /* TBD: dma_buf_get() results in taking ref to buf and it won't ever get
  303. * free'ed until ref count goes to 0. So we must reduce the ref count
  304. * immediately after we find our corresponding ubwcp_buf.
  305. */
  306. dmabuf = dma_buf_get(dma_buf_fd);
  307. if (IS_ERR(dmabuf)) {
  308. ERR("dmabuf ptr not found for dma_buf_fd = %d", dma_buf_fd);
  309. return NULL;
  310. }
  311. dma_buf_put(dmabuf);
  312. return dmabuf;
  313. }
  314. EXPORT_SYMBOL(ubwcp_dma_buf_fd_to_dma_buf);
  315. /* get ubwcp_buf corresponding to the given dma_buf */
  316. static struct ubwcp_buf *dma_buf_to_ubwcp_buf(struct dma_buf *dmabuf)
  317. {
  318. struct ubwcp_buf *buf = NULL;
  319. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  320. unsigned long flags;
  321. if (!dmabuf || !ubwcp)
  322. return NULL;
  323. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  324. /* look up ubwcp_buf corresponding to this dma_buf */
  325. hash_for_each_possible(ubwcp->buf_table, buf, hnode, (u64)dmabuf) {
  326. if (buf->dma_buf == dmabuf)
  327. break;
  328. }
  329. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  330. return buf;
  331. }
  332. /* return ubwcp hardware version */
  333. int ubwcp_get_hw_version(struct ubwcp_ioctl_hw_version *ver)
  334. {
  335. struct ubwcp_driver *ubwcp;
  336. FENTRY();
  337. if (!ver) {
  338. ERR("invalid version ptr");
  339. return -EINVAL;
  340. }
  341. ubwcp = ubwcp_get_driver();
  342. if (!ubwcp)
  343. return -1;
  344. ver->major = ubwcp->hw_ver_major;
  345. ver->minor = ubwcp->hw_ver_minor;
  346. return 0;
  347. }
  348. EXPORT_SYMBOL(ubwcp_get_hw_version);
  349. static int add_ula_pa_memory(struct ubwcp_driver *ubwcp)
  350. {
  351. int ret;
  352. int nid;
  353. nid = memory_add_physaddr_to_nid(ubwcp->ula_pool_base);
  354. DBG("calling add_memory()...");
  355. trace_ubwcp_add_memory_start(ubwcp->ula_pool_size);
  356. ret = add_memory(nid, ubwcp->ula_pool_base, ubwcp->ula_pool_size, MHP_NONE);
  357. trace_ubwcp_add_memory_end(ubwcp->ula_pool_size);
  358. if (ret) {
  359. ERR("add_memory() failed st:0x%lx sz:0x%lx err: %d",
  360. ubwcp->ula_pool_base,
  361. ubwcp->ula_pool_size,
  362. ret);
  363. /* Fix to put driver in invalid state */
  364. } else {
  365. DBG("add_memory() ula_pool_base:0x%llx, size:0x%zx, kernel addr:0x%p",
  366. ubwcp->ula_pool_base,
  367. ubwcp->ula_pool_size,
  368. page_to_virt(pfn_to_page(PFN_DOWN(ubwcp->ula_pool_base))));
  369. }
  370. return ret;
  371. }
  372. static int inc_num_non_lin_buffers(struct ubwcp_driver *ubwcp)
  373. {
  374. int ret = 0;
  375. atomic_inc(&ubwcp->num_non_lin_buffers);
  376. mutex_lock(&ubwcp->mem_hotplug_lock);
  377. if (!ubwcp->mem_online) {
  378. if (atomic_read(&ubwcp->num_non_lin_buffers) == 0) {
  379. ret = -EINVAL;
  380. ERR("Bad state: num_non_lin_buffers should not be 0");
  381. /* Fix to put driver in invalid state */
  382. goto err_power_on;
  383. }
  384. ret = ubwcp_power(ubwcp, true);
  385. if (ret)
  386. goto err_power_on;
  387. ret = add_ula_pa_memory(ubwcp);
  388. if (ret)
  389. goto err_add_memory;
  390. ubwcp->mem_online = true;
  391. }
  392. mutex_unlock(&ubwcp->mem_hotplug_lock);
  393. return 0;
  394. err_add_memory:
  395. ubwcp_power(ubwcp, false);
  396. err_power_on:
  397. atomic_dec(&ubwcp->num_non_lin_buffers);
  398. mutex_unlock(&ubwcp->mem_hotplug_lock);
  399. return ret;
  400. }
  401. static int dec_num_non_lin_buffers(struct ubwcp_driver *ubwcp)
  402. {
  403. int ret = 0;
  404. atomic_dec(&ubwcp->num_non_lin_buffers);
  405. mutex_lock(&ubwcp->mem_hotplug_lock);
  406. /* If this is the last buffer being freed, power off ubwcp */
  407. if (atomic_read(&ubwcp->num_non_lin_buffers) == 0) {
  408. unsigned long sync_remain = 0;
  409. unsigned long sync_offset = 0;
  410. unsigned long sync_size = 0;
  411. unsigned long sync_granule = UBWCP_SYNC_GRANULE;
  412. DBG("last buffer: ~~~~~~~~~~~");
  413. if (!ubwcp->mem_online) {
  414. ret = -EINVAL;
  415. ERR("Bad state: mem_online should not be false");
  416. /* Fix to put driver in invalid state */
  417. goto err_remove_mem;
  418. }
  419. DBG("set_direct_map_range_uncached() for ULA PA pool st:0x%lx num pages:%lu",
  420. ubwcp->ula_pool_base, ubwcp->ula_pool_size >> PAGE_SHIFT);
  421. trace_ubwcp_set_direct_map_range_uncached_start(ubwcp->ula_pool_size);
  422. ret = set_direct_map_range_uncached((unsigned long)phys_to_virt(
  423. ubwcp->ula_pool_base), ubwcp->ula_pool_size >> PAGE_SHIFT);
  424. trace_ubwcp_set_direct_map_range_uncached_end(ubwcp->ula_pool_size);
  425. if (ret) {
  426. ERR("set_direct_map_range_uncached failed st:0x%lx num pages:%lu err: %d",
  427. ubwcp->ula_pool_base,
  428. ubwcp->ula_pool_size >> PAGE_SHIFT, ret);
  429. goto err_remove_mem;
  430. } else {
  431. DBG("DONE: calling set_direct_map_range_uncached() for ULA PA pool");
  432. }
  433. DBG("Calling dma_sync_single_for_cpu() for ULA PA pool");
  434. trace_ubwcp_offline_sync_start(ubwcp->ula_pool_size);
  435. sync_remain = ubwcp->ula_pool_size;
  436. sync_offset = 0;
  437. while (sync_remain > 0) {
  438. if (atomic_read(&ubwcp->num_non_lin_buffers) > 0) {
  439. trace_ubwcp_offline_sync_end(ubwcp->ula_pool_size);
  440. DBG("Cancel memory offlining");
  441. DBG("Calling offline_and_remove_memory() for ULA PA pool");
  442. trace_ubwcp_offline_and_remove_memory_start(ubwcp->ula_pool_size);
  443. ret = offline_and_remove_memory(ubwcp->ula_pool_base,
  444. ubwcp->ula_pool_size);
  445. trace_ubwcp_offline_and_remove_memory_end(ubwcp->ula_pool_size);
  446. if (ret) {
  447. ERR("remove memory failed st:0x%lx sz:0x%lx err: %d",
  448. ubwcp->ula_pool_base,
  449. ubwcp->ula_pool_size, ret);
  450. goto err_remove_mem;
  451. } else {
  452. DBG("DONE: calling remove memory for ULA PA pool");
  453. }
  454. ret = add_ula_pa_memory(ubwcp);
  455. if (ret) {
  456. ERR("Bad state: failed to add back memory");
  457. /* Fix to put driver in invalid state */
  458. ubwcp->mem_online = false;
  459. }
  460. mutex_unlock(&ubwcp->mem_hotplug_lock);
  461. return ret;
  462. }
  463. if (sync_granule > sync_remain) {
  464. sync_size = sync_remain;
  465. sync_remain = 0;
  466. } else {
  467. sync_size = sync_granule;
  468. sync_remain -= sync_granule;
  469. }
  470. DBG("Partial sync offset:0x%lx size:0x%lx", sync_offset, sync_size);
  471. trace_ubwcp_dma_sync_single_for_cpu_start(sync_size);
  472. dma_sync_single_for_cpu(ubwcp->dev, ubwcp->ula_pool_base + sync_offset,
  473. sync_size, DMA_BIDIRECTIONAL);
  474. trace_ubwcp_dma_sync_single_for_cpu_end(sync_size);
  475. sync_offset += sync_size;
  476. }
  477. trace_ubwcp_offline_sync_end(ubwcp->ula_pool_size);
  478. DBG("Calling offline_and_remove_memory() for ULA PA pool");
  479. trace_ubwcp_offline_and_remove_memory_start(ubwcp->ula_pool_size);
  480. ret = offline_and_remove_memory(ubwcp->ula_pool_base, ubwcp->ula_pool_size);
  481. trace_ubwcp_offline_and_remove_memory_end(ubwcp->ula_pool_size);
  482. if (ret) {
  483. ERR("offline_and_remove_memory failed st:0x%lx sz:0x%lx err: %d",
  484. ubwcp->ula_pool_base,
  485. ubwcp->ula_pool_size, ret);
  486. /* Fix to put driver in invalid state */
  487. goto err_remove_mem;
  488. } else {
  489. DBG("DONE: calling offline_and_remove_memory() for ULA PA pool");
  490. }
  491. DBG("Calling power OFF ...");
  492. ubwcp_power(ubwcp, false);
  493. ubwcp->mem_online = false;
  494. }
  495. mutex_unlock(&ubwcp->mem_hotplug_lock);
  496. return 0;
  497. err_remove_mem:
  498. atomic_inc(&ubwcp->num_non_lin_buffers);
  499. mutex_unlock(&ubwcp->mem_hotplug_lock);
  500. DBG("returning error: %d", ret);
  501. return ret;
  502. }
  503. /**
  504. *
  505. * Initialize ubwcp buffer for the given dma_buf. This
  506. * initializes ubwcp internal data structures and possibly hw to
  507. * use ubwcp for this buffer.
  508. *
  509. * @param dmabuf : ptr to the buffer to be configured for ubwcp
  510. *
  511. * @return int : 0 on success, otherwise error code
  512. */
  513. static int ubwcp_init_buffer(struct dma_buf *dmabuf)
  514. {
  515. struct ubwcp_buf *buf;
  516. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  517. unsigned long flags;
  518. FENTRY();
  519. trace_ubwcp_init_buffer_start(dmabuf);
  520. if (!ubwcp) {
  521. trace_ubwcp_init_buffer_end(dmabuf);
  522. return -1;
  523. }
  524. if (!dmabuf) {
  525. ERR("NULL dmabuf input ptr");
  526. trace_ubwcp_init_buffer_end(dmabuf);
  527. return -EINVAL;
  528. }
  529. if (dma_buf_to_ubwcp_buf(dmabuf)) {
  530. ERR("dma_buf already initialized for ubwcp");
  531. trace_ubwcp_init_buffer_end(dmabuf);
  532. return -EEXIST;
  533. }
  534. buf = kzalloc(sizeof(*buf), GFP_KERNEL);
  535. if (!buf) {
  536. ERR("failed to alloc for new ubwcp_buf");
  537. trace_ubwcp_init_buffer_end(dmabuf);
  538. return -ENOMEM;
  539. }
  540. mutex_init(&buf->lock);
  541. buf->dma_buf = dmabuf;
  542. buf->ubwcp = ubwcp;
  543. buf->buf_attr.image_format = UBWCP_LINEAR;
  544. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  545. hash_add(ubwcp->buf_table, &buf->hnode, (u64)buf->dma_buf);
  546. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  547. trace_ubwcp_init_buffer_end(dmabuf);
  548. return 0;
  549. }
  550. static void dump_attributes(struct ubwcp_buffer_attrs *attr)
  551. {
  552. DBG_BUF_ATTR("");
  553. DBG_BUF_ATTR("image_format: %d", attr->image_format);
  554. DBG_BUF_ATTR("major_ubwc_ver: %d", attr->major_ubwc_ver);
  555. DBG_BUF_ATTR("minor_ubwc_ver: %d", attr->minor_ubwc_ver);
  556. DBG_BUF_ATTR("compression_type: %d", attr->compression_type);
  557. DBG_BUF_ATTR("lossy_params: %llu", attr->lossy_params);
  558. DBG_BUF_ATTR("width: %d", attr->width);
  559. DBG_BUF_ATTR("height: %d", attr->height);
  560. DBG_BUF_ATTR("stride: %d", attr->stride);
  561. DBG_BUF_ATTR("scanlines: %d", attr->scanlines);
  562. DBG_BUF_ATTR("planar_padding: %d", attr->planar_padding);
  563. DBG_BUF_ATTR("subsample: %d", attr->subsample);
  564. DBG_BUF_ATTR("sub_system_target: %d", attr->sub_system_target);
  565. DBG_BUF_ATTR("y_offset: %d", attr->y_offset);
  566. DBG_BUF_ATTR("batch_size: %d", attr->batch_size);
  567. DBG_BUF_ATTR("");
  568. }
  569. /* validate buffer attributes */
  570. static bool ubwcp_buf_attrs_valid(struct ubwcp_buffer_attrs *attr)
  571. {
  572. bool valid_format;
  573. switch (attr->image_format) {
  574. case UBWCP_LINEAR:
  575. case UBWCP_RGBA8888:
  576. case UBWCP_NV12:
  577. case UBWCP_NV12_Y:
  578. case UBWCP_NV12_UV:
  579. case UBWCP_NV124R:
  580. case UBWCP_NV124R_Y:
  581. case UBWCP_NV124R_UV:
  582. case UBWCP_TP10:
  583. case UBWCP_TP10_Y:
  584. case UBWCP_TP10_UV:
  585. case UBWCP_P010:
  586. case UBWCP_P010_Y:
  587. case UBWCP_P010_UV:
  588. case UBWCP_P016:
  589. case UBWCP_P016_Y:
  590. case UBWCP_P016_UV:
  591. valid_format = true;
  592. break;
  593. default:
  594. valid_format = false;
  595. }
  596. if (!valid_format) {
  597. ERR("invalid image format: %d", attr->image_format);
  598. goto err;
  599. }
  600. if (attr->major_ubwc_ver || attr->minor_ubwc_ver) {
  601. ERR("major/minor ubwc ver must be 0. major: %d minor: %d",
  602. attr->major_ubwc_ver, attr->minor_ubwc_ver);
  603. goto err;
  604. }
  605. if (attr->compression_type != UBWCP_COMPRESSION_LOSSLESS) {
  606. ERR("compression_type is not valid: %d",
  607. attr->compression_type);
  608. goto err;
  609. }
  610. if (attr->lossy_params != 0) {
  611. ERR("lossy_params is not valid: %d", attr->lossy_params);
  612. goto err;
  613. }
  614. //TBD: some upper limit for width?
  615. if (attr->width > 10*1024) {
  616. ERR("width is invalid (above upper limit): %d", attr->width);
  617. goto err;
  618. }
  619. //TBD: some upper limit for height?
  620. if (attr->height > 10*1024) {
  621. ERR("height is invalid (above upper limit): %d", attr->height);
  622. goto err;
  623. }
  624. /* TBD: what's the upper limit for stride? 8K is likely too high. */
  625. if (!IS_ALIGNED(attr->stride, 64) ||
  626. (attr->stride < attr->width) ||
  627. (attr->stride > 4*8192)) {
  628. ERR("stride is not valid (aligned to 64 and <= 8192): %d",
  629. attr->stride);
  630. goto err;
  631. }
  632. /* TBD: currently assume height + 10. Replace 10 with right num from camera. */
  633. if ((attr->scanlines < attr->height) ||
  634. (attr->scanlines > attr->height + 10)) {
  635. ERR("scanlines is not valid - height: %d scanlines: %d",
  636. attr->height, attr->scanlines);
  637. goto err;
  638. }
  639. if (attr->planar_padding > 4096) {
  640. ERR("planar_padding is not valid. (<= 4096): %d",
  641. attr->planar_padding);
  642. goto err;
  643. }
  644. if (attr->subsample != UBWCP_SUBSAMPLE_4_2_0) {
  645. ERR("subsample is not valid: %d", attr->subsample);
  646. goto err;
  647. }
  648. if (attr->sub_system_target & ~UBWCP_SUBSYSTEM_TARGET_CPU) {
  649. ERR("sub_system_target other that CPU is not supported: %d",
  650. attr->sub_system_target);
  651. goto err;
  652. }
  653. if (!(attr->sub_system_target & UBWCP_SUBSYSTEM_TARGET_CPU)) {
  654. ERR("sub_system_target is not set to CPU: %d",
  655. attr->sub_system_target);
  656. goto err;
  657. }
  658. if (attr->y_offset != 0) {
  659. ERR("y_offset is not valid: %d", attr->y_offset);
  660. goto err;
  661. }
  662. if (attr->batch_size != 1) {
  663. ERR("batch_size is not valid: %d", attr->batch_size);
  664. goto err;
  665. }
  666. dump_attributes(attr);
  667. return true;
  668. err:
  669. dump_attributes(attr);
  670. return false;
  671. }
  672. /* return true if image format has only Y plane*/
  673. bool ubwcp_image_y_only(u16 format)
  674. {
  675. switch (format) {
  676. case UBWCP_NV12_Y:
  677. case UBWCP_NV124R_Y:
  678. case UBWCP_TP10_Y:
  679. case UBWCP_P010_Y:
  680. case UBWCP_P016_Y:
  681. return true;
  682. default:
  683. return false;
  684. }
  685. }
  686. /* return true if image format has only UV plane*/
  687. bool ubwcp_image_uv_only(u16 format)
  688. {
  689. switch (format) {
  690. case UBWCP_NV12_UV:
  691. case UBWCP_NV124R_UV:
  692. case UBWCP_TP10_UV:
  693. case UBWCP_P010_UV:
  694. case UBWCP_P016_UV:
  695. return true;
  696. default:
  697. return false;
  698. }
  699. }
  700. /* calculate and return metadata buffer size for a given plane
  701. * and buffer attributes
  702. * NOTE: in this function, we will only pass in NV12 format.
  703. * NOT NV12_Y or NV12_UV etc.
  704. * the Y or UV information is in the "plane"
  705. * "format" here purely means "encoding format" and no information
  706. * if some plane data is missing.
  707. */
  708. static size_t metadata_buf_sz(struct ubwcp_driver *ubwcp,
  709. enum ubwcp_std_image_format format,
  710. u32 width, u32 height, u8 plane)
  711. {
  712. size_t size;
  713. u64 pitch;
  714. u64 lines;
  715. u64 tile_width;
  716. u32 tile_height;
  717. struct ubwcp_image_format_info f_info;
  718. struct ubwcp_plane_info p_info;
  719. f_info = ubwcp->format_info[format];
  720. DBG_BUF_ATTR("");
  721. DBG_BUF_ATTR("");
  722. DBG_BUF_ATTR("Calculating metadata buffer size: format = %d, plane = %d", format, plane);
  723. if (plane >= f_info.planes) {
  724. ERR("Format does not have requested plane info: format: %d, plane: %d",
  725. format, plane);
  726. WARN(1, "Fix this!!!!!");
  727. return 0;
  728. }
  729. p_info = f_info.p_info[plane];
  730. /* UV plane */
  731. if (plane == 1) {
  732. width = width/2;
  733. height = height/2;
  734. }
  735. tile_width = p_info.tilesize_p.width;
  736. tile_height = p_info.tilesize_p.height;
  737. /* pitch: # of tiles in a row
  738. * lines: # of tile rows
  739. */
  740. pitch = UBWCP_ALIGN((width + tile_width - 1)/tile_width, META_DATA_PITCH_ALIGN);
  741. lines = UBWCP_ALIGN((height + tile_height - 1)/tile_height, META_DATA_HEIGHT_ALIGN);
  742. DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
  743. DBG_BUF_ATTR("tile params : %d x %d (pixels)", tile_width, tile_height);
  744. DBG_BUF_ATTR("pitch : %d (%d)", pitch, width/tile_width);
  745. DBG_BUF_ATTR("lines : %d (%d)", lines, height);
  746. DBG_BUF_ATTR("size (p*l*bytes) : %d", pitch*lines*1);
  747. /* x1 below is only to clarify that we are multiplying by 1 bytes/tile */
  748. size = UBWCP_ALIGN(pitch*lines*1, META_DATA_SIZE_ALIGN);
  749. DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", size, size);
  750. return size;
  751. }
  752. /* calculate and return size of pixel data buffer for a given plane
  753. * and buffer attributes
  754. */
  755. static size_t pixeldata_buf_sz(struct ubwcp_driver *ubwcp,
  756. u16 format, u32 width,
  757. u32 height, u8 plane)
  758. {
  759. size_t size;
  760. u64 pitch;
  761. u64 lines;
  762. u16 pixel_bytes;
  763. u16 per_pixel;
  764. u64 macro_tile_width_p;
  765. u64 macro_tile_height_p;
  766. struct ubwcp_image_format_info f_info;
  767. struct ubwcp_plane_info p_info;
  768. f_info = ubwcp->format_info[format];
  769. DBG_BUF_ATTR("");
  770. DBG_BUF_ATTR("");
  771. DBG_BUF_ATTR("Calculating Pixeldata buffer size: format = %d, plane = %d", format, plane);
  772. if (plane >= f_info.planes) {
  773. ERR("Format does not have requested plane info: format: %d, plane: %d",
  774. format, plane);
  775. WARN(1, "Fix this!!!!!");
  776. return 0;
  777. }
  778. p_info = f_info.p_info[plane];
  779. pixel_bytes = p_info.pixel_bytes;
  780. per_pixel = p_info.per_pixel;
  781. /* UV plane */
  782. if (plane == 1) {
  783. width = width/2;
  784. height = height/2;
  785. }
  786. macro_tile_width_p = p_info.macrotilesize_p.width;
  787. macro_tile_height_p = p_info.macrotilesize_p.height;
  788. /* align pixel width and height macro tile width and height */
  789. pitch = UBWCP_ALIGN(width, macro_tile_width_p);
  790. lines = UBWCP_ALIGN(height, macro_tile_height_p);
  791. DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
  792. DBG_BUF_ATTR("macro tile params: %d x %d (pixels)", macro_tile_width_p,
  793. macro_tile_height_p);
  794. DBG_BUF_ATTR("bytes_per_pixel : %d/%d", pixel_bytes, per_pixel);
  795. DBG_BUF_ATTR("pitch : %d", pitch);
  796. DBG_BUF_ATTR("lines : %d", lines);
  797. DBG_BUF_ATTR("size (p*l*bytes) : %d", (pitch*lines*pixel_bytes)/per_pixel);
  798. size = UBWCP_ALIGN((pitch*lines*pixel_bytes)/per_pixel, PIXEL_DATA_SIZE_ALIGN);
  799. DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", size, size);
  800. return size;
  801. }
  802. static int get_tile_height(struct ubwcp_driver *ubwcp, enum ubwcp_std_image_format format,
  803. u8 plane)
  804. {
  805. struct ubwcp_image_format_info f_info;
  806. struct ubwcp_plane_info p_info;
  807. f_info = ubwcp->format_info[format];
  808. p_info = f_info.p_info[plane];
  809. return p_info.tilesize_p.height;
  810. }
  811. /*
  812. * plane: must be 0 or 1 (1st plane == 0, 2nd plane == 1)
  813. */
  814. static size_t ubwcp_ula_size(struct ubwcp_driver *ubwcp, u16 format,
  815. u32 stride_b, u32 scanlines, u8 plane,
  816. bool add_tile_pad)
  817. {
  818. size_t size;
  819. DBG_BUF_ATTR("%s(format = %d, plane = %d)", __func__, format, plane);
  820. /* UV plane */
  821. if (plane == 1)
  822. scanlines = scanlines/2;
  823. if (add_tile_pad) {
  824. int tile_height = get_tile_height(ubwcp, format, plane);
  825. /* Align plane size to plane tile height */
  826. scanlines = ((scanlines + tile_height - 1) / tile_height) * tile_height;
  827. }
  828. size = stride_b*scanlines;
  829. DBG_BUF_ATTR("Size of plane-%u: (%u * %u) = %zu (0x%zx)",
  830. plane, stride_b, scanlines, size, size);
  831. return size;
  832. }
  833. int missing_plane_from_format(u16 ioctl_image_format)
  834. {
  835. int missing_plane;
  836. switch (ioctl_image_format) {
  837. case UBWCP_NV12_Y:
  838. missing_plane = 2;
  839. break;
  840. case UBWCP_NV12_UV:
  841. missing_plane = 1;
  842. break;
  843. case UBWCP_NV124R_Y:
  844. missing_plane = 2;
  845. break;
  846. case UBWCP_NV124R_UV:
  847. missing_plane = 1;
  848. break;
  849. case UBWCP_TP10_Y:
  850. missing_plane = 2;
  851. break;
  852. case UBWCP_TP10_UV:
  853. missing_plane = 1;
  854. break;
  855. case UBWCP_P010_Y:
  856. missing_plane = 2;
  857. break;
  858. case UBWCP_P010_UV:
  859. missing_plane = 1;
  860. break;
  861. case UBWCP_P016_Y:
  862. missing_plane = 2;
  863. break;
  864. case UBWCP_P016_UV:
  865. missing_plane = 1;
  866. break;
  867. default:
  868. missing_plane = 0;
  869. }
  870. return missing_plane;
  871. }
  872. int planes_in_format(enum ubwcp_std_image_format format)
  873. {
  874. if (format == RGBA)
  875. return 1;
  876. else
  877. return 2;
  878. }
  879. enum ubwcp_std_image_format to_std_format(u16 ioctl_image_format)
  880. {
  881. switch (ioctl_image_format) {
  882. case UBWCP_RGBA8888:
  883. return RGBA;
  884. case UBWCP_NV12:
  885. case UBWCP_NV12_Y:
  886. case UBWCP_NV12_UV:
  887. return NV12;
  888. case UBWCP_NV124R:
  889. case UBWCP_NV124R_Y:
  890. case UBWCP_NV124R_UV:
  891. return NV124R;
  892. case UBWCP_TP10:
  893. case UBWCP_TP10_Y:
  894. case UBWCP_TP10_UV:
  895. return TP10;
  896. case UBWCP_P010:
  897. case UBWCP_P010_Y:
  898. case UBWCP_P010_UV:
  899. return P010;
  900. case UBWCP_P016:
  901. case UBWCP_P016_Y:
  902. case UBWCP_P016_UV:
  903. return P016;
  904. default:
  905. WARN(1, "Fix this!!!");
  906. return STD_IMAGE_FORMAT_INVALID;
  907. }
  908. }
  909. unsigned int ubwcp_get_hw_image_format_value(u16 ioctl_image_format)
  910. {
  911. enum ubwcp_std_image_format format;
  912. format = to_std_format(ioctl_image_format);
  913. switch (format) {
  914. case RGBA:
  915. return HW_BUFFER_FORMAT_RGBA;
  916. case NV12:
  917. return HW_BUFFER_FORMAT_NV12;
  918. case NV124R:
  919. return HW_BUFFER_FORMAT_NV124R;
  920. case P010:
  921. return HW_BUFFER_FORMAT_P010;
  922. case TP10:
  923. return HW_BUFFER_FORMAT_TP10;
  924. case P016:
  925. return HW_BUFFER_FORMAT_P016;
  926. default:
  927. WARN(1, "Fix this!!!!!");
  928. return 0;
  929. }
  930. }
  931. static int ubwcp_validate_uv_align(struct ubwcp_driver *ubwcp,
  932. struct ubwcp_buffer_attrs *attr,
  933. size_t ula_y_plane_size,
  934. size_t uv_start_offset)
  935. {
  936. int ret = 0;
  937. size_t ula_y_plane_size_align;
  938. size_t y_tile_align_bytes;
  939. int y_tile_height;
  940. int planes;
  941. /* Only validate UV align if there is both a Y and UV plane */
  942. planes = planes_in_format(to_std_format(attr->image_format));
  943. if (planes != 2)
  944. return 0;
  945. /* Check it is cache line size aligned */
  946. if ((uv_start_offset % 64) != 0) {
  947. ret = -EINVAL;
  948. ERR("uv_start_offset %zu not cache line aligned",
  949. uv_start_offset);
  950. goto err;
  951. }
  952. /*
  953. * Check that UV plane does not overlap with any of the Y plane’s tiles
  954. */
  955. y_tile_height = get_tile_height(ubwcp, to_std_format(attr->image_format), 0);
  956. y_tile_align_bytes = y_tile_height * attr->stride;
  957. ula_y_plane_size_align = ((ula_y_plane_size + y_tile_align_bytes - 1) /
  958. y_tile_align_bytes) * y_tile_align_bytes;
  959. if (uv_start_offset < ula_y_plane_size_align) {
  960. ret = -EINVAL;
  961. ERR("uv offset %zu less than y plane align %zu for y plane size %zu",
  962. uv_start_offset, ula_y_plane_size_align,
  963. ula_y_plane_size);
  964. goto err;
  965. }
  966. return 0;
  967. err:
  968. return ret;
  969. }
  970. /* calculate ULA buffer parms
  971. * TBD: how do we make sure uv_start address (not the offset)
  972. * is aligned per requirement: cache line
  973. */
  974. static int ubwcp_calc_ula_params(struct ubwcp_driver *ubwcp,
  975. struct ubwcp_buffer_attrs *attr,
  976. size_t *ula_size,
  977. size_t *ula_y_plane_size,
  978. size_t *uv_start_offset)
  979. {
  980. size_t size;
  981. enum ubwcp_std_image_format format;
  982. int planes;
  983. int missing_plane;
  984. u32 stride;
  985. u32 scanlines;
  986. u32 planar_padding;
  987. stride = attr->stride;
  988. scanlines = attr->scanlines;
  989. planar_padding = attr->planar_padding;
  990. /* convert ioctl image format to standard image format */
  991. format = to_std_format(attr->image_format);
  992. /* Number of "expected" planes in "the standard defined" image format */
  993. planes = planes_in_format(format);
  994. /* any plane missing?
  995. * valid missing_plane values:
  996. * 0 == no plane missing
  997. * 1 == 1st plane missing
  998. * 2 == 2nd plane missing
  999. */
  1000. missing_plane = missing_plane_from_format(attr->image_format);
  1001. DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
  1002. DBG_BUF_ATTR("planes_in_format : %d", planes);
  1003. DBG_BUF_ATTR("missing_plane : %d", missing_plane);
  1004. DBG_BUF_ATTR("Planar Padding : %d", planar_padding);
  1005. if (planes == 1) {
  1006. /* uv_start beyond ULA range */
  1007. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
  1008. *uv_start_offset = size;
  1009. *ula_y_plane_size = size;
  1010. } else {
  1011. if (!missing_plane) {
  1012. /* size for both planes and padding */
  1013. /* Don't pad out Y plane as client would not expect this padding */
  1014. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, false);
  1015. *ula_y_plane_size = size;
  1016. size += planar_padding;
  1017. *uv_start_offset = size;
  1018. size += ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
  1019. } else {
  1020. if (missing_plane == 2) {
  1021. /* Y-only image, set uv_start beyond ULA range */
  1022. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
  1023. *uv_start_offset = size;
  1024. *ula_y_plane_size = size;
  1025. } else {
  1026. /* first plane data is not there */
  1027. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
  1028. *uv_start_offset = 0; /* uv data is at the beginning */
  1029. *ula_y_plane_size = 0;
  1030. }
  1031. }
  1032. }
  1033. //TBD: cleanup
  1034. *ula_size = size;
  1035. DBG_BUF_ATTR("Before page align: Total ULA_Size: %d (0x%x) (planes + planar padding)",
  1036. *ula_size, *ula_size);
  1037. *ula_size = UBWCP_ALIGN(size, 4096);
  1038. DBG_BUF_ATTR("After page align : Total ULA_Size: %d (0x%x) (planes + planar padding)",
  1039. *ula_size, *ula_size);
  1040. return 0;
  1041. }
  1042. /* calculate UBWCP buffer parms */
  1043. static int ubwcp_calc_ubwcp_buf_params(struct ubwcp_driver *ubwcp,
  1044. struct ubwcp_buffer_attrs *attr,
  1045. size_t *md_p0, size_t *pd_p0,
  1046. size_t *md_p1, size_t *pd_p1,
  1047. size_t *stride_tp10_b)
  1048. {
  1049. int planes;
  1050. int missing_plane;
  1051. enum ubwcp_std_image_format format;
  1052. size_t stride_tp10_p;
  1053. FENTRY();
  1054. /* convert ioctl image format to standard image format */
  1055. format = to_std_format(attr->image_format);
  1056. missing_plane = missing_plane_from_format(attr->image_format);
  1057. planes = planes_in_format(format); //pass in 0 (RGB) should return 1
  1058. DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
  1059. DBG_BUF_ATTR("planes_in_format : %d", planes);
  1060. DBG_BUF_ATTR("missing_plane : %d", missing_plane);
  1061. if (!missing_plane) {
  1062. *md_p0 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  1063. *pd_p0 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  1064. if (planes == 2) {
  1065. *md_p1 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  1066. *pd_p1 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  1067. }
  1068. } else {
  1069. if (missing_plane == 1) {
  1070. *md_p0 = 0;
  1071. *pd_p0 = 0;
  1072. *md_p1 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  1073. *pd_p1 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  1074. } else {
  1075. *md_p0 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  1076. *pd_p0 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  1077. *md_p1 = 0;
  1078. *pd_p1 = 0;
  1079. }
  1080. }
  1081. if (format == TP10) {
  1082. stride_tp10_p = UBWCP_ALIGN(attr->width, 192);
  1083. *stride_tp10_b = (stride_tp10_p/3) + stride_tp10_p;
  1084. } else {
  1085. *stride_tp10_b = 0;
  1086. }
  1087. return 0;
  1088. }
  1089. /* reserve ULA address space of the given size */
  1090. static phys_addr_t ubwcp_ula_alloc(struct ubwcp_driver *ubwcp, size_t size)
  1091. {
  1092. phys_addr_t pa;
  1093. mutex_lock(&ubwcp->ula_lock);
  1094. pa = gen_pool_alloc(ubwcp->ula_pool, size);
  1095. DBG("addr: %p, size: %zx", pa, size);
  1096. mutex_unlock(&ubwcp->ula_lock);
  1097. return pa;
  1098. }
  1099. /* free ULA address space of the given address and size */
  1100. static void ubwcp_ula_free(struct ubwcp_driver *ubwcp, phys_addr_t pa, size_t size)
  1101. {
  1102. mutex_lock(&ubwcp->ula_lock);
  1103. if (!gen_pool_has_addr(ubwcp->ula_pool, pa, size)) {
  1104. ERR("Attempt to free mem not from gen_pool: pa: %p, size: %zx", pa, size);
  1105. goto err;
  1106. }
  1107. DBG("addr: %p, size: %zx", pa, size);
  1108. gen_pool_free(ubwcp->ula_pool, pa, size);
  1109. mutex_unlock(&ubwcp->ula_lock);
  1110. return;
  1111. err:
  1112. mutex_unlock(&ubwcp->ula_lock);
  1113. }
  1114. /* free up or expand current_pa and return the new pa */
  1115. static phys_addr_t ubwcp_ula_realloc(struct ubwcp_driver *ubwcp,
  1116. phys_addr_t pa,
  1117. size_t size,
  1118. size_t new_size)
  1119. {
  1120. if (size == new_size)
  1121. return pa;
  1122. if (pa)
  1123. ubwcp_ula_free(ubwcp, pa, size);
  1124. return ubwcp_ula_alloc(ubwcp, new_size);
  1125. }
  1126. /* unmap dma buf */
  1127. static void ubwcp_dma_unmap(struct ubwcp_buf *buf)
  1128. {
  1129. FENTRY();
  1130. if (buf->dma_buf && buf->attachment) {
  1131. DBG("Calling dma_buf_unmap_attachment()");
  1132. dma_buf_unmap_attachment(buf->attachment, buf->sgt, DMA_BIDIRECTIONAL);
  1133. buf->sgt = NULL;
  1134. dma_buf_detach(buf->dma_buf, buf->attachment);
  1135. buf->attachment = NULL;
  1136. }
  1137. }
  1138. /* dma map ubwcp buffer */
  1139. static int ubwcp_dma_map(struct ubwcp_buf *buf,
  1140. struct device *dev,
  1141. size_t iova_min_size,
  1142. dma_addr_t *iova)
  1143. {
  1144. int ret = 0;
  1145. struct dma_buf *dma_buf = buf->dma_buf;
  1146. struct dma_buf_attachment *attachment;
  1147. struct sg_table *sgt;
  1148. size_t dma_len;
  1149. /* Map buffer to SMMU and get IOVA */
  1150. attachment = dma_buf_attach(dma_buf, dev);
  1151. if (IS_ERR(attachment)) {
  1152. ret = PTR_ERR(attachment);
  1153. ERR("dma_buf_attach() failed: %d", ret);
  1154. goto err;
  1155. }
  1156. dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
  1157. dma_set_seg_boundary(dev, (unsigned long)DMA_BIT_MASK(64));
  1158. sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
  1159. if (IS_ERR_OR_NULL(sgt)) {
  1160. ret = PTR_ERR(sgt);
  1161. ERR("dma_buf_map_attachment() failed: %d", ret);
  1162. goto err_detach;
  1163. }
  1164. if (sgt->nents != 1) {
  1165. ERR("nents = %d", sgt->nents);
  1166. goto err_unmap;
  1167. }
  1168. /* ensure that dma_buf is big enough for the new attrs */
  1169. dma_len = sg_dma_len(sgt->sgl);
  1170. if (dma_len < iova_min_size) {
  1171. ERR("dma len: %d is less than min ubwcp buffer size: %d",
  1172. dma_len, iova_min_size);
  1173. goto err_unmap;
  1174. }
  1175. *iova = sg_dma_address(sgt->sgl);
  1176. buf->attachment = attachment;
  1177. buf->sgt = sgt;
  1178. return ret;
  1179. err_unmap:
  1180. dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
  1181. err_detach:
  1182. dma_buf_detach(dma_buf, attachment);
  1183. err:
  1184. if (!ret)
  1185. ret = -1;
  1186. return ret;
  1187. }
  1188. static void
  1189. ubwcp_pixel_to_bytes(struct ubwcp_driver *ubwcp,
  1190. enum ubwcp_std_image_format format,
  1191. u32 width_p, u32 height_p,
  1192. u32 *width_b, u32 *height_b)
  1193. {
  1194. u16 pixel_bytes;
  1195. u16 per_pixel;
  1196. struct ubwcp_image_format_info f_info;
  1197. struct ubwcp_plane_info p_info;
  1198. f_info = ubwcp->format_info[format];
  1199. p_info = f_info.p_info[0];
  1200. pixel_bytes = p_info.pixel_bytes;
  1201. per_pixel = p_info.per_pixel;
  1202. *width_b = (width_p*pixel_bytes)/per_pixel;
  1203. *height_b = (height_p*pixel_bytes)/per_pixel;
  1204. }
  1205. static void reset_buf_attrs(struct ubwcp_buf *buf)
  1206. {
  1207. struct ubwcp_hw_meta_metadata *mmdata;
  1208. struct ubwcp_driver *ubwcp;
  1209. ubwcp = buf->ubwcp;
  1210. mmdata = &buf->mmdata;
  1211. ubwcp_dma_unmap(buf);
  1212. /* reset ula params */
  1213. if (buf->ula_size) {
  1214. ubwcp_ula_free(ubwcp, buf->ula_pa, buf->ula_size);
  1215. buf->ula_size = 0;
  1216. buf->ula_pa = 0;
  1217. }
  1218. /* reset ubwcp params */
  1219. memset(mmdata, 0, sizeof(*mmdata));
  1220. buf->buf_attr_set = false;
  1221. buf->buf_attr.image_format = UBWCP_LINEAR;
  1222. }
  1223. static void print_mmdata_desc(struct ubwcp_hw_meta_metadata *mmdata)
  1224. {
  1225. DBG_BUF_ATTR("");
  1226. DBG_BUF_ATTR("--------MM_DATA DESC ---------");
  1227. DBG_BUF_ATTR("uv_start_addr : 0x%08llx (cache addr) (actual: 0x%llx)",
  1228. mmdata->uv_start_addr, mmdata->uv_start_addr << 6);
  1229. DBG_BUF_ATTR("format : 0x%08x", mmdata->format);
  1230. DBG_BUF_ATTR("stride : 0x%08x (cache addr) (actual: 0x%x)",
  1231. mmdata->stride, mmdata->stride << 6);
  1232. DBG_BUF_ATTR("stride_ubwcp : 0x%08x (cache addr) (actual: 0x%zx)",
  1233. mmdata->stride_ubwcp, mmdata->stride_ubwcp << 6);
  1234. DBG_BUF_ATTR("metadata_base_y : 0x%08x (page addr) (actual: 0x%llx)",
  1235. mmdata->metadata_base_y, mmdata->metadata_base_y << 12);
  1236. DBG_BUF_ATTR("metadata_base_uv: 0x%08x (page addr) (actual: 0x%zx)",
  1237. mmdata->metadata_base_uv, mmdata->metadata_base_uv << 12);
  1238. DBG_BUF_ATTR("buffer_y_offset : 0x%08x (page addr) (actual: 0x%zx)",
  1239. mmdata->buffer_y_offset, mmdata->buffer_y_offset << 12);
  1240. DBG_BUF_ATTR("buffer_uv_offset: 0x%08x (page addr) (actual: 0x%zx)",
  1241. mmdata->buffer_uv_offset, mmdata->buffer_uv_offset << 12);
  1242. DBG_BUF_ATTR("width_height : 0x%08x (width: 0x%x height: 0x%x)",
  1243. mmdata->width_height, mmdata->width_height >> 16, mmdata->width_height & 0xFFFF);
  1244. DBG_BUF_ATTR("");
  1245. }
  1246. /* set buffer attributes:
  1247. * Failure:
  1248. * If a call to ubwcp_set_buf_attrs() fails, any attributes set from a previously
  1249. * successful ubwcp_set_buf_attrs() will be also removed. Thus,
  1250. * ubwcp_set_buf_attrs() implicitly does "unset previous attributes" and
  1251. * then "try to set these new attributes".
  1252. *
  1253. * The result of a failed call to ubwcp_set_buf_attrs() will leave the buffer
  1254. * in a linear mode, NOT with attributes from earlier successful call.
  1255. */
  1256. int ubwcp_set_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
  1257. {
  1258. int ret = 0;
  1259. size_t ula_size = 0;
  1260. size_t uv_start_offset = 0;
  1261. size_t ula_y_plane_size = 0;
  1262. phys_addr_t ula_pa = 0x0;
  1263. struct ubwcp_buf *buf;
  1264. struct ubwcp_driver *ubwcp;
  1265. size_t metadata_p0;
  1266. size_t pixeldata_p0;
  1267. size_t metadata_p1;
  1268. size_t pixeldata_p1;
  1269. size_t iova_min_size;
  1270. size_t stride_tp10_b;
  1271. dma_addr_t iova_base;
  1272. struct ubwcp_hw_meta_metadata *mmdata;
  1273. u64 uv_start;
  1274. u32 stride_b;
  1275. u32 width_b;
  1276. u32 height_b;
  1277. enum ubwcp_std_image_format std_image_format;
  1278. bool is_non_lin_buf;
  1279. FENTRY();
  1280. trace_ubwcp_set_buf_attrs_start(dmabuf);
  1281. if (!dmabuf) {
  1282. ERR("NULL dmabuf input ptr");
  1283. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1284. return -EINVAL;
  1285. }
  1286. if (!attr) {
  1287. ERR("NULL attr ptr");
  1288. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1289. return -EINVAL;
  1290. }
  1291. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1292. if (!buf) {
  1293. ERR("No corresponding ubwcp_buf for the passed in dma_buf");
  1294. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1295. return -EINVAL;
  1296. }
  1297. mutex_lock(&buf->lock);
  1298. if (buf->locked) {
  1299. ERR("Cannot set attr when buffer is locked");
  1300. ret = -EBUSY;
  1301. goto unlock;
  1302. }
  1303. ubwcp = buf->ubwcp;
  1304. mmdata = &buf->mmdata;
  1305. is_non_lin_buf = (buf->buf_attr.image_format != UBWCP_LINEAR);
  1306. //TBD: now that we have single exit point for all errors,
  1307. //we can limit this call to error only?
  1308. //also see if this can be part of reset_buf_attrs()
  1309. DBG_BUF_ATTR("resetting mmap to linear");
  1310. /* remove any earlier dma buf mmap configuration */
  1311. ret = ubwcp->mmap_config_fptr(buf->dma_buf, true, 0, 0);
  1312. if (ret) {
  1313. ERR("dma_buf_mmap_config() failed: %d", ret);
  1314. goto unlock;
  1315. }
  1316. if (!ubwcp_buf_attrs_valid(attr)) {
  1317. ERR("Invalid buf attrs");
  1318. goto err;
  1319. }
  1320. DBG_BUF_ATTR("valid buf attrs");
  1321. if (attr->image_format == UBWCP_LINEAR) {
  1322. DBG_BUF_ATTR("Linear format requested");
  1323. /* linear format request with permanent range xlation doesn't
  1324. * make sense. need to define behavior if this happens.
  1325. * note: with perm set, desc is allocated to this buffer.
  1326. */
  1327. //TBD: UBWCP_ASSERT(!buf->perm);
  1328. if (buf->buf_attr_set)
  1329. reset_buf_attrs(buf);
  1330. if (is_non_lin_buf) {
  1331. /*
  1332. * Changing buffer from ubwc to linear so decrement
  1333. * number of ubwc buffers
  1334. */
  1335. ret = dec_num_non_lin_buffers(ubwcp);
  1336. }
  1337. mutex_unlock(&buf->lock);
  1338. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1339. return ret;
  1340. }
  1341. std_image_format = to_std_format(attr->image_format);
  1342. if (std_image_format == STD_IMAGE_FORMAT_INVALID) {
  1343. ERR("Unable to map ioctl image format to std image format");
  1344. goto err;
  1345. }
  1346. /* Calculate uncompressed-buffer size. */
  1347. DBG_BUF_ATTR("");
  1348. DBG_BUF_ATTR("");
  1349. DBG_BUF_ATTR("Calculating ula params -->");
  1350. ret = ubwcp_calc_ula_params(ubwcp, attr, &ula_size, &ula_y_plane_size, &uv_start_offset);
  1351. if (ret) {
  1352. ERR("ubwcp_calc_ula_params() failed: %d", ret);
  1353. goto err;
  1354. }
  1355. ret = ubwcp_validate_uv_align(ubwcp, attr, ula_y_plane_size, uv_start_offset);
  1356. if (ret) {
  1357. ERR("ubwcp_validate_uv_align() failed: %d", ret);
  1358. goto err;
  1359. }
  1360. DBG_BUF_ATTR("");
  1361. DBG_BUF_ATTR("");
  1362. DBG_BUF_ATTR("Calculating ubwcp params -->");
  1363. ret = ubwcp_calc_ubwcp_buf_params(ubwcp, attr,
  1364. &metadata_p0, &pixeldata_p0,
  1365. &metadata_p1, &pixeldata_p1,
  1366. &stride_tp10_b);
  1367. if (ret) {
  1368. ERR("ubwcp_calc_buf_params() failed: %d", ret);
  1369. goto err;
  1370. }
  1371. iova_min_size = metadata_p0 + pixeldata_p0 + metadata_p1 + pixeldata_p1;
  1372. DBG_BUF_ATTR("");
  1373. DBG_BUF_ATTR("");
  1374. DBG_BUF_ATTR("------Summary ULA Calculated Params ------");
  1375. DBG_BUF_ATTR("ULA Size : %8zu (0x%8zx)", ula_size, ula_size);
  1376. DBG_BUF_ATTR("UV Start Offset : %8zu (0x%8zx)", uv_start_offset, uv_start_offset);
  1377. DBG_BUF_ATTR("------Summary UBCP Calculated Params ------");
  1378. DBG_BUF_ATTR("metadata_p0 : %8d (0x%8zx)", metadata_p0, metadata_p0);
  1379. DBG_BUF_ATTR("pixeldata_p0 : %8d (0x%8zx)", pixeldata_p0, pixeldata_p0);
  1380. DBG_BUF_ATTR("metadata_p1 : %8d (0x%8zx)", metadata_p1, metadata_p1);
  1381. DBG_BUF_ATTR("pixeldata_p1 : %8d (0x%8zx)", pixeldata_p1, pixeldata_p1);
  1382. DBG_BUF_ATTR("stride_tp10 : %8d (0x%8zx)", stride_tp10_b, stride_tp10_b);
  1383. DBG_BUF_ATTR("iova_min_size : %8d (0x%8zx)", iova_min_size, iova_min_size);
  1384. DBG_BUF_ATTR("");
  1385. if (buf->buf_attr_set) {
  1386. /* if buf attr were previously set, these must not be 0 */
  1387. /* TBD: do we need this check in production code? */
  1388. if (!buf->ula_pa) {
  1389. WARN(1, "ula_pa cannot be 0 if buf_attr_set is true!!!");
  1390. goto err;
  1391. }
  1392. if (!buf->ula_size) {
  1393. WARN(1, "ula_size cannot be 0 if buf_attr_set is true!!!");
  1394. goto err;
  1395. }
  1396. }
  1397. /* assign ULA PA with uncompressed-size range */
  1398. ula_pa = ubwcp_ula_realloc(ubwcp, buf->ula_pa, buf->ula_size, ula_size);
  1399. if (!ula_pa) {
  1400. ERR("ubwcp_ula_alloc/realloc() failed. running out of ULA PA space?");
  1401. goto err;
  1402. }
  1403. buf->ula_size = ula_size;
  1404. buf->ula_pa = ula_pa;
  1405. DBG_BUF_ATTR("Allocated ULA_PA: 0x%p of size: 0x%zx", ula_pa, ula_size);
  1406. DBG_BUF_ATTR("");
  1407. /* inform ULA-PA to dma-heap: needed for dma-heap to do CMOs later on */
  1408. DBG_BUF_ATTR("Calling mmap_config(): ULA_PA: 0x%p size: 0x%zx", ula_pa, ula_size);
  1409. ret = ubwcp->mmap_config_fptr(buf->dma_buf, false, buf->ula_pa,
  1410. buf->ula_size);
  1411. if (ret) {
  1412. ERR("dma_buf_mmap_config() failed: %d", ret);
  1413. goto err;
  1414. }
  1415. /* dma map only the first time attribute is set */
  1416. if (!buf->buf_attr_set) {
  1417. /* linear -> ubwcp. map ubwcp buffer */
  1418. ret = ubwcp_dma_map(buf, ubwcp->dev_buf_cb, iova_min_size, &iova_base);
  1419. if (ret) {
  1420. ERR("ubwcp_dma_map() failed: %d", ret);
  1421. goto err;
  1422. }
  1423. DBG_BUF_ATTR("dma_buf IOVA range: 0x%llx + min_size (0x%zx): 0x%llx",
  1424. iova_base, iova_min_size, iova_base + iova_min_size);
  1425. }
  1426. uv_start = ula_pa + uv_start_offset;
  1427. if (!IS_ALIGNED(uv_start, 64)) {
  1428. ERR("ERROR: uv_start is NOT aligned to cache line");
  1429. goto err;
  1430. }
  1431. /* Convert height and width to bytes for writing to mmdata */
  1432. if (std_image_format != TP10) {
  1433. ubwcp_pixel_to_bytes(ubwcp, std_image_format, attr->width,
  1434. attr->height, &width_b, &height_b);
  1435. } else {
  1436. /* for tp10 image compression, we need to program p010 width/height */
  1437. ubwcp_pixel_to_bytes(ubwcp, P010, attr->width,
  1438. attr->height, &width_b, &height_b);
  1439. }
  1440. stride_b = attr->stride;
  1441. /* create the mmdata descriptor */
  1442. memset(mmdata, 0, sizeof(*mmdata));
  1443. mmdata->uv_start_addr = CACHE_ADDR(uv_start);
  1444. mmdata->format = ubwcp_get_hw_image_format_value(attr->image_format);
  1445. if (std_image_format != TP10) {
  1446. mmdata->stride = CACHE_ADDR(stride_b); /* uncompressed stride */
  1447. } else {
  1448. mmdata->stride = CACHE_ADDR(stride_tp10_b); /* compressed stride */
  1449. mmdata->stride_ubwcp = CACHE_ADDR(stride_b); /* uncompressed stride */
  1450. }
  1451. mmdata->metadata_base_y = PAGE_ADDR(iova_base);
  1452. mmdata->metadata_base_uv = PAGE_ADDR(iova_base + metadata_p0 + pixeldata_p0);
  1453. mmdata->buffer_y_offset = PAGE_ADDR(metadata_p0);
  1454. mmdata->buffer_uv_offset = PAGE_ADDR(metadata_p1);
  1455. /* NOTE: For version 1.1, both width & height needs to be in bytes.
  1456. * For other versions, width in bytes & height in pixels.
  1457. */
  1458. if ((ubwcp->hw_ver_major == 1) && (ubwcp->hw_ver_minor == 1))
  1459. mmdata->width_height = width_b << 16 | height_b;
  1460. else
  1461. mmdata->width_height = width_b << 16 | attr->height;
  1462. print_mmdata_desc(mmdata);
  1463. if (!is_non_lin_buf) {
  1464. /*
  1465. * Changing buffer from linear to ubwc so increment
  1466. * number of ubwc buffers
  1467. */
  1468. ret = inc_num_non_lin_buffers(ubwcp);
  1469. }
  1470. if (ret) {
  1471. ERR("inc_num_non_lin_buffers failed: %d", ret);
  1472. goto err;
  1473. }
  1474. buf->buf_attr = *attr;
  1475. buf->buf_attr_set = true;
  1476. //TBD: UBWCP_ASSERT(!buf->perm);
  1477. mutex_unlock(&buf->lock);
  1478. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1479. return 0;
  1480. err:
  1481. reset_buf_attrs(buf);
  1482. if (is_non_lin_buf) {
  1483. /*
  1484. * Changing buffer from ubwc to linear so decrement
  1485. * number of ubwc buffers
  1486. */
  1487. dec_num_non_lin_buffers(ubwcp);
  1488. }
  1489. unlock:
  1490. mutex_unlock(&buf->lock);
  1491. if (!ret)
  1492. ret = -1;
  1493. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1494. return ret;
  1495. }
  1496. EXPORT_SYMBOL(ubwcp_set_buf_attrs);
  1497. /* Set buffer attributes ioctl */
  1498. static int ubwcp_set_buf_attrs_ioctl(struct ubwcp_ioctl_buffer_attrs *attr_ioctl)
  1499. {
  1500. struct dma_buf *dmabuf;
  1501. dmabuf = ubwcp_dma_buf_fd_to_dma_buf(attr_ioctl->fd);
  1502. return ubwcp_set_buf_attrs(dmabuf, &attr_ioctl->attr);
  1503. }
  1504. /* Free up the buffer descriptor */
  1505. static void ubwcp_buf_desc_free(struct ubwcp_driver *ubwcp, struct ubwcp_desc *desc)
  1506. {
  1507. int idx = desc->idx;
  1508. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  1509. mutex_lock(&ubwcp->desc_lock);
  1510. desc_list[idx].idx = -1;
  1511. desc_list[idx].ptr = NULL;
  1512. DBG("freed descriptor_id: %d", idx);
  1513. mutex_unlock(&ubwcp->desc_lock);
  1514. }
  1515. /* Allocate next available buffer descriptor. */
  1516. static struct ubwcp_desc *ubwcp_buf_desc_allocate(struct ubwcp_driver *ubwcp)
  1517. {
  1518. int idx;
  1519. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  1520. mutex_lock(&ubwcp->desc_lock);
  1521. for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
  1522. if (desc_list[idx].idx == -1) {
  1523. desc_list[idx].idx = idx;
  1524. desc_list[idx].ptr = ubwcp->buffer_desc_base +
  1525. idx*UBWCP_BUFFER_DESC_OFFSET;
  1526. DBG("allocated descriptor_id: %d", idx);
  1527. mutex_unlock(&ubwcp->desc_lock);
  1528. return &desc_list[idx];
  1529. }
  1530. }
  1531. mutex_unlock(&ubwcp->desc_lock);
  1532. return NULL;
  1533. }
  1534. /**
  1535. * Lock buffer for CPU access. This prepares ubwcp hw to allow
  1536. * CPU access to the compressed buffer. It will perform
  1537. * necessary address translation configuration and cache maintenance ops
  1538. * so that CPU can safely access ubwcp buffer, if this call is
  1539. * successful.
  1540. * Allocate descriptor if not already,
  1541. * perform CMO and then enable range check
  1542. *
  1543. * @param dmabuf : ptr to the dma buf
  1544. * @param direction : direction of access
  1545. *
  1546. * @return int : 0 on success, otherwise error code
  1547. */
  1548. static int ubwcp_lock(struct dma_buf *dmabuf, enum dma_data_direction dir)
  1549. {
  1550. int ret = 0;
  1551. struct ubwcp_buf *buf;
  1552. struct ubwcp_driver *ubwcp;
  1553. FENTRY();
  1554. trace_ubwcp_lock_start(dmabuf);
  1555. if (!dmabuf) {
  1556. ERR("NULL dmabuf input ptr");
  1557. trace_ubwcp_lock_end(dmabuf);
  1558. return -EINVAL;
  1559. }
  1560. if (!valid_dma_direction(dir)) {
  1561. ERR("invalid direction: %d", dir);
  1562. trace_ubwcp_lock_end(dmabuf);
  1563. return -EINVAL;
  1564. }
  1565. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1566. if (!buf) {
  1567. ERR("ubwcp_buf ptr not found");
  1568. trace_ubwcp_lock_end(dmabuf);
  1569. return -1;
  1570. }
  1571. mutex_lock(&buf->lock);
  1572. if (!buf->buf_attr_set) {
  1573. ERR("lock() called on buffer, but attr not set");
  1574. goto err;
  1575. }
  1576. if (buf->buf_attr.image_format == UBWCP_LINEAR) {
  1577. ERR("lock() called on linear buffer");
  1578. goto err;
  1579. }
  1580. if (!buf->locked) {
  1581. DBG("first lock on buffer");
  1582. ubwcp = buf->ubwcp;
  1583. /* buf->desc could already be allocated because of perm range xlation */
  1584. if (!buf->desc) {
  1585. /* allocate a buffer descriptor */
  1586. buf->desc = ubwcp_buf_desc_allocate(buf->ubwcp);
  1587. if (!buf->desc) {
  1588. ERR("ubwcp_allocate_buf_desc() failed");
  1589. goto err;
  1590. }
  1591. memcpy(buf->desc->ptr, &buf->mmdata, sizeof(buf->mmdata));
  1592. /* Flushing of updated mmdata:
  1593. * mmdata is iocoherent and ubwcp will get it from CPU cache -
  1594. * *as long as* it has not cached that itself during previous
  1595. * access to the same descriptor.
  1596. *
  1597. * During unlock of previous use of this descriptor,
  1598. * we do hw flush, which will get rid of this mmdata from
  1599. * ubwcp cache.
  1600. *
  1601. * In addition, we also do a hw flush after enable_range_ck().
  1602. * That will also get rid of any speculative fetch of mmdata
  1603. * by the ubwcp hw. At this time, the assumption is that ubwcp
  1604. * will cache mmdata only for active descriptor. But if ubwcp
  1605. * is speculatively fetching mmdata for all descriptors
  1606. * (irrespetive of enabled or not), the flush during lock
  1607. * will be necessary to make sure ubwcp sees updated mmdata
  1608. * that we just updated
  1609. */
  1610. /* program ULA range for this buffer */
  1611. DBG("setting range check: descriptor_id: %d, addr: %p, size: %zx",
  1612. buf->desc->idx, buf->ula_pa, buf->ula_size);
  1613. ubwcp_hw_set_range_check(ubwcp->base, buf->desc->idx, buf->ula_pa,
  1614. buf->ula_size);
  1615. }
  1616. /* enable range check */
  1617. DBG("enabling range check, descriptor_id: %d", buf->desc->idx);
  1618. mutex_lock(&ubwcp->hw_range_ck_lock);
  1619. ubwcp_hw_enable_range_check(ubwcp->base, buf->desc->idx);
  1620. mutex_unlock(&ubwcp->hw_range_ck_lock);
  1621. /* Flush/invalidate UBWCP caches */
  1622. /* Why: cpu could have done a speculative fetch before
  1623. * enable_range_ck() and ubwcp in process of returning "default" data
  1624. * we don't want that stashing of default data pending.
  1625. * we force completion of that and then we also cpu invalidate which
  1626. * will get rid of that line.
  1627. */
  1628. trace_ubwcp_hw_flush_start(buf->ula_size);
  1629. ubwcp_flush(ubwcp);
  1630. trace_ubwcp_hw_flush_end(buf->ula_size);
  1631. /* Flush/invalidate ULA PA from CPU caches
  1632. * TBD: if (dir == READ or BIDIRECTION) //NOT for write
  1633. * -- Confirm with Chris if this can be skipped for write
  1634. */
  1635. trace_ubwcp_dma_sync_single_for_cpu_start(buf->ula_size);
  1636. dma_sync_single_for_cpu(ubwcp->dev, buf->ula_pa, buf->ula_size, dir);
  1637. trace_ubwcp_dma_sync_single_for_cpu_end(buf->ula_size);
  1638. buf->lock_dir = dir;
  1639. buf->locked = true;
  1640. } else {
  1641. DBG("buf already locked");
  1642. /* TBD: what if new buffer direction is not same as previous?
  1643. * must update the dir.
  1644. */
  1645. }
  1646. buf->lock_count++;
  1647. DBG("new lock_count: %d", buf->lock_count);
  1648. mutex_unlock(&buf->lock);
  1649. trace_ubwcp_lock_end(dmabuf);
  1650. return ret;
  1651. err:
  1652. mutex_unlock(&buf->lock);
  1653. if (!ret)
  1654. ret = -1;
  1655. trace_ubwcp_lock_end(dmabuf);
  1656. return ret;
  1657. }
  1658. /* This can be called as a result of external unlock() call or
  1659. * internally if free() is called without unlock().
  1660. */
  1661. static int unlock_internal(struct ubwcp_buf *buf, enum dma_data_direction dir, bool free_buffer)
  1662. {
  1663. int ret = 0;
  1664. struct ubwcp_driver *ubwcp;
  1665. DBG("current lock_count: %d", buf->lock_count);
  1666. if (free_buffer) {
  1667. buf->lock_count = 0;
  1668. DBG("Forced lock_count: %d", buf->lock_count);
  1669. } else {
  1670. buf->lock_count--;
  1671. DBG("new lock_count: %d", buf->lock_count);
  1672. if (buf->lock_count) {
  1673. DBG("more than 1 lock on buffer. waiting until last unlock");
  1674. return 0;
  1675. }
  1676. }
  1677. ubwcp = buf->ubwcp;
  1678. /* Flush/invalidate ULA PA from CPU caches */
  1679. //TBD: if (dir == WRITE or BIDIRECTION)
  1680. trace_ubwcp_dma_sync_single_for_device_start(buf->ula_size);
  1681. dma_sync_single_for_device(ubwcp->dev, buf->ula_pa, buf->ula_size, dir);
  1682. trace_ubwcp_dma_sync_single_for_device_end(buf->ula_size);
  1683. /* disable range check with ubwcp flush */
  1684. DBG("disabling range check");
  1685. //TBD: could combine these 2 locks into a single lock to make it simpler
  1686. mutex_lock(&ubwcp->ubwcp_flush_lock);
  1687. mutex_lock(&ubwcp->hw_range_ck_lock);
  1688. trace_ubwcp_hw_flush_start(buf->ula_size);
  1689. ret = ubwcp_hw_disable_range_check_with_flush(ubwcp->base, buf->desc->idx);
  1690. trace_ubwcp_hw_flush_end(buf->ula_size);
  1691. if (ret)
  1692. ERR("disable_range_check_with_flush() failed: %d", ret);
  1693. mutex_unlock(&ubwcp->hw_range_ck_lock);
  1694. mutex_unlock(&ubwcp->ubwcp_flush_lock);
  1695. /* release descriptor if perm range xlation is not set */
  1696. if (!buf->perm) {
  1697. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1698. buf->desc = NULL;
  1699. }
  1700. buf->locked = false;
  1701. return ret;
  1702. }
  1703. /**
  1704. * Unlock buffer from CPU access. This prepares ubwcp hw to
  1705. * safely allow for device access to the compressed buffer including any
  1706. * necessary cache maintenance ops. It may also free up certain ubwcp
  1707. * resources that could result in error when accessed by CPU in
  1708. * unlocked state.
  1709. *
  1710. * @param dmabuf : ptr to the dma buf
  1711. * @param direction : direction of access
  1712. *
  1713. * @return int : 0 on success, otherwise error code
  1714. */
  1715. static int ubwcp_unlock(struct dma_buf *dmabuf, enum dma_data_direction dir)
  1716. {
  1717. struct ubwcp_buf *buf;
  1718. int ret;
  1719. FENTRY();
  1720. trace_ubwcp_unlock_start(dmabuf);
  1721. if (!dmabuf) {
  1722. ERR("NULL dmabuf input ptr");
  1723. trace_ubwcp_unlock_end(dmabuf);
  1724. return -EINVAL;
  1725. }
  1726. if (!valid_dma_direction(dir)) {
  1727. ERR("invalid direction: %d", dir);
  1728. trace_ubwcp_unlock_end(dmabuf);
  1729. return -EINVAL;
  1730. }
  1731. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1732. if (!buf) {
  1733. ERR("ubwcp_buf not found");
  1734. trace_ubwcp_unlock_end(dmabuf);
  1735. return -1;
  1736. }
  1737. if (!buf->locked) {
  1738. ERR("unlock() called on buffer which not in locked state");
  1739. trace_ubwcp_unlock_end(dmabuf);
  1740. return -1;
  1741. }
  1742. error_print_count = 0;
  1743. mutex_lock(&buf->lock);
  1744. ret = unlock_internal(buf, dir, false);
  1745. mutex_unlock(&buf->lock);
  1746. trace_ubwcp_unlock_end(dmabuf);
  1747. return ret;
  1748. }
  1749. /* Return buffer attributes for the given buffer */
  1750. int ubwcp_get_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
  1751. {
  1752. int ret = 0;
  1753. struct ubwcp_buf *buf;
  1754. FENTRY();
  1755. if (!dmabuf) {
  1756. ERR("NULL dmabuf input ptr");
  1757. return -EINVAL;
  1758. }
  1759. if (!attr) {
  1760. ERR("NULL attr ptr");
  1761. return -EINVAL;
  1762. }
  1763. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1764. if (!buf) {
  1765. ERR("ubwcp_buf ptr not found");
  1766. return -1;
  1767. }
  1768. mutex_lock(&buf->lock);
  1769. if (!buf->buf_attr_set) {
  1770. ERR("buffer attributes not set");
  1771. mutex_unlock(&buf->lock);
  1772. return -1;
  1773. }
  1774. *attr = buf->buf_attr;
  1775. mutex_unlock(&buf->lock);
  1776. return ret;
  1777. }
  1778. EXPORT_SYMBOL(ubwcp_get_buf_attrs);
  1779. /* Set permanent range translation.
  1780. * enable: Descriptor will be reserved for this buffer until disabled,
  1781. * making lock/unlock quicker.
  1782. * disable: Descriptor will not be reserved for this buffer. Instead,
  1783. * descriptor will be allocated and released for each lock/unlock.
  1784. * If currently allocated but not being used, descriptor will be
  1785. * released.
  1786. */
  1787. int ubwcp_set_perm_range_translation(struct dma_buf *dmabuf, bool enable)
  1788. {
  1789. int ret = 0;
  1790. struct ubwcp_buf *buf;
  1791. FENTRY();
  1792. if (!dmabuf) {
  1793. ERR("NULL dmabuf input ptr");
  1794. return -EINVAL;
  1795. }
  1796. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1797. if (!buf) {
  1798. ERR("ubwcp_buf not found");
  1799. return -1;
  1800. }
  1801. /* not implemented */
  1802. if (1) {
  1803. ERR("API not implemented yet");
  1804. return -1;
  1805. }
  1806. /* TBD: make sure we acquire buf lock while setting this so there is
  1807. * no race condition with attr_set/lock/unlock
  1808. */
  1809. buf->perm = enable;
  1810. /* if "disable" and we have allocated a desc and it is not being
  1811. * used currently, release it
  1812. */
  1813. if (!enable && buf->desc && !buf->locked) {
  1814. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1815. buf->desc = NULL;
  1816. /* Flush/invalidate UBWCP caches */
  1817. //TBD: need to do anything?
  1818. }
  1819. return ret;
  1820. }
  1821. EXPORT_SYMBOL(ubwcp_set_perm_range_translation);
  1822. /**
  1823. * Free up ubwcp resources for this buffer.
  1824. *
  1825. * @param dmabuf : ptr to the dma buf
  1826. *
  1827. * @return int : 0 on success, otherwise error code
  1828. */
  1829. static int ubwcp_free_buffer(struct dma_buf *dmabuf)
  1830. {
  1831. int ret = 0;
  1832. struct ubwcp_buf *buf;
  1833. struct ubwcp_driver *ubwcp;
  1834. unsigned long flags;
  1835. bool is_non_lin_buf;
  1836. FENTRY();
  1837. trace_ubwcp_free_buffer_start(dmabuf);
  1838. if (!dmabuf) {
  1839. ERR("NULL dmabuf input ptr");
  1840. trace_ubwcp_free_buffer_end(dmabuf);
  1841. return -EINVAL;
  1842. }
  1843. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1844. if (!buf) {
  1845. ERR("ubwcp_buf ptr not found");
  1846. trace_ubwcp_free_buffer_end(dmabuf);
  1847. return -1;
  1848. }
  1849. mutex_lock(&buf->lock);
  1850. ubwcp = buf->ubwcp;
  1851. is_non_lin_buf = (buf->buf_attr.image_format != UBWCP_LINEAR);
  1852. if (buf->locked) {
  1853. DBG("free() called without unlock. unlock()'ing first...");
  1854. ret = unlock_internal(buf, buf->lock_dir, true);
  1855. if (ret)
  1856. ERR("unlock_internal(): failed : %d, but continuing free()", ret);
  1857. }
  1858. /* if we are still holding a desc, release it. this can happen only if perm == true */
  1859. if (buf->desc) {
  1860. WARN_ON(!buf->perm); /* TBD: change to BUG() later...*/
  1861. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1862. buf->desc = NULL;
  1863. }
  1864. if (buf->buf_attr_set)
  1865. reset_buf_attrs(buf);
  1866. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  1867. hash_del(&buf->hnode);
  1868. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  1869. kfree(buf);
  1870. if (is_non_lin_buf)
  1871. dec_num_non_lin_buffers(ubwcp);
  1872. trace_ubwcp_free_buffer_end(dmabuf);
  1873. return 0;
  1874. }
  1875. /* file open: TBD: increment ref count? */
  1876. static int ubwcp_open(struct inode *i, struct file *f)
  1877. {
  1878. return 0;
  1879. }
  1880. /* file open: TBD: decrement ref count? */
  1881. static int ubwcp_close(struct inode *i, struct file *f)
  1882. {
  1883. return 0;
  1884. }
  1885. /* handle IOCTLs */
  1886. static long ubwcp_ioctl(struct file *file, unsigned int ioctl_num, unsigned long ioctl_param)
  1887. {
  1888. struct ubwcp_ioctl_buffer_attrs buf_attr_ioctl;
  1889. struct ubwcp_ioctl_hw_version hw_ver;
  1890. switch (ioctl_num) {
  1891. case UBWCP_IOCTL_SET_BUF_ATTR:
  1892. if (copy_from_user(&buf_attr_ioctl, (const void __user *) ioctl_param,
  1893. sizeof(buf_attr_ioctl))) {
  1894. ERR("ERROR: copy_from_user() failed");
  1895. return -EFAULT;
  1896. }
  1897. DBG("IOCTL : SET_BUF_ATTR: fd = %d", buf_attr_ioctl.fd);
  1898. return ubwcp_set_buf_attrs_ioctl(&buf_attr_ioctl);
  1899. case UBWCP_IOCTL_GET_HW_VER:
  1900. DBG("IOCTL : GET_HW_VER");
  1901. ubwcp_get_hw_version(&hw_ver);
  1902. if (copy_to_user((void __user *)ioctl_param, &hw_ver, sizeof(hw_ver))) {
  1903. ERR("ERROR: copy_to_user() failed");
  1904. return -EFAULT;
  1905. }
  1906. break;
  1907. default:
  1908. ERR("Invalid ioctl_num = %d", ioctl_num);
  1909. return -EINVAL;
  1910. }
  1911. return 0;
  1912. }
  1913. static const struct file_operations ubwcp_fops = {
  1914. .owner = THIS_MODULE,
  1915. .open = ubwcp_open,
  1916. .release = ubwcp_close,
  1917. .unlocked_ioctl = ubwcp_ioctl,
  1918. };
  1919. static int ubwcp_debugfs_init(struct ubwcp_driver *ubwcp)
  1920. {
  1921. struct dentry *debugfs_root;
  1922. debugfs_root = debugfs_create_dir("ubwcp", NULL);
  1923. if (!debugfs_root) {
  1924. pr_warn("Failed to create debugfs for ubwcp\n");
  1925. return -1;
  1926. }
  1927. debugfs_create_u32("debug_trace_enable", 0644, debugfs_root, &ubwcp_debug_trace_enable);
  1928. ubwcp->debugfs_root = debugfs_root;
  1929. return 0;
  1930. }
  1931. static void ubwcp_debugfs_deinit(struct ubwcp_driver *ubwcp)
  1932. {
  1933. debugfs_remove_recursive(ubwcp->debugfs_root);
  1934. }
  1935. /* ubwcp char device initialization */
  1936. static int ubwcp_cdev_init(struct ubwcp_driver *ubwcp)
  1937. {
  1938. int ret;
  1939. dev_t devt;
  1940. struct class *dev_class;
  1941. struct device *dev_sys;
  1942. /* allocate major device number (/proc/devices -> major_num ubwcp) */
  1943. ret = alloc_chrdev_region(&devt, 0, UBWCP_NUM_DEVICES, UBWCP_DEVICE_NAME);
  1944. if (ret) {
  1945. ERR("alloc_chrdev_region() failed: %d", ret);
  1946. return ret;
  1947. }
  1948. /* create device class (/sys/class/ubwcp_class) */
  1949. dev_class = class_create(THIS_MODULE, "ubwcp_class");
  1950. if (IS_ERR(dev_class)) {
  1951. ERR("class_create() failed");
  1952. return -1;
  1953. }
  1954. /* Create device and register with sysfs
  1955. * (/sys/class/ubwcp_class/ubwcp/... -> dev/power/subsystem/uevent)
  1956. */
  1957. dev_sys = device_create(dev_class, NULL, devt, NULL,
  1958. UBWCP_DEVICE_NAME);
  1959. if (IS_ERR(dev_sys)) {
  1960. ERR("device_create() failed");
  1961. return -1;
  1962. }
  1963. /* register file operations and get cdev */
  1964. cdev_init(&ubwcp->cdev, &ubwcp_fops);
  1965. /* associate cdev and device major/minor with file system
  1966. * can do file ops on /dev/ubwcp after this
  1967. */
  1968. ret = cdev_add(&ubwcp->cdev, devt, 1);
  1969. if (ret) {
  1970. ERR("cdev_add() failed");
  1971. return -1;
  1972. }
  1973. ubwcp->devt = devt;
  1974. ubwcp->dev_class = dev_class;
  1975. ubwcp->dev_sys = dev_sys;
  1976. return 0;
  1977. }
  1978. static void ubwcp_cdev_deinit(struct ubwcp_driver *ubwcp)
  1979. {
  1980. device_destroy(ubwcp->dev_class, ubwcp->devt);
  1981. class_destroy(ubwcp->dev_class);
  1982. cdev_del(&ubwcp->cdev);
  1983. unregister_chrdev_region(ubwcp->devt, UBWCP_NUM_DEVICES);
  1984. }
  1985. struct handler_node {
  1986. struct list_head list;
  1987. u32 client_id;
  1988. ubwcp_error_handler_t handler;
  1989. void *data;
  1990. };
  1991. int ubwcp_register_error_handler(u32 client_id, ubwcp_error_handler_t handler,
  1992. void *data)
  1993. {
  1994. struct handler_node *node;
  1995. unsigned long flags;
  1996. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1997. if (!ubwcp)
  1998. return -EINVAL;
  1999. if (client_id != -1)
  2000. return -EINVAL;
  2001. if (!handler)
  2002. return -EINVAL;
  2003. node = kzalloc(sizeof(*node), GFP_KERNEL);
  2004. if (!node)
  2005. return -ENOMEM;
  2006. node->client_id = client_id;
  2007. node->handler = handler;
  2008. node->data = data;
  2009. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  2010. list_add_tail(&node->list, &ubwcp->err_handler_list);
  2011. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  2012. return 0;
  2013. }
  2014. EXPORT_SYMBOL(ubwcp_register_error_handler);
  2015. static void ubwcp_notify_error_handlers(struct unwcp_err_info *err)
  2016. {
  2017. struct handler_node *node;
  2018. unsigned long flags;
  2019. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2020. if (!ubwcp)
  2021. return;
  2022. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  2023. list_for_each_entry(node, &ubwcp->err_handler_list, list)
  2024. node->handler(err, node->data);
  2025. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  2026. }
  2027. int ubwcp_unregister_error_handler(u32 client_id)
  2028. {
  2029. int ret = -EINVAL;
  2030. struct handler_node *node;
  2031. unsigned long flags;
  2032. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2033. if (!ubwcp)
  2034. return -EINVAL;
  2035. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  2036. list_for_each_entry(node, &ubwcp->err_handler_list, list)
  2037. if (node->client_id == client_id) {
  2038. list_del(&node->list);
  2039. kfree(node);
  2040. ret = 0;
  2041. break;
  2042. }
  2043. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  2044. return ret;
  2045. }
  2046. EXPORT_SYMBOL(ubwcp_unregister_error_handler);
  2047. /* get ubwcp_buf corresponding to the ULA PA*/
  2048. static struct dma_buf *get_dma_buf_from_ulapa(phys_addr_t addr)
  2049. {
  2050. struct ubwcp_buf *buf = NULL;
  2051. struct dma_buf *ret_buf = NULL;
  2052. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2053. unsigned long flags;
  2054. u32 i;
  2055. if (!ubwcp)
  2056. return NULL;
  2057. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  2058. hash_for_each(ubwcp->buf_table, i, buf, hnode) {
  2059. if (buf->ula_pa <= addr && addr < buf->ula_pa + buf->ula_size) {
  2060. ret_buf = buf->dma_buf;
  2061. break;
  2062. }
  2063. }
  2064. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  2065. return ret_buf;
  2066. }
  2067. /* get ubwcp_buf corresponding to the IOVA*/
  2068. static struct dma_buf *get_dma_buf_from_iova(unsigned long addr)
  2069. {
  2070. struct ubwcp_buf *buf = NULL;
  2071. struct dma_buf *ret_buf = NULL;
  2072. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2073. unsigned long flags;
  2074. u32 i;
  2075. if (!ubwcp)
  2076. return NULL;
  2077. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  2078. hash_for_each(ubwcp->buf_table, i, buf, hnode) {
  2079. unsigned long iova_base;
  2080. unsigned int iova_size;
  2081. if (!buf->sgt)
  2082. continue;
  2083. iova_base = sg_dma_address(buf->sgt->sgl);
  2084. iova_size = sg_dma_len(buf->sgt->sgl);
  2085. if (iova_base <= addr && addr < iova_base + iova_size) {
  2086. ret_buf = buf->dma_buf;
  2087. break;
  2088. }
  2089. }
  2090. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  2091. return ret_buf;
  2092. }
  2093. #define ERR_PRINT_COUNT_MAX 21
  2094. /* TBD: use proper rate limit for debug prints */
  2095. int ubwcp_iommu_fault_handler(struct iommu_domain *domain, struct device *dev,
  2096. unsigned long iova, int flags, void *data)
  2097. {
  2098. int ret = 0;
  2099. struct unwcp_err_info err;
  2100. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2101. struct device *cb_dev = (struct device *)data;
  2102. if (!ubwcp) {
  2103. ret = -EINVAL;
  2104. goto err;
  2105. }
  2106. error_print_count++;
  2107. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  2108. err.err_code = UBWCP_SMMU_FAULT;
  2109. if (cb_dev == ubwcp->dev_desc_cb)
  2110. err.smmu_err.iommu_dev_id = UBWCP_DESC_CB_ID;
  2111. else if (cb_dev == ubwcp->dev_buf_cb)
  2112. err.smmu_err.iommu_dev_id = UBWCP_BUF_CB_ID;
  2113. else
  2114. err.smmu_err.iommu_dev_id = UBWCP_UNKNOWN_CB_ID;
  2115. ERR("smmu fault error: iommu_dev_id:%d iova 0x%llx flags:0x%x",
  2116. err.smmu_err.iommu_dev_id, iova, flags);
  2117. err.smmu_err.dmabuf = get_dma_buf_from_iova(iova);
  2118. err.smmu_err.iova = iova;
  2119. err.smmu_err.iommu_fault_flags = flags;
  2120. ubwcp_notify_error_handlers(&err);
  2121. }
  2122. err:
  2123. return ret;
  2124. }
  2125. irqreturn_t ubwcp_irq_handler(int irq, void *ptr)
  2126. {
  2127. struct ubwcp_driver *ubwcp;
  2128. void __iomem *base;
  2129. u64 src;
  2130. phys_addr_t addr;
  2131. struct unwcp_err_info err;
  2132. error_print_count++;
  2133. ubwcp = (struct ubwcp_driver *) ptr;
  2134. base = ubwcp->base;
  2135. if (irq == ubwcp->irq_range_ck_rd) {
  2136. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  2137. src = ubwcp_hw_interrupt_src_address(base, 0);
  2138. addr = src << 6;
  2139. ERR("check range read error: src: 0x%llx", addr);
  2140. err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
  2141. err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2142. err.translation_err.ula_pa = addr;
  2143. err.translation_err.read = true;
  2144. ubwcp_notify_error_handlers(&err);
  2145. }
  2146. ubwcp_hw_interrupt_clear(ubwcp->base, 0);
  2147. } else if (irq == ubwcp->irq_range_ck_wr) {
  2148. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  2149. src = ubwcp_hw_interrupt_src_address(base, 1);
  2150. addr = src << 6;
  2151. ERR("check range write error: src: 0x%llx", addr);
  2152. err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
  2153. err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2154. err.translation_err.ula_pa = addr;
  2155. err.translation_err.read = false;
  2156. ubwcp_notify_error_handlers(&err);
  2157. }
  2158. ubwcp_hw_interrupt_clear(ubwcp->base, 1);
  2159. } else if (irq == ubwcp->irq_encode) {
  2160. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  2161. src = ubwcp_hw_interrupt_src_address(base, 3);
  2162. addr = src << 6;
  2163. ERR("encode error: src: 0x%llx", addr);
  2164. err.err_code = UBWCP_ENCODE_ERROR;
  2165. err.enc_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2166. err.enc_err.ula_pa = addr;
  2167. ubwcp_notify_error_handlers(&err);
  2168. }
  2169. ubwcp_hw_interrupt_clear(ubwcp->base, 3); //TBD: encode is bit-3 instead of bit-2
  2170. } else if (irq == ubwcp->irq_decode) {
  2171. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  2172. src = ubwcp_hw_interrupt_src_address(base, 2);
  2173. addr = src << 6;
  2174. ERR("decode error: src: 0x%llx", addr);
  2175. err.err_code = UBWCP_DECODE_ERROR;
  2176. err.dec_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2177. err.dec_err.ula_pa = addr;
  2178. ubwcp_notify_error_handlers(&err);
  2179. }
  2180. ubwcp_hw_interrupt_clear(ubwcp->base, 2); //TBD: decode is bit-2 instead of bit-3
  2181. } else {
  2182. ERR("unknown irq: %d", irq);
  2183. return IRQ_NONE;
  2184. }
  2185. return IRQ_HANDLED;
  2186. }
  2187. static int ubwcp_interrupt_register(struct platform_device *pdev, struct ubwcp_driver *ubwcp)
  2188. {
  2189. int ret = 0;
  2190. struct device *dev = &pdev->dev;
  2191. FENTRY();
  2192. ubwcp->irq_range_ck_rd = platform_get_irq(pdev, 0);
  2193. if (ubwcp->irq_range_ck_rd < 0)
  2194. return ubwcp->irq_range_ck_rd;
  2195. ubwcp->irq_range_ck_wr = platform_get_irq(pdev, 1);
  2196. if (ubwcp->irq_range_ck_wr < 0)
  2197. return ubwcp->irq_range_ck_wr;
  2198. ubwcp->irq_encode = platform_get_irq(pdev, 2);
  2199. if (ubwcp->irq_encode < 0)
  2200. return ubwcp->irq_encode;
  2201. ubwcp->irq_decode = platform_get_irq(pdev, 3);
  2202. if (ubwcp->irq_decode < 0)
  2203. return ubwcp->irq_decode;
  2204. DBG("got irqs: %d %d %d %d", ubwcp->irq_range_ck_rd,
  2205. ubwcp->irq_range_ck_wr,
  2206. ubwcp->irq_encode,
  2207. ubwcp->irq_decode);
  2208. ret = devm_request_irq(dev, ubwcp->irq_range_ck_rd, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2209. if (ret) {
  2210. ERR("request_irq() failed. irq: %d ret: %d",
  2211. ubwcp->irq_range_ck_rd, ret);
  2212. return ret;
  2213. }
  2214. ret = devm_request_irq(dev, ubwcp->irq_range_ck_wr, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2215. if (ret) {
  2216. ERR("request_irq() failed. irq: %d ret: %d",
  2217. ubwcp->irq_range_ck_wr, ret);
  2218. return ret;
  2219. }
  2220. ret = devm_request_irq(dev, ubwcp->irq_encode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2221. if (ret) {
  2222. ERR("request_irq() failed. irq: %d ret: %d",
  2223. ubwcp->irq_encode, ret);
  2224. return ret;
  2225. }
  2226. ret = devm_request_irq(dev, ubwcp->irq_decode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2227. if (ret) {
  2228. ERR("request_irq() failed. irq: %d ret: %d",
  2229. ubwcp->irq_decode, ret);
  2230. return ret;
  2231. }
  2232. return ret;
  2233. }
  2234. /* ubwcp device probe */
  2235. static int qcom_ubwcp_probe(struct platform_device *pdev)
  2236. {
  2237. int ret = 0;
  2238. struct ubwcp_driver *ubwcp;
  2239. struct device *ubwcp_dev = &pdev->dev;
  2240. FENTRY();
  2241. ubwcp = devm_kzalloc(ubwcp_dev, sizeof(*ubwcp), GFP_KERNEL);
  2242. if (!ubwcp) {
  2243. ERR("devm_kzalloc() failed");
  2244. return -ENOMEM;
  2245. }
  2246. ubwcp->dev = &pdev->dev;
  2247. ret = dma_set_mask_and_coherent(ubwcp->dev, DMA_BIT_MASK(64));
  2248. #ifdef UBWCP_USE_SMC
  2249. {
  2250. struct resource res;
  2251. of_address_to_resource(ubwcp_dev->of_node, 0, &res);
  2252. ubwcp->base = (void __iomem *) res.start;
  2253. DBG("Using SMC calls. base: %p", ubwcp->base);
  2254. }
  2255. #else
  2256. ubwcp->base = devm_platform_ioremap_resource(pdev, 0);
  2257. if (IS_ERR(ubwcp->base)) {
  2258. ERR("devm ioremap() failed: %d", PTR_ERR(ubwcp->base));
  2259. return PTR_ERR(ubwcp->base);
  2260. }
  2261. DBG("ubwcp->base: %p", ubwcp->base);
  2262. #endif
  2263. ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 0, &ubwcp->ula_pool_base);
  2264. if (ret) {
  2265. ERR("failed reading ula_range (base): %d", ret);
  2266. return ret;
  2267. }
  2268. DBG("ubwcp: ula_range: base = 0x%lx", ubwcp->ula_pool_base);
  2269. ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 1, &ubwcp->ula_pool_size);
  2270. if (ret) {
  2271. ERR("failed reading ula_range (size): %d", ret);
  2272. return ret;
  2273. }
  2274. DBG("ubwcp: ula_range: size = 0x%lx", ubwcp->ula_pool_size);
  2275. INIT_LIST_HEAD(&ubwcp->err_handler_list);
  2276. atomic_set(&ubwcp->num_non_lin_buffers, 0);
  2277. ubwcp->mem_online = false;
  2278. mutex_init(&ubwcp->desc_lock);
  2279. spin_lock_init(&ubwcp->buf_table_lock);
  2280. mutex_init(&ubwcp->mem_hotplug_lock);
  2281. mutex_init(&ubwcp->ula_lock);
  2282. mutex_init(&ubwcp->ubwcp_flush_lock);
  2283. mutex_init(&ubwcp->hw_range_ck_lock);
  2284. spin_lock_init(&ubwcp->err_handler_list_lock);
  2285. if (ubwcp_interrupt_register(pdev, ubwcp))
  2286. return -1;
  2287. /* Regulator */
  2288. ubwcp->vdd = devm_regulator_get(ubwcp_dev, "vdd");
  2289. if (IS_ERR_OR_NULL(ubwcp->vdd)) {
  2290. ret = PTR_ERR(ubwcp->vdd);
  2291. ERR("devm_regulator_get() failed: %d", ret);
  2292. return -1;
  2293. }
  2294. ret = ubwcp_init_clocks(ubwcp, ubwcp_dev);
  2295. if (ret) {
  2296. ERR("failed to initialize ubwcp clocks err: %d", ret);
  2297. return ret;
  2298. }
  2299. if (ubwcp_power(ubwcp, true))
  2300. return -1;
  2301. if (ubwcp_cdev_init(ubwcp))
  2302. return -1;
  2303. if (ubwcp_debugfs_init(ubwcp))
  2304. return -1;
  2305. /* create ULA pool */
  2306. ubwcp->ula_pool = gen_pool_create(12, -1);
  2307. if (!ubwcp->ula_pool) {
  2308. ERR("failed gen_pool_create()");
  2309. ret = -1;
  2310. goto err_pool_create;
  2311. }
  2312. ret = gen_pool_add(ubwcp->ula_pool, ubwcp->ula_pool_base, ubwcp->ula_pool_size, -1);
  2313. if (ret) {
  2314. ERR("failed gen_pool_add(): %d", ret);
  2315. ret = -1;
  2316. goto err_pool_add;
  2317. }
  2318. /* register the default config mmap function. */
  2319. ubwcp->mmap_config_fptr = msm_ubwcp_dma_buf_configure_mmap;
  2320. hash_init(ubwcp->buf_table);
  2321. ubwcp_buf_desc_list_init(ubwcp);
  2322. image_format_init(ubwcp);
  2323. /* one time hw init */
  2324. ubwcp_hw_one_time_init(ubwcp->base);
  2325. ubwcp_hw_version(ubwcp->base, &ubwcp->hw_ver_major, &ubwcp->hw_ver_minor);
  2326. pr_err("ubwcp: hw version: major %d, minor %d\n", ubwcp->hw_ver_major, ubwcp->hw_ver_minor);
  2327. if (ubwcp->hw_ver_major == 0) {
  2328. ERR("Failed to read HW version");
  2329. ret = -1;
  2330. goto err_pool_add;
  2331. }
  2332. /* set pdev->dev->driver_data = ubwcp */
  2333. platform_set_drvdata(pdev, ubwcp);
  2334. /* enable all 4 interrupts */
  2335. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, true);
  2336. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, true);
  2337. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, true);
  2338. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, true);
  2339. /* Turn OFF until buffers are allocated */
  2340. if (ubwcp_power(ubwcp, false)) {
  2341. ret = -1;
  2342. goto err_power_off;
  2343. }
  2344. ret = msm_ubwcp_set_ops(ubwcp_init_buffer, ubwcp_free_buffer, ubwcp_lock, ubwcp_unlock);
  2345. if (ret) {
  2346. ERR("msm_ubwcp_set_ops() failed: %d, but IGNORED", ret);
  2347. /* TBD: ignore return error during testing phase.
  2348. * This allows us to rmmod/insmod for faster dev cycle.
  2349. * In final version: return error and de-register driver if set_ops fails.
  2350. */
  2351. ret = 0;
  2352. //goto err_power_off;
  2353. } else {
  2354. DBG("msm_ubwcp_set_ops(): success"); }
  2355. me = ubwcp;
  2356. return ret;
  2357. err_power_off:
  2358. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2359. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2360. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2361. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2362. err_pool_add:
  2363. gen_pool_destroy(ubwcp->ula_pool);
  2364. err_pool_create:
  2365. ubwcp_cdev_deinit(ubwcp);
  2366. return ret;
  2367. }
  2368. /* buffer context bank device probe */
  2369. static int ubwcp_probe_cb_buf(struct platform_device *pdev)
  2370. {
  2371. struct ubwcp_driver *ubwcp;
  2372. struct iommu_domain *domain = NULL;
  2373. FENTRY();
  2374. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2375. if (!ubwcp) {
  2376. ERR("failed to get ubwcp ptr");
  2377. return -EINVAL;
  2378. }
  2379. /* save the buffer cb device */
  2380. ubwcp->dev_buf_cb = &pdev->dev;
  2381. domain = iommu_get_domain_for_dev(ubwcp->dev_buf_cb);
  2382. if (domain)
  2383. iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_buf_cb);
  2384. return 0;
  2385. }
  2386. /* descriptor context bank device probe */
  2387. static int ubwcp_probe_cb_desc(struct platform_device *pdev)
  2388. {
  2389. int ret = 0;
  2390. struct ubwcp_driver *ubwcp;
  2391. struct iommu_domain *domain = NULL;
  2392. FENTRY();
  2393. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2394. if (!ubwcp) {
  2395. ERR("failed to get ubwcp ptr");
  2396. return -EINVAL;
  2397. }
  2398. ubwcp->buffer_desc_size = UBWCP_BUFFER_DESC_OFFSET *
  2399. UBWCP_BUFFER_DESC_COUNT;
  2400. ubwcp->dev_desc_cb = &pdev->dev;
  2401. dma_set_max_seg_size(ubwcp->dev_desc_cb, DMA_BIT_MASK(32));
  2402. dma_set_seg_boundary(ubwcp->dev_desc_cb, (unsigned long)DMA_BIT_MASK(64));
  2403. /* Allocate buffer descriptors. UBWCP is iocoherent device.
  2404. * Thus we don't need to flush after updates to buffer descriptors.
  2405. */
  2406. ubwcp->buffer_desc_base = dma_alloc_coherent(ubwcp->dev_desc_cb,
  2407. ubwcp->buffer_desc_size,
  2408. &ubwcp->buffer_desc_dma_handle,
  2409. GFP_KERNEL);
  2410. if (!ubwcp->buffer_desc_base) {
  2411. ERR("failed to allocate desc buffer");
  2412. return -ENOMEM;
  2413. }
  2414. DBG("desc_base = %p size = %zu", ubwcp->buffer_desc_base,
  2415. ubwcp->buffer_desc_size);
  2416. ret = ubwcp_power(ubwcp, true);
  2417. if (ret) {
  2418. ERR("failed to power on");
  2419. goto err;
  2420. }
  2421. ubwcp_hw_set_buf_desc(ubwcp->base, (u64) ubwcp->buffer_desc_dma_handle,
  2422. UBWCP_BUFFER_DESC_OFFSET);
  2423. ret = ubwcp_power(ubwcp, false);
  2424. if (ret) {
  2425. ERR("failed to power off");
  2426. goto err;
  2427. }
  2428. domain = iommu_get_domain_for_dev(ubwcp->dev_desc_cb);
  2429. if (domain)
  2430. iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_desc_cb);
  2431. return ret;
  2432. err:
  2433. dma_free_coherent(ubwcp->dev_desc_cb,
  2434. ubwcp->buffer_desc_size,
  2435. ubwcp->buffer_desc_base,
  2436. ubwcp->buffer_desc_dma_handle);
  2437. ubwcp->buffer_desc_base = NULL;
  2438. ubwcp->buffer_desc_dma_handle = 0;
  2439. ubwcp->dev_desc_cb = NULL;
  2440. return -1;
  2441. }
  2442. /* buffer context bank device remove */
  2443. static int ubwcp_remove_cb_buf(struct platform_device *pdev)
  2444. {
  2445. struct ubwcp_driver *ubwcp;
  2446. FENTRY();
  2447. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2448. if (!ubwcp) {
  2449. ERR("failed to get ubwcp ptr");
  2450. return -EINVAL;
  2451. }
  2452. /* remove buf_cb reference */
  2453. ubwcp->dev_buf_cb = NULL;
  2454. return 0;
  2455. }
  2456. /* descriptor context bank device remove */
  2457. static int ubwcp_remove_cb_desc(struct platform_device *pdev)
  2458. {
  2459. struct ubwcp_driver *ubwcp;
  2460. FENTRY();
  2461. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2462. if (!ubwcp) {
  2463. ERR("failed to get ubwcp ptr");
  2464. return -EINVAL;
  2465. }
  2466. if (!ubwcp->dev_desc_cb) {
  2467. ERR("ubwcp->dev_desc_cb == NULL");
  2468. return -1;
  2469. }
  2470. ubwcp_power(ubwcp, true);
  2471. ubwcp_hw_set_buf_desc(ubwcp->base, 0x0, 0x0);
  2472. ubwcp_power(ubwcp, false);
  2473. dma_free_coherent(ubwcp->dev_desc_cb,
  2474. ubwcp->buffer_desc_size,
  2475. ubwcp->buffer_desc_base,
  2476. ubwcp->buffer_desc_dma_handle);
  2477. ubwcp->buffer_desc_base = NULL;
  2478. ubwcp->buffer_desc_dma_handle = 0;
  2479. return 0;
  2480. }
  2481. /* ubwcp device remove */
  2482. static int qcom_ubwcp_remove(struct platform_device *pdev)
  2483. {
  2484. size_t avail;
  2485. size_t psize;
  2486. struct ubwcp_driver *ubwcp;
  2487. FENTRY();
  2488. /* get pdev->dev->driver_data = ubwcp */
  2489. ubwcp = platform_get_drvdata(pdev);
  2490. if (!ubwcp) {
  2491. ERR("ubwcp == NULL");
  2492. return -1;
  2493. }
  2494. ubwcp_power(ubwcp, true);
  2495. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2496. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2497. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2498. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2499. ubwcp_power(ubwcp, false);
  2500. /* before destroying, make sure pool is empty. otherwise pool_destroy() panics.
  2501. * TBD: remove this check for production code and let it panic
  2502. */
  2503. avail = gen_pool_avail(ubwcp->ula_pool);
  2504. psize = gen_pool_size(ubwcp->ula_pool);
  2505. if (psize != avail) {
  2506. ERR("gen_pool is not empty! avail: %zx size: %zx", avail, psize);
  2507. ERR("skipping pool destroy....cause it will PANIC. Fix this!!!!");
  2508. WARN(1, "Fix this!");
  2509. } else {
  2510. gen_pool_destroy(ubwcp->ula_pool);
  2511. }
  2512. ubwcp_debugfs_deinit(ubwcp);
  2513. ubwcp_cdev_deinit(ubwcp);
  2514. return 0;
  2515. }
  2516. /* top level ubwcp device probe function */
  2517. static int ubwcp_probe(struct platform_device *pdev)
  2518. {
  2519. const char *compatible = "";
  2520. FENTRY();
  2521. trace_ubwcp_probe(pdev);
  2522. if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
  2523. return qcom_ubwcp_probe(pdev);
  2524. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
  2525. return ubwcp_probe_cb_desc(pdev);
  2526. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
  2527. return ubwcp_probe_cb_buf(pdev);
  2528. of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
  2529. ERR("unknown device: %s", compatible);
  2530. WARN_ON(1);
  2531. return -EINVAL;
  2532. }
  2533. /* top level ubwcp device remove function */
  2534. static int ubwcp_remove(struct platform_device *pdev)
  2535. {
  2536. const char *compatible = "";
  2537. FENTRY();
  2538. trace_ubwcp_remove(pdev);
  2539. /* TBD: what if buffers are still allocated? locked? etc.
  2540. * also should turn off power?
  2541. */
  2542. if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
  2543. return qcom_ubwcp_remove(pdev);
  2544. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
  2545. return ubwcp_remove_cb_desc(pdev);
  2546. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
  2547. return ubwcp_remove_cb_buf(pdev);
  2548. of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
  2549. ERR("unknown device: %s", compatible);
  2550. WARN_ON(1);
  2551. return -EINVAL;
  2552. }
  2553. static const struct of_device_id ubwcp_dt_match[] = {
  2554. {.compatible = "qcom,ubwcp"},
  2555. {.compatible = "qcom,ubwcp-context-bank-desc"},
  2556. {.compatible = "qcom,ubwcp-context-bank-buf"},
  2557. {}
  2558. };
  2559. struct platform_driver ubwcp_platform_driver = {
  2560. .probe = ubwcp_probe,
  2561. .remove = ubwcp_remove,
  2562. .driver = {
  2563. .name = "qcom,ubwcp",
  2564. .of_match_table = ubwcp_dt_match,
  2565. },
  2566. };
  2567. int ubwcp_init(void)
  2568. {
  2569. int ret = 0;
  2570. DBG("+++++++++++");
  2571. ret = platform_driver_register(&ubwcp_platform_driver);
  2572. if (ret)
  2573. ERR("platform_driver_register() failed: %d", ret);
  2574. return ret;
  2575. }
  2576. void ubwcp_exit(void)
  2577. {
  2578. platform_driver_unregister(&ubwcp_platform_driver);
  2579. DBG("-----------");
  2580. }
  2581. module_init(ubwcp_init);
  2582. module_exit(ubwcp_exit);
  2583. MODULE_LICENSE("GPL");