ubwcp_main.c 90 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  4. */
  5. #define pr_fmt(fmt) "%s: %s(): " fmt, KBUILD_MODNAME, __func__
  6. #include <linux/module.h>
  7. #include <linux/kernel.h>
  8. #include <linux/dma-buf.h>
  9. #include <linux/slab.h>
  10. #include <linux/cdev.h>
  11. #include <linux/hashtable.h>
  12. #include <linux/scatterlist.h>
  13. #include <linux/types.h>
  14. #include <linux/platform_device.h>
  15. #include <linux/of.h>
  16. #include <linux/of_platform.h>
  17. #include <linux/of_address.h>
  18. #include <linux/genalloc.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/regulator/consumer.h>
  21. #include <linux/numa.h>
  22. #include <linux/memory_hotplug.h>
  23. #include <asm/page.h>
  24. #include <linux/delay.h>
  25. #include <linux/ubwcp_dma_heap.h>
  26. #include <linux/debugfs.h>
  27. #include <linux/clk.h>
  28. #include <linux/iommu.h>
  29. #include <linux/set_memory.h>
  30. #include <linux/range.h>
  31. #include <linux/qcom_scm.h>
  32. MODULE_IMPORT_NS(DMA_BUF);
  33. #include "include/kernel/ubwcp.h"
  34. #include "ubwcp_hw.h"
  35. #include "include/uapi/ubwcp_ioctl.h"
  36. #define CREATE_TRACE_POINTS
  37. #include "ubwcp_trace.h"
  38. #define UBWCP_NUM_DEVICES 1
  39. #define UBWCP_DEVICE_NAME "ubwcp"
  40. #define UBWCP_BUFFER_DESC_OFFSET 64
  41. #define UBWCP_BUFFER_DESC_COUNT 256
  42. #define CACHE_ADDR(x) ((x) >> 6)
  43. #define PAGE_ADDR(x) ((x) >> 12)
  44. #define UBWCP_ALIGN(_x, _y) ((((_x) + (_y) - 1)/(_y))*(_y))
  45. #define DBG_BUF_ATTR(fmt, args...) do { if (unlikely(ubwcp_debug_trace_enable)) \
  46. pr_err(fmt "\n", ##args); \
  47. } while (0)
  48. #define DBG(fmt, args...) do { if (unlikely(ubwcp_debug_trace_enable)) \
  49. pr_err(fmt "\n", ##args); \
  50. } while (0)
  51. #define ERR(fmt, args...) pr_err_ratelimited("%d: ~~~ERROR~~~: " fmt "\n", __LINE__, ##args)
  52. #define META_DATA_PITCH_ALIGN 64
  53. #define META_DATA_HEIGHT_ALIGN 16
  54. #define META_DATA_SIZE_ALIGN 4096
  55. #define PIXEL_DATA_SIZE_ALIGN 4096
  56. #define UBWCP_SYNC_GRANULE 0x4000000L /* 64 MB */
  57. /* Max values for attributes */
  58. #define MAX_ATTR_WIDTH (10*1024)
  59. #define MAX_ATTR_HEIGHT (10*1024)
  60. #define MAX_ATTR_STRIDE (64*1024)
  61. #define MAX_ATTR_PLANAR_PAD 4096
  62. #define MAX_ATTR_SCANLN_HT_DELTA (32*1024)
  63. enum ula_remove_mem_status {
  64. ULA_REMOVE_MEM_SUCCESS = 0,
  65. ULA_REMOVE_MEM_ABORTED = 1
  66. };
  67. struct ubwcp_desc {
  68. int idx;
  69. void *ptr;
  70. };
  71. struct tile_dimension {
  72. u16 width;
  73. u16 height;
  74. };
  75. struct ubwcp_plane_info {
  76. u16 pixel_bytes;
  77. u16 per_pixel;
  78. struct tile_dimension tilesize_p; /* pixels */
  79. struct tile_dimension macrotilesize_p; /* pixels */
  80. };
  81. struct ubwcp_image_format_info {
  82. u16 planes;
  83. struct ubwcp_plane_info p_info[2];
  84. };
  85. enum ubwcp_std_image_format {
  86. RGBA = 0,
  87. NV12 = 1,
  88. NV124R = 2,
  89. P010 = 3,
  90. TP10 = 4,
  91. P016 = 5,
  92. INFO_FORMAT_LIST_SIZE,
  93. };
  94. enum ubwcp_state {
  95. UBWCP_STATE_READY = 0,
  96. UBWCP_STATE_INVALID = -1,
  97. UBWCP_STATE_FAULT = -2,
  98. };
  99. struct ubwcp_prefetch_tgt_ctrl {
  100. atomic_t cpu_count;
  101. bool enable;
  102. int result;
  103. };
  104. struct ubwcp_driver {
  105. /* cdev related */
  106. dev_t devt;
  107. struct class *dev_class; //sysfs dev class
  108. struct device *dev_sys; //sysfs dev
  109. struct cdev cdev; //char dev
  110. /* debugfs */
  111. struct dentry *debugfs_root;
  112. bool read_err_irq_en;
  113. bool write_err_irq_en;
  114. bool decode_err_irq_en;
  115. bool encode_err_irq_en;
  116. bool single_tile_en;
  117. /* ubwcp devices */
  118. struct device *dev; //ubwcp device
  119. struct device *dev_desc_cb; //smmu dev for descriptors
  120. struct device *dev_buf_cb; //smmu dev for ubwcp buffers
  121. void __iomem *base; //ubwcp base address
  122. struct regulator *vdd;
  123. struct clk **clocks;
  124. int num_clocks;
  125. /* interrupts */
  126. int irq_range_ck_rd;
  127. int irq_range_ck_wr;
  128. int irq_encode;
  129. int irq_decode;
  130. /* ula address pool */
  131. u64 ula_pool_base;
  132. u64 ula_pool_size;
  133. struct gen_pool *ula_pool;
  134. configure_mmap mmap_config_fptr;
  135. /* HW version */
  136. u32 hw_ver_major;
  137. u32 hw_ver_minor;
  138. /* keep track of all potential buffers.
  139. * hash table index'ed using dma_buf ptr.
  140. * 2**13 = 8192 hash values
  141. */
  142. DECLARE_HASHTABLE(buf_table, 13);
  143. /* buffer descriptor */
  144. void *buffer_desc_base; /* CPU address */
  145. dma_addr_t buffer_desc_dma_handle; /* dma address */
  146. size_t buffer_desc_size;
  147. struct ubwcp_desc desc_list[UBWCP_BUFFER_DESC_COUNT];
  148. struct ubwcp_image_format_info format_info[INFO_FORMAT_LIST_SIZE];
  149. /* driver state */
  150. enum ubwcp_state state;
  151. atomic_t num_non_lin_buffers;
  152. bool mem_online;
  153. struct mutex desc_lock; /* allocate/free descriptors */
  154. spinlock_t buf_table_lock; /* add/remove dma_buf into list of managed bufffers */
  155. struct mutex mem_hotplug_lock; /* memory hotplug lock */
  156. struct mutex ula_lock; /* allocate/free ula */
  157. struct mutex ubwcp_flush_lock; /* ubwcp flush */
  158. struct mutex hw_range_ck_lock; /* range ck */
  159. struct list_head err_handler_list; /* error handler list */
  160. spinlock_t err_handler_list_lock; /* err_handler_list lock */
  161. struct dev_pagemap pgmap;
  162. /* power state tracking */
  163. int power_on;
  164. struct mutex power_ctrl_lock;
  165. struct ubwcp_prefetch_tgt_ctrl ctrl;
  166. };
  167. struct ubwcp_buf {
  168. struct hlist_node hnode;
  169. struct ubwcp_driver *ubwcp;
  170. struct ubwcp_buffer_attrs buf_attr;
  171. bool perm;
  172. struct ubwcp_desc *desc;
  173. bool buf_attr_set;
  174. enum dma_data_direction dma_dir;
  175. int lock_count;
  176. /* dma_buf info */
  177. struct dma_buf *dma_buf;
  178. struct dma_buf_attachment *attachment;
  179. struct sg_table *sgt;
  180. /* ula info */
  181. phys_addr_t ula_pa;
  182. size_t ula_size;
  183. /* meta metadata */
  184. struct ubwcp_hw_meta_metadata mmdata;
  185. struct mutex lock;
  186. };
  187. static struct ubwcp_driver *me;
  188. static u32 ubwcp_debug_trace_enable;
  189. static void prefetch_tgt_per_cpu(void *info)
  190. {
  191. int ret = 0;
  192. struct ubwcp_prefetch_tgt_ctrl *ctrl;
  193. ctrl = (struct ubwcp_prefetch_tgt_ctrl *) info;
  194. ret = qcom_scm_prefetch_tgt_ctrl(ctrl->enable);
  195. if (ret) {
  196. //ctrl->result = ret;
  197. //ERR("scm call failed, ret: %d enable: %d", ret, ctrl->enable);
  198. DBG("scm call failed, ret: %d missing the matching TZ?", ret);
  199. }
  200. atomic_dec(&ctrl->cpu_count);
  201. }
  202. /* Enable/disable generation of prefetch target opcode. smc call must be done from each core
  203. * to update the core specific register. Not thread-safe.
  204. */
  205. static int prefetch_tgt(struct ubwcp_driver *ubwcp, bool enable)
  206. {
  207. int cpu;
  208. trace_ubwcp_prefetch_tgt_start(enable);
  209. DBG("enable: %d", enable);
  210. ubwcp->ctrl.enable = enable;
  211. ubwcp->ctrl.result = 0;
  212. atomic_set(&ubwcp->ctrl.cpu_count, 0);
  213. cpus_read_lock();
  214. for_each_cpu(cpu, cpu_online_mask) {
  215. atomic_inc(&ubwcp->ctrl.cpu_count);
  216. smp_call_function_single(cpu, prefetch_tgt_per_cpu, (void *) &ubwcp->ctrl, false);
  217. }
  218. cpus_read_unlock();
  219. while (atomic_read(&ubwcp->ctrl.cpu_count))
  220. ;
  221. DBG("done");
  222. trace_ubwcp_prefetch_tgt_end(enable);
  223. return ubwcp->ctrl.result;
  224. }
  225. static struct ubwcp_driver *ubwcp_get_driver(void)
  226. {
  227. if (!me)
  228. WARN(1, "ubwcp: driver ptr requested but driver not initialized");
  229. return me;
  230. }
  231. static void image_format_init(struct ubwcp_driver *ubwcp)
  232. { /* planes, bytes/p, Tp , MTp */
  233. ubwcp->format_info[RGBA] = (struct ubwcp_image_format_info)
  234. {1, {{4, 1, {16, 4}, {64, 16}}}};
  235. ubwcp->format_info[NV12] = (struct ubwcp_image_format_info)
  236. {2, {{1, 1, {32, 8}, {128, 32}},
  237. {2, 1, {16, 8}, { 64, 32}}}};
  238. ubwcp->format_info[NV124R] = (struct ubwcp_image_format_info)
  239. {2, {{1, 1, {64, 4}, {256, 16}},
  240. {2, 1, {32, 4}, {128, 16}}}};
  241. ubwcp->format_info[P010] = (struct ubwcp_image_format_info)
  242. {2, {{2, 1, {32, 4}, {128, 16}},
  243. {4, 1, {16, 4}, { 64, 16}}}};
  244. ubwcp->format_info[TP10] = (struct ubwcp_image_format_info)
  245. {2, {{4, 3, {48, 4}, {192, 16}},
  246. {8, 3, {24, 4}, { 96, 16}}}};
  247. ubwcp->format_info[P016] = (struct ubwcp_image_format_info)
  248. {2, {{2, 1, {32, 4}, {128, 16}},
  249. {4, 1, {16, 4}, { 64, 16}}}};
  250. }
  251. static void ubwcp_buf_desc_list_init(struct ubwcp_driver *ubwcp)
  252. {
  253. int idx;
  254. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  255. for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
  256. desc_list[idx].idx = -1;
  257. desc_list[idx].ptr = NULL;
  258. }
  259. }
  260. static int ubwcp_init_clocks(struct ubwcp_driver *ubwcp, struct device *dev)
  261. {
  262. const char *cname;
  263. struct property *prop;
  264. int i;
  265. ubwcp->num_clocks =
  266. of_property_count_strings(dev->of_node, "clock-names");
  267. if (ubwcp->num_clocks < 1) {
  268. ubwcp->num_clocks = 0;
  269. return 0;
  270. }
  271. ubwcp->clocks = devm_kzalloc(dev,
  272. sizeof(*ubwcp->clocks) * ubwcp->num_clocks, GFP_KERNEL);
  273. if (!ubwcp->clocks)
  274. return -ENOMEM;
  275. i = 0;
  276. of_property_for_each_string(dev->of_node, "clock-names",
  277. prop, cname) {
  278. struct clk *c = devm_clk_get(dev, cname);
  279. if (IS_ERR(c)) {
  280. ERR("Couldn't get clock: %s\n", cname);
  281. return PTR_ERR(c);
  282. }
  283. ubwcp->clocks[i] = c;
  284. ++i;
  285. }
  286. return 0;
  287. }
  288. static int ubwcp_enable_clocks(struct ubwcp_driver *ubwcp)
  289. {
  290. int i, ret = 0;
  291. for (i = 0; i < ubwcp->num_clocks; ++i) {
  292. ret = clk_prepare_enable(ubwcp->clocks[i]);
  293. if (ret) {
  294. ERR("Couldn't enable clock #%d\n", i);
  295. while (i--)
  296. clk_disable_unprepare(ubwcp->clocks[i]);
  297. break;
  298. }
  299. }
  300. return ret;
  301. }
  302. static void ubwcp_disable_clocks(struct ubwcp_driver *ubwcp)
  303. {
  304. int i;
  305. for (i = ubwcp->num_clocks; i; --i)
  306. clk_disable_unprepare(ubwcp->clocks[i - 1]);
  307. }
  308. /* UBWCP Power control
  309. * Due to hw bug, ubwcp block cannot handle prefetch target opcode. Thus we disable the opcode
  310. * when ubwcp is powered on and enable it back when ubwcp is powered off.
  311. */
  312. static int ubwcp_power(struct ubwcp_driver *ubwcp, bool enable)
  313. {
  314. int ret = 0;
  315. mutex_lock(&ubwcp->power_ctrl_lock);
  316. if (enable) {
  317. ubwcp->power_on++;
  318. if (ubwcp->power_on != 1)
  319. goto done;
  320. } else {
  321. ubwcp->power_on--;
  322. if (ubwcp->power_on != 0)
  323. goto done;
  324. }
  325. if (enable) {
  326. ret = prefetch_tgt(ubwcp, 0);
  327. if (ret)
  328. goto done;
  329. ret = regulator_enable(ubwcp->vdd);
  330. if (ret) {
  331. ERR("regulator call (enable: %d) failed: %d", enable, ret);
  332. goto done;
  333. }
  334. ret = ubwcp_enable_clocks(ubwcp);
  335. if (ret) {
  336. ERR("enable clocks failed: %d", ret);
  337. regulator_disable(ubwcp->vdd);
  338. goto done;
  339. }
  340. } else {
  341. ret = regulator_disable(ubwcp->vdd);
  342. if (ret) {
  343. ERR("regulator call (enable: %d) failed: %d", enable, ret);
  344. goto done;
  345. }
  346. ubwcp_disable_clocks(ubwcp);
  347. ret = prefetch_tgt(ubwcp, 1);
  348. }
  349. done:
  350. mutex_unlock(&ubwcp->power_ctrl_lock);
  351. return ret;
  352. }
  353. /* get ubwcp_buf corresponding to the given dma_buf */
  354. static struct ubwcp_buf *dma_buf_to_ubwcp_buf(struct dma_buf *dmabuf)
  355. {
  356. struct ubwcp_buf *buf = NULL;
  357. struct ubwcp_buf *ret_buf = NULL;
  358. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  359. unsigned long flags;
  360. if (!dmabuf || !ubwcp)
  361. return NULL;
  362. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  363. /* look up ubwcp_buf corresponding to this dma_buf */
  364. hash_for_each_possible(ubwcp->buf_table, buf, hnode, (u64)dmabuf) {
  365. if (buf->dma_buf == dmabuf) {
  366. ret_buf = buf;
  367. break;
  368. }
  369. }
  370. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  371. return ret_buf;
  372. }
  373. /* return ubwcp hardware version */
  374. int ubwcp_get_hw_version(struct ubwcp_ioctl_hw_version *ver)
  375. {
  376. struct ubwcp_driver *ubwcp;
  377. if (!ver) {
  378. ERR("invalid version ptr");
  379. return -EINVAL;
  380. }
  381. ubwcp = ubwcp_get_driver();
  382. if (!ubwcp)
  383. return -1;
  384. if (ubwcp->state == UBWCP_STATE_INVALID)
  385. return -EPERM;
  386. ver->major = ubwcp->hw_ver_major;
  387. ver->minor = ubwcp->hw_ver_minor;
  388. return 0;
  389. }
  390. EXPORT_SYMBOL(ubwcp_get_hw_version);
  391. static int ula_add_mem(struct ubwcp_driver *ubwcp)
  392. {
  393. int ret = 0;
  394. int nid;
  395. void *ptr;
  396. nid = memory_add_physaddr_to_nid(ubwcp->ula_pool_base);
  397. DBG("calling memremap_pages()...");
  398. ubwcp->pgmap.type = MEMORY_DEVICE_GENERIC;
  399. ubwcp->pgmap.nr_range = 1;
  400. ubwcp->pgmap.range.start = ubwcp->ula_pool_base;
  401. ubwcp->pgmap.range.end = ubwcp->ula_pool_base + ubwcp->ula_pool_size - 1;
  402. trace_ubwcp_memremap_pages_start(ubwcp->ula_pool_size);
  403. ptr = memremap_pages(&ubwcp->pgmap, nid);
  404. trace_ubwcp_memremap_pages_end(ubwcp->ula_pool_size);
  405. if (IS_ERR(ptr)) {
  406. ret = IS_ERR(ptr);
  407. ERR("memremap_pages() failed st:0x%lx sz:0x%lx err: %d",
  408. ubwcp->ula_pool_base,
  409. ubwcp->ula_pool_size,
  410. ret);
  411. } else {
  412. DBG("memremap_pages() ula_pool_base:0x%llx, size:0x%zx, kernel addr:0x%p",
  413. ubwcp->ula_pool_base,
  414. ubwcp->ula_pool_size,
  415. page_to_virt(pfn_to_page(PFN_DOWN(ubwcp->ula_pool_base))));
  416. }
  417. return ret;
  418. }
  419. static int ula_map_uncached(u64 base, u64 size)
  420. {
  421. int ret;
  422. trace_ubwcp_set_direct_map_range_uncached_start(size);
  423. ret = set_direct_map_range_uncached((unsigned long)phys_to_virt(base), size >> PAGE_SHIFT);
  424. trace_ubwcp_set_direct_map_range_uncached_end(size);
  425. if (ret)
  426. ERR("set_direct_map_range_uncached failed st:0x%lx num pages:%lu err: %d",
  427. base, size >> PAGE_SHIFT, ret);
  428. return ret;
  429. }
  430. static void ula_unmap(struct ubwcp_driver *ubwcp)
  431. {
  432. DBG("Calling memunmap_pages() for ULA PA pool");
  433. trace_ubwcp_memunmap_pages_start(ubwcp->ula_pool_size);
  434. memunmap_pages(&ubwcp->pgmap);
  435. trace_ubwcp_memunmap_pages_end(ubwcp->ula_pool_size);
  436. }
  437. static void ula_sync_for_cpu(struct device *dev, u64 addr, unsigned long size)
  438. {
  439. trace_ubwcp_dma_sync_single_for_cpu_start(size, DMA_BIDIRECTIONAL);
  440. dma_sync_single_for_cpu(dev, addr, size, DMA_BIDIRECTIONAL);
  441. trace_ubwcp_dma_sync_single_for_cpu_end(size, DMA_BIDIRECTIONAL);
  442. }
  443. /** Remove ula memory in chunks
  444. * Abort if new buffer addition is detected
  445. * If remove succeeds or aborted, return success
  446. * status value indicates if mem was removed or aborted (not removed)
  447. * Otherwise return failure
  448. */
  449. static int ula_remove_mem(struct ubwcp_driver *ubwcp, enum ula_remove_mem_status *status)
  450. {
  451. int ret = 0;
  452. unsigned long sync_remain = ubwcp->ula_pool_size;
  453. unsigned long sync_offset = 0;
  454. unsigned long sync_size = 0;
  455. ret = ula_map_uncached(ubwcp->ula_pool_base, ubwcp->ula_pool_size);
  456. if (ret)
  457. return ret;
  458. trace_ubwcp_offline_sync_start(ubwcp->ula_pool_size);
  459. while (sync_remain > 0) {
  460. if (atomic_read(&ubwcp->num_non_lin_buffers) > 0) {
  461. trace_ubwcp_offline_sync_end(ubwcp->ula_pool_size);
  462. ula_unmap(ubwcp);
  463. if (ula_add_mem(ubwcp)) {
  464. ERR("remove mem: failed to add back during abort");
  465. return -1;
  466. }
  467. *status = ULA_REMOVE_MEM_ABORTED;
  468. return 0;
  469. }
  470. if (UBWCP_SYNC_GRANULE > sync_remain) {
  471. sync_size = sync_remain;
  472. sync_remain = 0;
  473. } else {
  474. sync_size = UBWCP_SYNC_GRANULE;
  475. sync_remain -= UBWCP_SYNC_GRANULE;
  476. }
  477. ula_sync_for_cpu(ubwcp->dev, ubwcp->ula_pool_base + sync_offset, sync_size);
  478. sync_offset += sync_size;
  479. }
  480. trace_ubwcp_offline_sync_end(ubwcp->ula_pool_size);
  481. ula_unmap(ubwcp);
  482. *status = ULA_REMOVE_MEM_SUCCESS;
  483. return 0;
  484. }
  485. static int inc_num_non_lin_buffers(struct ubwcp_driver *ubwcp)
  486. {
  487. atomic_inc(&ubwcp->num_non_lin_buffers);
  488. mutex_lock(&ubwcp->mem_hotplug_lock);
  489. if (!ubwcp->mem_online) {
  490. if (atomic_read(&ubwcp->num_non_lin_buffers) == 0) {
  491. ERR("Bad state: num_non_lin_buffers should not be 0");
  492. goto err;
  493. }
  494. if (ubwcp_power(ubwcp, true))
  495. goto err;
  496. if (ula_add_mem(ubwcp))
  497. goto err_add_memory;
  498. ubwcp->mem_online = true;
  499. }
  500. mutex_unlock(&ubwcp->mem_hotplug_lock);
  501. return 0;
  502. err_add_memory:
  503. ubwcp_power(ubwcp, false);
  504. err:
  505. atomic_dec(&ubwcp->num_non_lin_buffers);
  506. mutex_unlock(&ubwcp->mem_hotplug_lock);
  507. ubwcp->state = UBWCP_STATE_FAULT;
  508. ERR("state set to fault");
  509. return -1;
  510. }
  511. static int dec_num_non_lin_buffers(struct ubwcp_driver *ubwcp)
  512. {
  513. int ret;
  514. enum ula_remove_mem_status remove_status;
  515. atomic_dec(&ubwcp->num_non_lin_buffers);
  516. mutex_lock(&ubwcp->mem_hotplug_lock);
  517. if (atomic_read(&ubwcp->num_non_lin_buffers) == 0) {
  518. DBG("last buffer: ~~~~~~~~~~~");
  519. if (!ubwcp->mem_online) {
  520. ERR("Bad state: mem_online should not be false");
  521. goto err;
  522. }
  523. ret = ula_remove_mem(ubwcp, &remove_status);
  524. if (ret)
  525. goto err;
  526. if (remove_status == ULA_REMOVE_MEM_SUCCESS) {
  527. ubwcp->mem_online = false;
  528. if (ubwcp_power(ubwcp, false))
  529. goto err;
  530. } else if (remove_status == ULA_REMOVE_MEM_ABORTED) {
  531. DBG("ula memory offline aborted");
  532. } else {
  533. ERR("unexpected ula remove status: %d", remove_status);
  534. goto err;
  535. }
  536. }
  537. mutex_unlock(&ubwcp->mem_hotplug_lock);
  538. return 0;
  539. err:
  540. atomic_inc(&ubwcp->num_non_lin_buffers);
  541. mutex_unlock(&ubwcp->mem_hotplug_lock);
  542. ubwcp->state = UBWCP_STATE_FAULT;
  543. ERR("state set to fault");
  544. return -1;
  545. }
  546. /**
  547. *
  548. * Initialize ubwcp buffer for the given dma_buf. This
  549. * initializes ubwcp internal data structures and possibly hw to
  550. * use ubwcp for this buffer.
  551. *
  552. * @param dmabuf : ptr to the buffer to be configured for ubwcp
  553. *
  554. * @return int : 0 on success, otherwise error code
  555. */
  556. static int ubwcp_init_buffer(struct dma_buf *dmabuf)
  557. {
  558. struct ubwcp_buf *buf;
  559. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  560. unsigned long flags;
  561. trace_ubwcp_init_buffer_start(dmabuf);
  562. if (!ubwcp) {
  563. trace_ubwcp_init_buffer_end(dmabuf);
  564. return -1;
  565. }
  566. if (ubwcp->state != UBWCP_STATE_READY) {
  567. ERR("driver in invalid state: %d", ubwcp->state);
  568. trace_ubwcp_init_buffer_end(dmabuf);
  569. return -EPERM;
  570. }
  571. if (!dmabuf) {
  572. ERR("NULL dmabuf input ptr");
  573. trace_ubwcp_init_buffer_end(dmabuf);
  574. return -EINVAL;
  575. }
  576. if (dma_buf_to_ubwcp_buf(dmabuf)) {
  577. ERR("dma_buf already initialized for ubwcp");
  578. trace_ubwcp_init_buffer_end(dmabuf);
  579. return -EEXIST;
  580. }
  581. buf = kzalloc(sizeof(*buf), GFP_KERNEL);
  582. if (!buf) {
  583. ERR("failed to alloc for new ubwcp_buf");
  584. trace_ubwcp_init_buffer_end(dmabuf);
  585. return -ENOMEM;
  586. }
  587. mutex_init(&buf->lock);
  588. buf->dma_buf = dmabuf;
  589. buf->ubwcp = ubwcp;
  590. buf->buf_attr.image_format = UBWCP_LINEAR;
  591. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  592. hash_add(ubwcp->buf_table, &buf->hnode, (u64)buf->dma_buf);
  593. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  594. trace_ubwcp_init_buffer_end(dmabuf);
  595. return 0;
  596. }
  597. static void dump_attributes(struct ubwcp_buffer_attrs *attr)
  598. {
  599. DBG_BUF_ATTR("");
  600. DBG_BUF_ATTR("image_format: %d", attr->image_format);
  601. DBG_BUF_ATTR("major_ubwc_ver: %d", attr->major_ubwc_ver);
  602. DBG_BUF_ATTR("minor_ubwc_ver: %d", attr->minor_ubwc_ver);
  603. DBG_BUF_ATTR("compression_type: %d", attr->compression_type);
  604. DBG_BUF_ATTR("lossy_params: %llu", attr->lossy_params);
  605. DBG_BUF_ATTR("width: %d", attr->width);
  606. DBG_BUF_ATTR("height: %d", attr->height);
  607. DBG_BUF_ATTR("stride: %d", attr->stride);
  608. DBG_BUF_ATTR("scanlines: %d", attr->scanlines);
  609. DBG_BUF_ATTR("planar_padding: %d", attr->planar_padding);
  610. DBG_BUF_ATTR("subsample: %d", attr->subsample);
  611. DBG_BUF_ATTR("sub_system_target: %d", attr->sub_system_target);
  612. DBG_BUF_ATTR("y_offset: %d", attr->y_offset);
  613. DBG_BUF_ATTR("batch_size: %d", attr->batch_size);
  614. DBG_BUF_ATTR("");
  615. }
  616. static int to_std_format(u16 ioctl_image_format, enum ubwcp_std_image_format *format)
  617. {
  618. switch (ioctl_image_format) {
  619. case UBWCP_RGBA8888:
  620. *format = RGBA;
  621. return 0;
  622. case UBWCP_NV12:
  623. case UBWCP_NV12_Y:
  624. case UBWCP_NV12_UV:
  625. *format = NV12;
  626. return 0;
  627. case UBWCP_NV124R:
  628. case UBWCP_NV124R_Y:
  629. case UBWCP_NV124R_UV:
  630. *format = NV124R;
  631. return 0;
  632. case UBWCP_TP10:
  633. case UBWCP_TP10_Y:
  634. case UBWCP_TP10_UV:
  635. *format = TP10;
  636. return 0;
  637. case UBWCP_P010:
  638. case UBWCP_P010_Y:
  639. case UBWCP_P010_UV:
  640. *format = P010;
  641. return 0;
  642. case UBWCP_P016:
  643. case UBWCP_P016_Y:
  644. case UBWCP_P016_UV:
  645. *format = P016;
  646. return 0;
  647. default:
  648. ERR("Failed to convert ioctl image format to std format: %d", ioctl_image_format);
  649. return -1;
  650. }
  651. }
  652. static int std_to_hw_img_fmt(enum ubwcp_std_image_format format, u16 *hw_fmt)
  653. {
  654. switch (format) {
  655. case RGBA:
  656. *hw_fmt = HW_BUFFER_FORMAT_RGBA;
  657. return 0;
  658. case NV12:
  659. *hw_fmt = HW_BUFFER_FORMAT_NV12;
  660. return 0;
  661. case NV124R:
  662. *hw_fmt = HW_BUFFER_FORMAT_NV124R;
  663. return 0;
  664. case P010:
  665. *hw_fmt = HW_BUFFER_FORMAT_P010;
  666. return 0;
  667. case TP10:
  668. *hw_fmt = HW_BUFFER_FORMAT_TP10;
  669. return 0;
  670. case P016:
  671. *hw_fmt = HW_BUFFER_FORMAT_P016;
  672. return 0;
  673. default:
  674. ERR("Failed to convert std image format to hw format: %d", format);
  675. return -1;
  676. }
  677. }
  678. static int get_stride_alignment(enum ubwcp_std_image_format format, u16 *align)
  679. {
  680. switch (format) {
  681. case TP10:
  682. *align = 64;
  683. return 0;
  684. case NV12:
  685. *align = 128;
  686. return 0;
  687. case RGBA:
  688. case NV124R:
  689. case P010:
  690. case P016:
  691. *align = 256;
  692. return 0;
  693. default:
  694. return -1;
  695. }
  696. }
  697. /* returns stride of compressed image */
  698. static u32 get_compressed_stride(struct ubwcp_driver *ubwcp,
  699. enum ubwcp_std_image_format format, u32 width)
  700. {
  701. struct ubwcp_plane_info p_info;
  702. u16 macro_tile_width_p;
  703. u16 pixel_bytes;
  704. u16 per_pixel;
  705. p_info = ubwcp->format_info[format].p_info[0];
  706. macro_tile_width_p = p_info.macrotilesize_p.width;
  707. pixel_bytes = p_info.pixel_bytes;
  708. per_pixel = p_info.per_pixel;
  709. return UBWCP_ALIGN(width, macro_tile_width_p)*pixel_bytes/per_pixel;
  710. }
  711. static void
  712. ubwcp_pixel_to_bytes(struct ubwcp_driver *ubwcp,
  713. enum ubwcp_std_image_format format,
  714. u32 width_p, u32 height_p,
  715. u32 *width_b, u32 *height_b)
  716. {
  717. u16 pixel_bytes;
  718. u16 per_pixel;
  719. struct ubwcp_image_format_info f_info;
  720. struct ubwcp_plane_info p_info;
  721. f_info = ubwcp->format_info[format];
  722. p_info = f_info.p_info[0];
  723. pixel_bytes = p_info.pixel_bytes;
  724. per_pixel = p_info.per_pixel;
  725. *width_b = (width_p*pixel_bytes)/per_pixel;
  726. *height_b = (height_p*pixel_bytes)/per_pixel;
  727. }
  728. /* check if linear stride conforms to hw limitations
  729. * always returns false for linear image
  730. */
  731. static bool stride_is_valid(struct ubwcp_driver *ubwcp,
  732. enum ubwcp_std_image_format format, u32 width, u32 lin_stride)
  733. {
  734. u32 compressed_stride;
  735. u32 width_b;
  736. u32 height_b;
  737. ubwcp_pixel_to_bytes(ubwcp, format, width, 0, &width_b, &height_b);
  738. if ((lin_stride < width_b) || (lin_stride > MAX_ATTR_STRIDE)) {
  739. ERR("Invalid stride: %u width: %u width_b: %u", lin_stride, width, width_b);
  740. return false;
  741. }
  742. if (format == TP10) {
  743. if(!IS_ALIGNED(lin_stride, 64)) {
  744. ERR("stride must be aligned to 64: %d", lin_stride);
  745. return false;
  746. }
  747. } else {
  748. compressed_stride = get_compressed_stride(ubwcp, format, width);
  749. if (lin_stride != compressed_stride) {
  750. ERR("linear stride: %d must be same as compressed stride: %d",
  751. lin_stride, compressed_stride);
  752. return false;
  753. }
  754. }
  755. return true;
  756. }
  757. static bool ioctl_format_is_valid(u16 ioctl_image_format)
  758. {
  759. switch (ioctl_image_format) {
  760. case UBWCP_LINEAR:
  761. case UBWCP_RGBA8888:
  762. case UBWCP_NV12:
  763. case UBWCP_NV12_Y:
  764. case UBWCP_NV12_UV:
  765. case UBWCP_NV124R:
  766. case UBWCP_NV124R_Y:
  767. case UBWCP_NV124R_UV:
  768. case UBWCP_TP10:
  769. case UBWCP_TP10_Y:
  770. case UBWCP_TP10_UV:
  771. case UBWCP_P010:
  772. case UBWCP_P010_Y:
  773. case UBWCP_P010_UV:
  774. case UBWCP_P016:
  775. case UBWCP_P016_Y:
  776. case UBWCP_P016_UV:
  777. return true;
  778. default:
  779. return false;
  780. }
  781. }
  782. /* validate buffer attributes */
  783. static bool ubwcp_buf_attrs_valid(struct ubwcp_driver *ubwcp, struct ubwcp_buffer_attrs *attr)
  784. {
  785. enum ubwcp_std_image_format format;
  786. if (attr->unused1 || attr->unused2 || attr->unused3 || attr->unused4 || attr->unused5 ||
  787. attr->unused6 || attr->unused7 || attr->unused8 || attr->unused9) {
  788. ERR("buf attr unused values must be set to 0");
  789. goto err;
  790. }
  791. if (!attr->width || !attr->height || !attr->stride || !attr->scanlines) {
  792. ERR("width/height/stride/scanlines cannot be 0");
  793. goto err;
  794. }
  795. if (!ioctl_format_is_valid(attr->image_format)) {
  796. ERR("invalid image format: %d", attr->image_format);
  797. goto err;
  798. }
  799. /* rest of the fields are ignored for linear format */
  800. if (attr->image_format == UBWCP_LINEAR) {
  801. goto valid;
  802. }
  803. if (to_std_format(attr->image_format, &format))
  804. goto err;
  805. if (attr->major_ubwc_ver || attr->minor_ubwc_ver) {
  806. ERR("major/minor ubwc ver must be 0. major: %d minor: %d",
  807. attr->major_ubwc_ver, attr->minor_ubwc_ver);
  808. goto err;
  809. }
  810. if (attr->compression_type != UBWCP_COMPRESSION_LOSSLESS) {
  811. ERR("compression_type is not valid: %d",
  812. attr->compression_type);
  813. goto err;
  814. }
  815. if (attr->lossy_params != 0) {
  816. ERR("lossy_params is not valid: %d", attr->lossy_params);
  817. goto err;
  818. }
  819. if (attr->width > MAX_ATTR_WIDTH) {
  820. ERR("width is invalid (above upper limit): %d", attr->width);
  821. goto err;
  822. }
  823. if (attr->height > MAX_ATTR_HEIGHT) {
  824. ERR("height is invalid (above upper limit): %d", attr->height);
  825. goto err;
  826. }
  827. if(!stride_is_valid(ubwcp, format, attr->width, attr->stride)) {
  828. ERR("stride is invalid: %d", attr->stride);
  829. goto err;
  830. }
  831. if ((attr->scanlines < attr->height) ||
  832. (attr->scanlines > attr->height + MAX_ATTR_SCANLN_HT_DELTA)) {
  833. ERR("scanlines is not valid - height: %d scanlines: %d",
  834. attr->height, attr->scanlines);
  835. goto err;
  836. }
  837. if (attr->planar_padding > MAX_ATTR_PLANAR_PAD) {
  838. ERR("planar_padding is not valid: %d", attr->planar_padding);
  839. goto err;
  840. }
  841. if (attr->subsample != UBWCP_SUBSAMPLE_4_2_0) {
  842. ERR("subsample is not valid: %d", attr->subsample);
  843. goto err;
  844. }
  845. if (attr->sub_system_target & ~UBWCP_SUBSYSTEM_TARGET_CPU) {
  846. ERR("sub_system_target other that CPU is not supported: %d",
  847. attr->sub_system_target);
  848. goto err;
  849. }
  850. if (!(attr->sub_system_target & UBWCP_SUBSYSTEM_TARGET_CPU)) {
  851. ERR("sub_system_target is not set to CPU: %d",
  852. attr->sub_system_target);
  853. goto err;
  854. }
  855. if (attr->y_offset != 0) {
  856. ERR("y_offset is not valid: %d", attr->y_offset);
  857. goto err;
  858. }
  859. if (attr->batch_size != 1) {
  860. ERR("batch_size is not valid: %d", attr->batch_size);
  861. goto err;
  862. }
  863. valid:
  864. dump_attributes(attr);
  865. return true;
  866. err:
  867. dump_attributes(attr);
  868. return false;
  869. }
  870. /* calculate and return metadata buffer size for a given plane
  871. * and buffer attributes
  872. */
  873. static int metadata_buf_sz(struct ubwcp_driver *ubwcp,
  874. enum ubwcp_std_image_format format,
  875. u32 width, u32 height, u8 plane, size_t *size)
  876. {
  877. u64 pitch;
  878. u64 lines;
  879. u64 tile_width;
  880. u32 tile_height;
  881. struct ubwcp_image_format_info f_info;
  882. struct ubwcp_plane_info p_info;
  883. f_info = ubwcp->format_info[format];
  884. DBG_BUF_ATTR("");
  885. DBG_BUF_ATTR("");
  886. DBG_BUF_ATTR("Calculating metadata buffer size: format = %d, plane = %d", format, plane);
  887. if (plane >= f_info.planes) {
  888. ERR("Missing plane info: format: %d, plane: %d", format, plane);
  889. return -1;
  890. }
  891. p_info = f_info.p_info[plane];
  892. /* UV plane */
  893. if (plane == 1) {
  894. width = width/2;
  895. height = height/2;
  896. }
  897. tile_width = p_info.tilesize_p.width;
  898. tile_height = p_info.tilesize_p.height;
  899. /* pitch: # of tiles in a row
  900. * lines: # of tile rows
  901. */
  902. pitch = UBWCP_ALIGN((width + tile_width - 1)/tile_width, META_DATA_PITCH_ALIGN);
  903. lines = UBWCP_ALIGN((height + tile_height - 1)/tile_height, META_DATA_HEIGHT_ALIGN);
  904. DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
  905. DBG_BUF_ATTR("tile params : %d x %d (pixels)", tile_width, tile_height);
  906. DBG_BUF_ATTR("pitch : %d (%d)", pitch, width/tile_width);
  907. DBG_BUF_ATTR("lines : %d (%d)", lines, height);
  908. DBG_BUF_ATTR("size (p*l*bytes) : %d", pitch*lines*1);
  909. /* x1 below is only to clarify that we are multiplying by 1 bytes/tile */
  910. *size = UBWCP_ALIGN(pitch*lines*1, META_DATA_SIZE_ALIGN);
  911. DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", *size, *size);
  912. return 0;
  913. }
  914. /* calculate and return size of pixel data buffer for a given plane
  915. * and buffer attributes
  916. */
  917. static int pixeldata_buf_sz(struct ubwcp_driver *ubwcp,
  918. u16 format, u32 width,
  919. u32 height, u8 plane, size_t *size)
  920. {
  921. u64 pitch;
  922. u64 lines;
  923. u16 pixel_bytes;
  924. u16 per_pixel;
  925. u64 macro_tile_width_p;
  926. u64 macro_tile_height_p;
  927. struct ubwcp_image_format_info f_info;
  928. struct ubwcp_plane_info p_info;
  929. f_info = ubwcp->format_info[format];
  930. DBG_BUF_ATTR("");
  931. DBG_BUF_ATTR("");
  932. DBG_BUF_ATTR("Calculating Pixeldata buffer size: format = %d, plane = %d", format, plane);
  933. if (plane >= f_info.planes) {
  934. ERR("Missing plane info: format: %d, plane: %d", format, plane);
  935. return -1;
  936. }
  937. p_info = f_info.p_info[plane];
  938. pixel_bytes = p_info.pixel_bytes;
  939. per_pixel = p_info.per_pixel;
  940. /* UV plane */
  941. if (plane == 1) {
  942. width = width/2;
  943. height = height/2;
  944. }
  945. macro_tile_width_p = p_info.macrotilesize_p.width;
  946. macro_tile_height_p = p_info.macrotilesize_p.height;
  947. /* align pixel width and height macro tile width and height */
  948. pitch = UBWCP_ALIGN(width, macro_tile_width_p);
  949. lines = UBWCP_ALIGN(height, macro_tile_height_p);
  950. DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
  951. DBG_BUF_ATTR("macro tile params: %d x %d (pixels)", macro_tile_width_p,
  952. macro_tile_height_p);
  953. DBG_BUF_ATTR("bytes_per_pixel : %d/%d", pixel_bytes, per_pixel);
  954. DBG_BUF_ATTR("pitch : %d", pitch);
  955. DBG_BUF_ATTR("lines : %d", lines);
  956. DBG_BUF_ATTR("size (p*l*bytes) : %d", (pitch*lines*pixel_bytes)/per_pixel);
  957. *size = UBWCP_ALIGN((pitch*lines*pixel_bytes)/per_pixel, PIXEL_DATA_SIZE_ALIGN);
  958. DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", *size, *size);
  959. return 0;
  960. }
  961. static int get_tile_height(struct ubwcp_driver *ubwcp, enum ubwcp_std_image_format format,
  962. u8 plane)
  963. {
  964. struct ubwcp_image_format_info f_info;
  965. struct ubwcp_plane_info p_info;
  966. f_info = ubwcp->format_info[format];
  967. p_info = f_info.p_info[plane];
  968. return p_info.tilesize_p.height;
  969. }
  970. /*
  971. * plane: must be 0 or 1 (1st plane == 0, 2nd plane == 1)
  972. */
  973. static size_t ubwcp_ula_size(struct ubwcp_driver *ubwcp, u16 format,
  974. u32 stride_b, u32 scanlines, u8 plane,
  975. bool add_tile_pad)
  976. {
  977. size_t size;
  978. DBG_BUF_ATTR("%s(format = %d, plane = %d)", __func__, format, plane);
  979. /* UV plane */
  980. if (plane == 1)
  981. scanlines = scanlines/2;
  982. if (add_tile_pad) {
  983. int tile_height = get_tile_height(ubwcp, format, plane);
  984. /* Align plane size to plane tile height */
  985. scanlines = ((scanlines + tile_height - 1) / tile_height) * tile_height;
  986. }
  987. size = stride_b*scanlines;
  988. DBG_BUF_ATTR("Size of plane-%u: (%u * %u) = %zu (0x%zx)",
  989. plane, stride_b, scanlines, size, size);
  990. return size;
  991. }
  992. static int missing_plane_from_format(u16 ioctl_image_format)
  993. {
  994. int missing_plane;
  995. switch (ioctl_image_format) {
  996. case UBWCP_NV12_Y:
  997. missing_plane = 2;
  998. break;
  999. case UBWCP_NV12_UV:
  1000. missing_plane = 1;
  1001. break;
  1002. case UBWCP_NV124R_Y:
  1003. missing_plane = 2;
  1004. break;
  1005. case UBWCP_NV124R_UV:
  1006. missing_plane = 1;
  1007. break;
  1008. case UBWCP_TP10_Y:
  1009. missing_plane = 2;
  1010. break;
  1011. case UBWCP_TP10_UV:
  1012. missing_plane = 1;
  1013. break;
  1014. case UBWCP_P010_Y:
  1015. missing_plane = 2;
  1016. break;
  1017. case UBWCP_P010_UV:
  1018. missing_plane = 1;
  1019. break;
  1020. case UBWCP_P016_Y:
  1021. missing_plane = 2;
  1022. break;
  1023. case UBWCP_P016_UV:
  1024. missing_plane = 1;
  1025. break;
  1026. default:
  1027. missing_plane = 0;
  1028. }
  1029. return missing_plane;
  1030. }
  1031. static int planes_in_format(enum ubwcp_std_image_format format)
  1032. {
  1033. if (format == RGBA)
  1034. return 1;
  1035. else
  1036. return 2;
  1037. }
  1038. static int ubwcp_validate_uv_align(struct ubwcp_driver *ubwcp,
  1039. struct ubwcp_buffer_attrs *attr,
  1040. size_t ula_y_plane_size,
  1041. size_t uv_start_offset)
  1042. {
  1043. int ret = 0;
  1044. size_t ula_y_plane_size_align;
  1045. size_t y_tile_align_bytes;
  1046. int y_tile_height;
  1047. int planes;
  1048. enum ubwcp_std_image_format format;
  1049. ret = to_std_format(attr->image_format, &format);
  1050. if (ret)
  1051. goto err;
  1052. /* Only validate UV align if there is both a Y and UV plane */
  1053. planes = planes_in_format(format);
  1054. if (planes != 2)
  1055. return 0;
  1056. /* Check it is cache line size aligned */
  1057. if ((uv_start_offset % 64) != 0) {
  1058. ret = -EINVAL;
  1059. ERR("uv_start_offset %zu not cache line aligned",
  1060. uv_start_offset);
  1061. goto err;
  1062. }
  1063. /*
  1064. * Check that UV plane does not overlap with any of the Y plane’s tiles
  1065. */
  1066. y_tile_height = get_tile_height(ubwcp, format, 0);
  1067. y_tile_align_bytes = y_tile_height * attr->stride;
  1068. ula_y_plane_size_align = ((ula_y_plane_size + y_tile_align_bytes - 1) /
  1069. y_tile_align_bytes) * y_tile_align_bytes;
  1070. if (uv_start_offset < ula_y_plane_size_align) {
  1071. ret = -EINVAL;
  1072. ERR("uv offset %zu less than y plane align %zu for y plane size %zu",
  1073. uv_start_offset, ula_y_plane_size_align,
  1074. ula_y_plane_size);
  1075. goto err;
  1076. }
  1077. return 0;
  1078. err:
  1079. return ret;
  1080. }
  1081. /* calculate ULA buffer parms */
  1082. static int ubwcp_calc_ula_params(struct ubwcp_driver *ubwcp,
  1083. struct ubwcp_buffer_attrs *attr,
  1084. size_t *ula_size,
  1085. size_t *ula_y_plane_size,
  1086. size_t *uv_start_offset)
  1087. {
  1088. size_t size;
  1089. enum ubwcp_std_image_format format;
  1090. int planes;
  1091. int missing_plane;
  1092. u32 stride;
  1093. u32 scanlines;
  1094. u32 planar_padding;
  1095. int ret;
  1096. ret = to_std_format(attr->image_format, &format);
  1097. if (ret)
  1098. return ret;
  1099. stride = attr->stride;
  1100. scanlines = attr->scanlines;
  1101. planar_padding = attr->planar_padding;
  1102. /* Number of "expected" planes in "the standard defined" image format */
  1103. planes = planes_in_format(format);
  1104. missing_plane = missing_plane_from_format(attr->image_format);
  1105. DBG_BUF_ATTR("ula params -->");
  1106. DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
  1107. DBG_BUF_ATTR("planes_in_format : %d", planes);
  1108. DBG_BUF_ATTR("missing_plane : %d", missing_plane);
  1109. DBG_BUF_ATTR("Planar Padding : %d", planar_padding);
  1110. if (planes == 1) {
  1111. /* uv_start beyond ULA range */
  1112. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
  1113. *uv_start_offset = size;
  1114. *ula_y_plane_size = size;
  1115. } else {
  1116. if (!missing_plane) {
  1117. /* size for both planes and padding */
  1118. /* Don't pad out Y plane as client would not expect this padding */
  1119. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, false);
  1120. *ula_y_plane_size = size;
  1121. size += planar_padding;
  1122. *uv_start_offset = size;
  1123. size += ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
  1124. } else {
  1125. if (missing_plane == 2) {
  1126. /* Y-only image, set uv_start beyond ULA range */
  1127. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
  1128. *uv_start_offset = size;
  1129. *ula_y_plane_size = size;
  1130. } else {
  1131. /* first plane data is not there */
  1132. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
  1133. *uv_start_offset = 0; /* uv data is at the beginning */
  1134. *ula_y_plane_size = 0;
  1135. }
  1136. }
  1137. }
  1138. *ula_size = UBWCP_ALIGN(size, 4096);
  1139. DBG_BUF_ATTR("ULA_Size: %zu (0x%x) (before 4K align: %zu)", *ula_size, *ula_size, size);
  1140. return 0;
  1141. }
  1142. /* calculate UBWCP buffer parms */
  1143. static int ubwcp_calc_ubwcp_buf_params(struct ubwcp_driver *ubwcp,
  1144. struct ubwcp_buffer_attrs *attr,
  1145. size_t *md_p0, size_t *pd_p0,
  1146. size_t *md_p1, size_t *pd_p1,
  1147. size_t *stride_tp10_b)
  1148. {
  1149. int planes;
  1150. int missing_plane;
  1151. enum ubwcp_std_image_format format;
  1152. size_t stride_tp10_p;
  1153. int ret;
  1154. ret = to_std_format(attr->image_format, &format);
  1155. if (ret)
  1156. return ret;
  1157. missing_plane = missing_plane_from_format(attr->image_format);
  1158. planes = planes_in_format(format);
  1159. DBG_BUF_ATTR("ubwcp params -->");
  1160. DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
  1161. DBG_BUF_ATTR("planes_in_format : %d", planes);
  1162. DBG_BUF_ATTR("missing_plane : %d", missing_plane);
  1163. *md_p0 = 0;
  1164. *pd_p0 = 0;
  1165. *md_p1 = 0;
  1166. *pd_p1 = 0;
  1167. *stride_tp10_b = 0;
  1168. if (missing_plane != 1) {
  1169. if (metadata_buf_sz(ubwcp, format, attr->width, attr->height, 0, md_p0))
  1170. return -1;
  1171. if (pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 0, pd_p0))
  1172. return -1;
  1173. }
  1174. if ((planes == 2) && (missing_plane != 2)){
  1175. if (metadata_buf_sz(ubwcp, format, attr->width, attr->height, 1, md_p1))
  1176. return -1;
  1177. if (pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 1, pd_p1))
  1178. return -1;
  1179. }
  1180. if (format == TP10) {
  1181. stride_tp10_p = UBWCP_ALIGN(attr->width, 192);
  1182. *stride_tp10_b = (stride_tp10_p/3) + stride_tp10_p;
  1183. }
  1184. return 0;
  1185. }
  1186. /* reserve ULA address space of the given size */
  1187. static phys_addr_t ubwcp_ula_alloc(struct ubwcp_driver *ubwcp, size_t size)
  1188. {
  1189. phys_addr_t pa;
  1190. mutex_lock(&ubwcp->ula_lock);
  1191. pa = gen_pool_alloc(ubwcp->ula_pool, size);
  1192. mutex_unlock(&ubwcp->ula_lock);
  1193. return pa;
  1194. }
  1195. /* free ULA address space of the given address and size */
  1196. static void ubwcp_ula_free(struct ubwcp_driver *ubwcp, phys_addr_t pa, size_t size)
  1197. {
  1198. mutex_lock(&ubwcp->ula_lock);
  1199. if (!gen_pool_has_addr(ubwcp->ula_pool, pa, size)) {
  1200. ERR("Attempt to free mem not from gen_pool: pa: %p, size: %zx", pa, size);
  1201. goto err;
  1202. }
  1203. DBG("addr: %p, size: %zx", pa, size);
  1204. gen_pool_free(ubwcp->ula_pool, pa, size);
  1205. mutex_unlock(&ubwcp->ula_lock);
  1206. return;
  1207. err:
  1208. mutex_unlock(&ubwcp->ula_lock);
  1209. }
  1210. /* free up or expand current_pa and return the new pa */
  1211. static phys_addr_t ubwcp_ula_realloc(struct ubwcp_driver *ubwcp,
  1212. phys_addr_t pa,
  1213. size_t size,
  1214. size_t new_size)
  1215. {
  1216. if (size == new_size)
  1217. return pa;
  1218. if (pa)
  1219. ubwcp_ula_free(ubwcp, pa, size);
  1220. return ubwcp_ula_alloc(ubwcp, new_size);
  1221. }
  1222. /* unmap dma buf */
  1223. static void ubwcp_dma_unmap(struct ubwcp_buf *buf)
  1224. {
  1225. if (buf->dma_buf && buf->attachment) {
  1226. DBG("Calling dma_buf_unmap_attachment()");
  1227. dma_buf_unmap_attachment(buf->attachment, buf->sgt, DMA_BIDIRECTIONAL);
  1228. buf->sgt = NULL;
  1229. dma_buf_detach(buf->dma_buf, buf->attachment);
  1230. buf->attachment = NULL;
  1231. }
  1232. }
  1233. static bool verify_dma_buf_size(struct ubwcp_buf *buf, size_t min_size)
  1234. {
  1235. size_t dma_len;
  1236. dma_len = sg_dma_len(buf->sgt->sgl);
  1237. if (dma_len < min_size) {
  1238. ERR("dma len: %zu is less than min ubwcp buffer size: %zu", dma_len, min_size);
  1239. return false;
  1240. } else
  1241. return true;
  1242. }
  1243. /* dma map ubwcp buffer */
  1244. static int ubwcp_dma_map(struct ubwcp_buf *buf,
  1245. struct device *dev,
  1246. dma_addr_t *iova)
  1247. {
  1248. int ret = 0;
  1249. struct dma_buf *dma_buf = buf->dma_buf;
  1250. struct dma_buf_attachment *attachment;
  1251. struct sg_table *sgt;
  1252. /* Map buffer to SMMU and get IOVA */
  1253. attachment = dma_buf_attach(dma_buf, dev);
  1254. if (IS_ERR(attachment)) {
  1255. ret = PTR_ERR(attachment);
  1256. ERR("dma_buf_attach() failed: %d", ret);
  1257. goto err;
  1258. }
  1259. dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
  1260. dma_set_seg_boundary(dev, (unsigned long)DMA_BIT_MASK(64));
  1261. sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
  1262. if (IS_ERR_OR_NULL(sgt)) {
  1263. ret = PTR_ERR(sgt);
  1264. ERR("dma_buf_map_attachment() failed: %d", ret);
  1265. goto err_detach;
  1266. }
  1267. if (sgt->nents != 1) {
  1268. ERR("nents = %d", sgt->nents);
  1269. goto err_unmap;
  1270. }
  1271. *iova = sg_dma_address(sgt->sgl);
  1272. buf->attachment = attachment;
  1273. buf->sgt = sgt;
  1274. return ret;
  1275. err_unmap:
  1276. dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
  1277. err_detach:
  1278. dma_buf_detach(dma_buf, attachment);
  1279. err:
  1280. if (!ret)
  1281. ret = -1;
  1282. return ret;
  1283. }
  1284. static void reset_buf_attrs(struct ubwcp_buf *buf)
  1285. {
  1286. struct ubwcp_hw_meta_metadata *mmdata;
  1287. struct ubwcp_driver *ubwcp;
  1288. ubwcp = buf->ubwcp;
  1289. mmdata = &buf->mmdata;
  1290. ubwcp_dma_unmap(buf);
  1291. /* reset ula params */
  1292. if (buf->ula_size) {
  1293. ubwcp_ula_free(ubwcp, buf->ula_pa, buf->ula_size);
  1294. buf->ula_size = 0;
  1295. buf->ula_pa = 0;
  1296. }
  1297. /* reset ubwcp params */
  1298. memset(mmdata, 0, sizeof(*mmdata));
  1299. buf->buf_attr_set = false;
  1300. buf->buf_attr.image_format = UBWCP_LINEAR;
  1301. }
  1302. static void print_mmdata_desc(struct ubwcp_hw_meta_metadata *mmdata)
  1303. {
  1304. DBG_BUF_ATTR("");
  1305. DBG_BUF_ATTR("--------MM_DATA DESC ---------");
  1306. DBG_BUF_ATTR("uv_start_addr : 0x%08llx (cache addr) (actual: 0x%llx)",
  1307. mmdata->uv_start_addr, mmdata->uv_start_addr << 6);
  1308. DBG_BUF_ATTR("format : 0x%08x", mmdata->format);
  1309. DBG_BUF_ATTR("stride : 0x%08x (cache addr) (actual: 0x%x)",
  1310. mmdata->stride, mmdata->stride << 6);
  1311. DBG_BUF_ATTR("stride_ubwcp : 0x%08x (cache addr) (actual: 0x%zx)",
  1312. mmdata->stride_ubwcp, mmdata->stride_ubwcp << 6);
  1313. DBG_BUF_ATTR("metadata_base_y : 0x%08x (page addr) (actual: 0x%llx)",
  1314. mmdata->metadata_base_y, mmdata->metadata_base_y << 12);
  1315. DBG_BUF_ATTR("metadata_base_uv: 0x%08x (page addr) (actual: 0x%zx)",
  1316. mmdata->metadata_base_uv, mmdata->metadata_base_uv << 12);
  1317. DBG_BUF_ATTR("buffer_y_offset : 0x%08x (page addr) (actual: 0x%zx)",
  1318. mmdata->buffer_y_offset, mmdata->buffer_y_offset << 12);
  1319. DBG_BUF_ATTR("buffer_uv_offset: 0x%08x (page addr) (actual: 0x%zx)",
  1320. mmdata->buffer_uv_offset, mmdata->buffer_uv_offset << 12);
  1321. DBG_BUF_ATTR("width_height : 0x%08x (width: 0x%x height: 0x%x)",
  1322. mmdata->width_height, mmdata->width_height >> 16, mmdata->width_height & 0xFFFF);
  1323. DBG_BUF_ATTR("");
  1324. }
  1325. /* set buffer attributes:
  1326. * Failure:
  1327. * This call may fail for multiple reasons and it will leave the buffer in an undefined state.
  1328. * In some situations it may leave the buffer in linear mapped state, and in other situations it
  1329. * may leave the buffer in previously set attributes state.
  1330. */
  1331. int ubwcp_set_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
  1332. {
  1333. int ret = 0;
  1334. size_t ula_size = 0;
  1335. size_t uv_start_offset = 0;
  1336. size_t ula_y_plane_size = 0;
  1337. phys_addr_t ula_pa = 0x0;
  1338. struct ubwcp_buf *buf;
  1339. struct ubwcp_driver *ubwcp;
  1340. size_t metadata_p0;
  1341. size_t pixeldata_p0;
  1342. size_t metadata_p1;
  1343. size_t pixeldata_p1;
  1344. size_t iova_min_size;
  1345. size_t stride_tp10_b;
  1346. dma_addr_t iova_base;
  1347. struct ubwcp_hw_meta_metadata *mmdata;
  1348. u64 uv_start;
  1349. u32 stride_b;
  1350. u32 width_b;
  1351. u32 height_b;
  1352. enum ubwcp_std_image_format std_image_format;
  1353. bool is_non_lin_buf;
  1354. u16 hw_img_format;
  1355. trace_ubwcp_set_buf_attrs_start(dmabuf);
  1356. if (!dmabuf) {
  1357. ERR("NULL dmabuf input ptr");
  1358. ret = -EINVAL;
  1359. goto err_validation;
  1360. }
  1361. if (!attr) {
  1362. ERR("NULL attr ptr");
  1363. ret = -EINVAL;
  1364. goto err_validation;
  1365. }
  1366. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1367. if (!buf) {
  1368. ERR("No corresponding ubwcp_buf for the passed in dma_buf");
  1369. ret = -EINVAL;
  1370. goto err_validation;
  1371. }
  1372. ubwcp = buf->ubwcp;
  1373. if (ubwcp->state != UBWCP_STATE_READY) {
  1374. ret = EPERM;
  1375. goto err_validation;
  1376. }
  1377. if (!ubwcp_buf_attrs_valid(ubwcp, attr)) {
  1378. ERR("Invalid buf attrs");
  1379. ret = -EINVAL;
  1380. goto err_validation;
  1381. }
  1382. mutex_lock(&buf->lock);
  1383. if (buf->lock_count) {
  1384. ERR("Cannot set attr when buffer is locked");
  1385. ret = -EBUSY;
  1386. goto unlock;
  1387. }
  1388. mmdata = &buf->mmdata;
  1389. is_non_lin_buf = (buf->buf_attr.image_format != UBWCP_LINEAR);
  1390. /* note: this also checks if buf is mmap'ed */
  1391. ret = ubwcp->mmap_config_fptr(buf->dma_buf, true, 0, 0);
  1392. if (ret) {
  1393. ERR("dma_buf_mmap_config(0,0) failed: %d", ret);
  1394. goto unlock;
  1395. }
  1396. if (attr->image_format == UBWCP_LINEAR) {
  1397. DBG_BUF_ATTR("Linear format requested");
  1398. if (buf->buf_attr_set)
  1399. reset_buf_attrs(buf);
  1400. if (is_non_lin_buf) {
  1401. /*
  1402. * Changing buffer from ubwc to linear so decrement
  1403. * number of ubwc buffers
  1404. */
  1405. ret = dec_num_non_lin_buffers(ubwcp);
  1406. }
  1407. mutex_unlock(&buf->lock);
  1408. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1409. return ret;
  1410. }
  1411. if (to_std_format(attr->image_format, &std_image_format)) {
  1412. ERR("Unable to map ioctl image format to std image format");
  1413. goto unlock;
  1414. }
  1415. if (std_to_hw_img_fmt(std_image_format, &hw_img_format)) {
  1416. ERR("Unable to map std image format to hw image format");
  1417. goto unlock;
  1418. }
  1419. /* Calculate uncompressed-buffer size. */
  1420. ret = ubwcp_calc_ula_params(ubwcp, attr, &ula_size, &ula_y_plane_size, &uv_start_offset);
  1421. if (ret) {
  1422. ERR("ubwcp_calc_ula_params() failed: %d", ret);
  1423. goto unlock;
  1424. }
  1425. ret = ubwcp_validate_uv_align(ubwcp, attr, ula_y_plane_size, uv_start_offset);
  1426. if (ret) {
  1427. ERR("ubwcp_validate_uv_align() failed: %d", ret);
  1428. goto unlock;
  1429. }
  1430. ret = ubwcp_calc_ubwcp_buf_params(ubwcp, attr, &metadata_p0, &pixeldata_p0, &metadata_p1,
  1431. &pixeldata_p1, &stride_tp10_b);
  1432. if (ret) {
  1433. ERR("ubwcp_calc_buf_params() failed: %d", ret);
  1434. goto unlock;
  1435. }
  1436. iova_min_size = metadata_p0 + pixeldata_p0 + metadata_p1 + pixeldata_p1;
  1437. DBG_BUF_ATTR("");
  1438. DBG_BUF_ATTR("");
  1439. DBG_BUF_ATTR("------Summary ULA Calculated Params ------");
  1440. DBG_BUF_ATTR("ULA Size : %8zu (0x%8zx)", ula_size, ula_size);
  1441. DBG_BUF_ATTR("UV Start Offset : %8zu (0x%8zx)", uv_start_offset, uv_start_offset);
  1442. DBG_BUF_ATTR("------Summary UBCP Calculated Params ------");
  1443. DBG_BUF_ATTR("metadata_p0 : %8d (0x%8zx)", metadata_p0, metadata_p0);
  1444. DBG_BUF_ATTR("pixeldata_p0 : %8d (0x%8zx)", pixeldata_p0, pixeldata_p0);
  1445. DBG_BUF_ATTR("metadata_p1 : %8d (0x%8zx)", metadata_p1, metadata_p1);
  1446. DBG_BUF_ATTR("pixeldata_p1 : %8d (0x%8zx)", pixeldata_p1, pixeldata_p1);
  1447. DBG_BUF_ATTR("stride_tp10 : %8d (0x%8zx)", stride_tp10_b, stride_tp10_b);
  1448. DBG_BUF_ATTR("iova_min_size : %8d (0x%8zx)", iova_min_size, iova_min_size);
  1449. DBG_BUF_ATTR("");
  1450. if (!ula_size) {
  1451. ERR("Invalid ula_size (0)");
  1452. goto unlock;
  1453. }
  1454. /* assign ULA PA with uncompressed-size range */
  1455. ula_pa = ubwcp_ula_realloc(ubwcp, buf->ula_pa, buf->ula_size, ula_size);
  1456. if (!ula_pa) {
  1457. ERR("ubwcp_ula_alloc/realloc() failed. running out of ULA PA space?");
  1458. goto err;
  1459. }
  1460. buf->ula_size = ula_size;
  1461. buf->ula_pa = ula_pa;
  1462. DBG_BUF_ATTR("Allocated ULA_PA: 0x%p of size: 0x%zx", ula_pa, ula_size);
  1463. DBG_BUF_ATTR("");
  1464. /* dma map only the first time attribute is set */
  1465. if (!buf->buf_attr_set) {
  1466. /* linear -> ubwcp. map ubwcp buffer */
  1467. ret = ubwcp_dma_map(buf, ubwcp->dev_buf_cb, &iova_base);
  1468. if (ret) {
  1469. ERR("ubwcp_dma_map() failed: %d", ret);
  1470. goto err;
  1471. }
  1472. DBG_BUF_ATTR("dma_buf IOVA range: 0x%llx + min_size (0x%zx): 0x%llx",
  1473. iova_base, iova_min_size, iova_base + iova_min_size);
  1474. }
  1475. if(!verify_dma_buf_size(buf, iova_min_size))
  1476. goto err;
  1477. uv_start = ula_pa + uv_start_offset;
  1478. if (!IS_ALIGNED(uv_start, 64)) {
  1479. ERR("ERROR: uv_start is NOT aligned to cache line");
  1480. goto err;
  1481. }
  1482. /* Convert height and width to bytes for writing to mmdata */
  1483. if (std_image_format != TP10) {
  1484. ubwcp_pixel_to_bytes(ubwcp, std_image_format, attr->width,
  1485. attr->height, &width_b, &height_b);
  1486. } else {
  1487. /* for tp10 image compression, we need to program p010 width/height */
  1488. ubwcp_pixel_to_bytes(ubwcp, P010, attr->width,
  1489. attr->height, &width_b, &height_b);
  1490. }
  1491. stride_b = attr->stride;
  1492. /* create the mmdata descriptor */
  1493. memset(mmdata, 0, sizeof(*mmdata));
  1494. mmdata->uv_start_addr = CACHE_ADDR(uv_start);
  1495. mmdata->format = hw_img_format;
  1496. if (std_image_format != TP10) {
  1497. mmdata->stride = CACHE_ADDR(stride_b); /* uncompressed stride */
  1498. } else {
  1499. mmdata->stride = CACHE_ADDR(stride_tp10_b); /* compressed stride */
  1500. mmdata->stride_ubwcp = CACHE_ADDR(stride_b); /* uncompressed stride */
  1501. }
  1502. mmdata->metadata_base_y = PAGE_ADDR(iova_base);
  1503. mmdata->metadata_base_uv = PAGE_ADDR(iova_base + metadata_p0 + pixeldata_p0);
  1504. mmdata->buffer_y_offset = PAGE_ADDR(metadata_p0);
  1505. mmdata->buffer_uv_offset = PAGE_ADDR(metadata_p1);
  1506. /* NOTE: For version 1.1, both width & height needs to be in bytes.
  1507. * For other versions, width in bytes & height in pixels.
  1508. */
  1509. if ((ubwcp->hw_ver_major == 1) && (ubwcp->hw_ver_minor == 1))
  1510. mmdata->width_height = width_b << 16 | height_b;
  1511. else
  1512. mmdata->width_height = width_b << 16 | attr->height;
  1513. print_mmdata_desc(mmdata);
  1514. if (!is_non_lin_buf) {
  1515. /*
  1516. * Changing buffer from linear to ubwc so increment
  1517. * number of ubwc buffers
  1518. */
  1519. ret = inc_num_non_lin_buffers(ubwcp);
  1520. }
  1521. if (ret) {
  1522. ERR("inc_num_non_lin_buffers failed: %d", ret);
  1523. goto err;
  1524. }
  1525. /* inform ULA-PA to dma-heap */
  1526. DBG_BUF_ATTR("Calling mmap_config(): ULA_PA: 0x%p size: 0x%zx", ula_pa, ula_size);
  1527. ret = ubwcp->mmap_config_fptr(buf->dma_buf, false, buf->ula_pa, buf->ula_size);
  1528. if (ret) {
  1529. ERR("dma_buf_mmap_config() failed: %d", ret);
  1530. if (!is_non_lin_buf)
  1531. dec_num_non_lin_buffers(ubwcp);
  1532. goto err;
  1533. }
  1534. buf->buf_attr = *attr;
  1535. buf->buf_attr_set = true;
  1536. mutex_unlock(&buf->lock);
  1537. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1538. return 0;
  1539. err:
  1540. reset_buf_attrs(buf);
  1541. if (is_non_lin_buf) {
  1542. /*
  1543. * Changing buffer from ubwc to linear so decrement
  1544. * number of ubwc buffers
  1545. */
  1546. dec_num_non_lin_buffers(ubwcp);
  1547. }
  1548. unlock:
  1549. mutex_unlock(&buf->lock);
  1550. err_validation:
  1551. if (!ret)
  1552. ret = -1;
  1553. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1554. return ret;
  1555. }
  1556. EXPORT_SYMBOL(ubwcp_set_buf_attrs);
  1557. /* Free up the buffer descriptor */
  1558. static void ubwcp_buf_desc_free(struct ubwcp_driver *ubwcp, struct ubwcp_desc *desc)
  1559. {
  1560. int idx = desc->idx;
  1561. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  1562. mutex_lock(&ubwcp->desc_lock);
  1563. desc_list[idx].idx = -1;
  1564. desc_list[idx].ptr = NULL;
  1565. DBG("freed descriptor_id: %d", idx);
  1566. mutex_unlock(&ubwcp->desc_lock);
  1567. }
  1568. /* Allocate next available buffer descriptor. */
  1569. static struct ubwcp_desc *ubwcp_buf_desc_allocate(struct ubwcp_driver *ubwcp)
  1570. {
  1571. int idx;
  1572. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  1573. mutex_lock(&ubwcp->desc_lock);
  1574. for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
  1575. if (desc_list[idx].idx == -1) {
  1576. desc_list[idx].idx = idx;
  1577. desc_list[idx].ptr = ubwcp->buffer_desc_base +
  1578. idx*UBWCP_BUFFER_DESC_OFFSET;
  1579. DBG("allocated descriptor_id: %d", idx);
  1580. mutex_unlock(&ubwcp->desc_lock);
  1581. return &desc_list[idx];
  1582. }
  1583. }
  1584. mutex_unlock(&ubwcp->desc_lock);
  1585. return NULL;
  1586. }
  1587. static int ubwcp_flush(struct ubwcp_driver *ubwcp)
  1588. {
  1589. int ret = 0;
  1590. mutex_lock(&ubwcp->ubwcp_flush_lock);
  1591. trace_ubwcp_hw_flush_start(0);
  1592. ret = ubwcp_hw_flush(ubwcp->base);
  1593. trace_ubwcp_hw_flush_end(0);
  1594. if (ret)
  1595. ERR("ubwcp_hw_flush() failed, ret = %d", ret);
  1596. mutex_unlock(&ubwcp->ubwcp_flush_lock);
  1597. return ret;
  1598. }
  1599. static int range_check_disable(struct ubwcp_driver *ubwcp, int idx)
  1600. {
  1601. int ret;
  1602. mutex_lock(&ubwcp->ubwcp_flush_lock);
  1603. mutex_lock(&ubwcp->hw_range_ck_lock);
  1604. trace_ubwcp_hw_flush_start(0);
  1605. ret = ubwcp_hw_disable_range_check_with_flush(ubwcp->base, idx);
  1606. trace_ubwcp_hw_flush_end(0);
  1607. if (ret)
  1608. ERR("disable_range_check_with_flush() failed: %d", ret);
  1609. mutex_unlock(&ubwcp->hw_range_ck_lock);
  1610. mutex_unlock(&ubwcp->ubwcp_flush_lock);
  1611. return ret;
  1612. }
  1613. static void range_check_enable(struct ubwcp_driver *ubwcp, int idx)
  1614. {
  1615. mutex_lock(&ubwcp->hw_range_ck_lock);
  1616. ubwcp_hw_enable_range_check(ubwcp->base, idx);
  1617. mutex_unlock(&ubwcp->hw_range_ck_lock);
  1618. }
  1619. /**
  1620. * Lock buffer for CPU access. This prepares ubwcp hw to allow
  1621. * CPU access to the compressed buffer. It will perform
  1622. * necessary address translation configuration and cache maintenance ops
  1623. * so that CPU can safely access ubwcp buffer, if this call is
  1624. * successful.
  1625. * Allocate descriptor if not already,
  1626. * perform CMO and then enable range check
  1627. *
  1628. * @param dmabuf : ptr to the dma buf
  1629. * @param direction : direction of access
  1630. *
  1631. * @return int : 0 on success, otherwise error code
  1632. */
  1633. static int ubwcp_lock(struct dma_buf *dmabuf, enum dma_data_direction dir)
  1634. {
  1635. int ret = 0;
  1636. struct ubwcp_buf *buf;
  1637. struct ubwcp_driver *ubwcp;
  1638. trace_ubwcp_lock_start(dmabuf);
  1639. if (!dmabuf) {
  1640. ERR("NULL dmabuf input ptr");
  1641. trace_ubwcp_lock_end(dmabuf);
  1642. return -EINVAL;
  1643. }
  1644. if (!valid_dma_direction(dir)) {
  1645. ERR("invalid direction: %d", dir);
  1646. trace_ubwcp_lock_end(dmabuf);
  1647. return -EINVAL;
  1648. }
  1649. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1650. if (!buf) {
  1651. ERR("ubwcp_buf ptr not found");
  1652. trace_ubwcp_lock_end(dmabuf);
  1653. return -1;
  1654. }
  1655. ubwcp = buf->ubwcp;
  1656. if (ubwcp->state != UBWCP_STATE_READY) {
  1657. ERR("driver in invalid state: %d", ubwcp->state);
  1658. trace_ubwcp_lock_end(dmabuf);
  1659. return -EPERM;
  1660. }
  1661. mutex_lock(&buf->lock);
  1662. if (!buf->buf_attr_set) {
  1663. ERR("lock() called on buffer, but attr not set");
  1664. goto err;
  1665. }
  1666. if (buf->buf_attr.image_format == UBWCP_LINEAR) {
  1667. ERR("lock() called on linear buffer");
  1668. goto err;
  1669. }
  1670. if (!buf->lock_count) {
  1671. DBG("first lock on buffer");
  1672. /* buf->desc could already be allocated because of perm range xlation */
  1673. if (!buf->desc) {
  1674. /* allocate a buffer descriptor */
  1675. buf->desc = ubwcp_buf_desc_allocate(buf->ubwcp);
  1676. if (!buf->desc) {
  1677. ERR("ubwcp_allocate_buf_desc() failed");
  1678. goto err;
  1679. }
  1680. memcpy(buf->desc->ptr, &buf->mmdata, sizeof(buf->mmdata));
  1681. /* Flushing of updated mmdata:
  1682. * mmdata is iocoherent and ubwcp will get it from CPU cache -
  1683. * *as long as* it has not cached that itself during previous
  1684. * access to the same descriptor.
  1685. *
  1686. * During unlock of previous use of this descriptor,
  1687. * we do hw flush, which will get rid of this mmdata from
  1688. * ubwcp cache.
  1689. *
  1690. * In addition, we also do a hw flush after enable_range_ck().
  1691. * That will also get rid of any speculative fetch of mmdata
  1692. * by the ubwcp hw. At this time, the assumption is that ubwcp
  1693. * will cache mmdata only for active descriptor. But if ubwcp
  1694. * is speculatively fetching mmdata for all descriptors
  1695. * (irrespetive of enabled or not), the flush during lock
  1696. * will be necessary to make sure ubwcp sees updated mmdata
  1697. * that we just updated
  1698. */
  1699. /* program ULA range for this buffer */
  1700. DBG("setting range check: descriptor_id: %d, addr: %p, size: %zx",
  1701. buf->desc->idx, buf->ula_pa, buf->ula_size);
  1702. ubwcp_hw_set_range_check(ubwcp->base, buf->desc->idx, buf->ula_pa,
  1703. buf->ula_size);
  1704. }
  1705. /* enable range check */
  1706. DBG("enabling range check, descriptor_id: %d", buf->desc->idx);
  1707. range_check_enable(ubwcp, buf->desc->idx);
  1708. /* Flush/invalidate UBWCP caches */
  1709. /* Why: cpu could have done a speculative fetch before
  1710. * enable_range_ck() and ubwcp in process of returning "default" data
  1711. * we don't want that stashing of default data pending.
  1712. * we force completion of that and then we also cpu invalidate which
  1713. * will get rid of that line.
  1714. */
  1715. ret = ubwcp_flush(ubwcp);
  1716. if (ret) {
  1717. ubwcp->state = UBWCP_STATE_FAULT;
  1718. ERR("state set to fault");
  1719. goto err_flush_failed;
  1720. }
  1721. /* Flush/invalidate ULA PA from CPU caches
  1722. * Always invalidate cache, even when writing.
  1723. * Upgrade direction to force invalidate.
  1724. */
  1725. if (dir == DMA_TO_DEVICE)
  1726. dir = DMA_BIDIRECTIONAL;
  1727. trace_ubwcp_dma_sync_single_for_cpu_start(buf->ula_size, dir);
  1728. dma_sync_single_for_cpu(ubwcp->dev, buf->ula_pa, buf->ula_size, dir);
  1729. trace_ubwcp_dma_sync_single_for_cpu_end(buf->ula_size, dir);
  1730. buf->dma_dir = dir;
  1731. } else {
  1732. DBG("buf already locked");
  1733. /* For write locks, always upgrade direction to bi_directional.
  1734. * A previous read lock will now become write lock.
  1735. * This will ensure a flush when the last unlock comes in.
  1736. */
  1737. if ((dir == DMA_TO_DEVICE) || (dir == DMA_BIDIRECTIONAL))
  1738. buf->dma_dir = DMA_BIDIRECTIONAL;
  1739. }
  1740. buf->lock_count++;
  1741. DBG("new lock_count: %d", buf->lock_count);
  1742. mutex_unlock(&buf->lock);
  1743. trace_ubwcp_lock_end(dmabuf);
  1744. return ret;
  1745. err_flush_failed:
  1746. range_check_disable(ubwcp, buf->desc->idx);
  1747. ubwcp_buf_desc_free(ubwcp, buf->desc);
  1748. buf->desc = NULL;
  1749. err:
  1750. mutex_unlock(&buf->lock);
  1751. if (!ret)
  1752. ret = -1;
  1753. trace_ubwcp_lock_end(dmabuf);
  1754. return ret;
  1755. }
  1756. /* This can be called as a result of external unlock() call or
  1757. * internally if free() is called without unlock().
  1758. */
  1759. static int unlock_internal(struct ubwcp_buf *buf, enum dma_data_direction dir, bool free_buffer)
  1760. {
  1761. int ret = 0;
  1762. struct ubwcp_driver *ubwcp;
  1763. DBG("current lock_count: %d", buf->lock_count);
  1764. if (free_buffer) {
  1765. buf->lock_count = 0;
  1766. DBG("Forced lock_count: %d", buf->lock_count);
  1767. } else {
  1768. /* for write unlocks, remember the direction so we flush on last unlock */
  1769. if ((dir == DMA_TO_DEVICE) || (dir == DMA_BIDIRECTIONAL))
  1770. buf->dma_dir = DMA_BIDIRECTIONAL;
  1771. buf->lock_count--;
  1772. DBG("new lock_count: %d", buf->lock_count);
  1773. if (buf->lock_count) {
  1774. DBG("more than 1 lock on buffer. waiting until last unlock");
  1775. return 0;
  1776. }
  1777. }
  1778. ubwcp = buf->ubwcp;
  1779. /* Only apply CMOs if there were potential CPU writes */
  1780. if (buf->dma_dir == DMA_TO_DEVICE || buf->dma_dir == DMA_BIDIRECTIONAL) {
  1781. /* Flush/invalidate ULA PA from CPU caches */
  1782. trace_ubwcp_dma_sync_single_for_device_start(buf->ula_size, buf->dma_dir);
  1783. dma_sync_single_for_device(ubwcp->dev, buf->ula_pa, buf->ula_size, buf->dma_dir);
  1784. trace_ubwcp_dma_sync_single_for_device_end(buf->ula_size, buf->dma_dir);
  1785. }
  1786. /* disable range check */
  1787. DBG("disabling range check");
  1788. ret = range_check_disable(ubwcp, buf->desc->idx);
  1789. if (ret) {
  1790. ubwcp->state = UBWCP_STATE_FAULT;
  1791. ERR("state set to fault");
  1792. }
  1793. /* release descriptor if perm range xlation is not set */
  1794. if (!buf->perm) {
  1795. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1796. buf->desc = NULL;
  1797. }
  1798. return ret;
  1799. }
  1800. /**
  1801. * Unlock buffer from CPU access. This prepares ubwcp hw to
  1802. * safely allow for device access to the compressed buffer including any
  1803. * necessary cache maintenance ops. It may also free up certain ubwcp
  1804. * resources that could result in error when accessed by CPU in
  1805. * unlocked state.
  1806. *
  1807. * @param dmabuf : ptr to the dma buf
  1808. * @param direction : direction of access
  1809. *
  1810. * @return int : 0 on success, otherwise error code
  1811. */
  1812. static int ubwcp_unlock(struct dma_buf *dmabuf, enum dma_data_direction dir)
  1813. {
  1814. struct ubwcp_buf *buf;
  1815. int ret;
  1816. trace_ubwcp_unlock_start(dmabuf);
  1817. if (!dmabuf) {
  1818. ERR("NULL dmabuf input ptr");
  1819. trace_ubwcp_unlock_end(dmabuf);
  1820. return -EINVAL;
  1821. }
  1822. if (!valid_dma_direction(dir)) {
  1823. ERR("invalid direction: %d", dir);
  1824. trace_ubwcp_unlock_end(dmabuf);
  1825. return -EINVAL;
  1826. }
  1827. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1828. if (!buf) {
  1829. ERR("ubwcp_buf not found");
  1830. trace_ubwcp_unlock_end(dmabuf);
  1831. return -1;
  1832. }
  1833. if (buf->ubwcp->state != UBWCP_STATE_READY) {
  1834. ERR("driver in invalid state: %d", buf->ubwcp->state);
  1835. trace_ubwcp_unlock_end(dmabuf);
  1836. return -EPERM;
  1837. }
  1838. mutex_lock(&buf->lock);
  1839. if (!buf->lock_count) {
  1840. ERR("unlock() called on buffer which not in locked state");
  1841. trace_ubwcp_unlock_end(dmabuf);
  1842. mutex_unlock(&buf->lock);
  1843. return -1;
  1844. }
  1845. ret = unlock_internal(buf, dir, false);
  1846. mutex_unlock(&buf->lock);
  1847. trace_ubwcp_unlock_end(dmabuf);
  1848. return ret;
  1849. }
  1850. /* Return buffer attributes for the given buffer */
  1851. int ubwcp_get_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
  1852. {
  1853. int ret = 0;
  1854. struct ubwcp_buf *buf;
  1855. if (!dmabuf) {
  1856. ERR("NULL dmabuf input ptr");
  1857. return -EINVAL;
  1858. }
  1859. if (!attr) {
  1860. ERR("NULL attr ptr");
  1861. return -EINVAL;
  1862. }
  1863. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1864. if (!buf) {
  1865. ERR("ubwcp_buf ptr not found");
  1866. return -1;
  1867. }
  1868. if (buf->ubwcp->state != UBWCP_STATE_READY) {
  1869. ERR("driver in invalid state: %d", buf->ubwcp->state);
  1870. return -EPERM;
  1871. }
  1872. mutex_lock(&buf->lock);
  1873. if (!buf->buf_attr_set) {
  1874. ERR("buffer attributes not set");
  1875. mutex_unlock(&buf->lock);
  1876. return -1;
  1877. }
  1878. *attr = buf->buf_attr;
  1879. mutex_unlock(&buf->lock);
  1880. return ret;
  1881. }
  1882. EXPORT_SYMBOL(ubwcp_get_buf_attrs);
  1883. /* Set permanent range translation.
  1884. * enable: Descriptor will be reserved for this buffer until disabled,
  1885. * making lock/unlock quicker.
  1886. * disable: Descriptor will not be reserved for this buffer. Instead,
  1887. * descriptor will be allocated and released for each lock/unlock.
  1888. * If currently allocated but not being used, descriptor will be
  1889. * released.
  1890. */
  1891. int ubwcp_set_perm_range_translation(struct dma_buf *dmabuf, bool enable)
  1892. {
  1893. int ret = 0;
  1894. struct ubwcp_buf *buf;
  1895. if (!dmabuf) {
  1896. ERR("NULL dmabuf input ptr");
  1897. return -EINVAL;
  1898. }
  1899. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1900. if (!buf) {
  1901. ERR("ubwcp_buf not found");
  1902. return -1;
  1903. }
  1904. if (buf->ubwcp->state != UBWCP_STATE_READY) {
  1905. ERR("driver in invalid state: %d", buf->ubwcp->state);
  1906. return -EPERM;
  1907. }
  1908. /* not implemented */
  1909. if (1) {
  1910. ERR("API not implemented yet");
  1911. return -1;
  1912. }
  1913. /* TBD: make sure we acquire buf lock while setting this so there is
  1914. * no race condition with attr_set/lock/unlock
  1915. */
  1916. buf->perm = enable;
  1917. /* if "disable" and we have allocated a desc and it is not being
  1918. * used currently, release it
  1919. */
  1920. if (!enable && buf->desc && !buf->lock_count) {
  1921. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1922. buf->desc = NULL;
  1923. /* Flush/invalidate UBWCP caches */
  1924. //TBD: need to do anything?
  1925. }
  1926. return ret;
  1927. }
  1928. EXPORT_SYMBOL(ubwcp_set_perm_range_translation);
  1929. /**
  1930. * Free up ubwcp resources for this buffer.
  1931. *
  1932. * @param dmabuf : ptr to the dma buf
  1933. *
  1934. * @return int : 0 on success, otherwise error code
  1935. */
  1936. static int ubwcp_free_buffer(struct dma_buf *dmabuf)
  1937. {
  1938. int ret = 0;
  1939. struct ubwcp_buf *buf;
  1940. struct ubwcp_driver *ubwcp;
  1941. unsigned long flags;
  1942. bool is_non_lin_buf;
  1943. trace_ubwcp_free_buffer_start(dmabuf);
  1944. if (!dmabuf) {
  1945. ERR("NULL dmabuf input ptr");
  1946. trace_ubwcp_free_buffer_end(dmabuf);
  1947. return -EINVAL;
  1948. }
  1949. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1950. if (!buf) {
  1951. ERR("ubwcp_buf ptr not found");
  1952. trace_ubwcp_free_buffer_end(dmabuf);
  1953. return -1;
  1954. }
  1955. ubwcp = buf->ubwcp;
  1956. if (ubwcp->state != UBWCP_STATE_READY) {
  1957. ERR("driver in invalid state: %d", ubwcp->state);
  1958. trace_ubwcp_free_buffer_end(dmabuf);
  1959. return -EPERM;
  1960. }
  1961. mutex_lock(&buf->lock);
  1962. is_non_lin_buf = (buf->buf_attr.image_format != UBWCP_LINEAR);
  1963. if (buf->lock_count) {
  1964. DBG("free before unlock (lock_count: %d). unlock()'ing first", buf->lock_count);
  1965. ret = unlock_internal(buf, buf->dma_dir, true);
  1966. if (ret)
  1967. ERR("unlock_internal(): failed : %d, but continuing free()", ret);
  1968. }
  1969. /* if we are still holding a desc, release it. this can happen only if perm == true */
  1970. if (buf->desc) {
  1971. if (!buf->perm) {
  1972. ubwcp->state = UBWCP_STATE_FAULT;
  1973. ERR("state set to fault");
  1974. }
  1975. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1976. buf->desc = NULL;
  1977. }
  1978. if (buf->buf_attr_set)
  1979. reset_buf_attrs(buf);
  1980. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  1981. hash_del(&buf->hnode);
  1982. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  1983. mutex_unlock(&buf->lock);
  1984. kfree(buf);
  1985. if (is_non_lin_buf)
  1986. dec_num_non_lin_buffers(ubwcp);
  1987. trace_ubwcp_free_buffer_end(dmabuf);
  1988. return ret;
  1989. }
  1990. /* file open: TBD: increment ref count? */
  1991. static int ubwcp_open(struct inode *i, struct file *f)
  1992. {
  1993. return 0;
  1994. }
  1995. /* file open: TBD: decrement ref count? */
  1996. static int ubwcp_close(struct inode *i, struct file *f)
  1997. {
  1998. return 0;
  1999. }
  2000. static int ioctl_set_buf_attr(struct ubwcp_driver *ubwcp, unsigned long ioctl_param)
  2001. {
  2002. int ret;
  2003. struct dma_buf *dmabuf;
  2004. struct ubwcp_ioctl_buffer_attrs buf_attr_ioctl;
  2005. if (copy_from_user(&buf_attr_ioctl, (const void __user *) ioctl_param,
  2006. sizeof(buf_attr_ioctl))) {
  2007. ERR("copy_from_user() failed");
  2008. return -EFAULT;
  2009. }
  2010. DBG("IOCTL: SET_BUF_ATTR: fd = %d", buf_attr_ioctl.fd);
  2011. dmabuf = dma_buf_get(buf_attr_ioctl.fd);
  2012. if (IS_ERR(dmabuf)) {
  2013. ERR("dmabuf ptr not found for dma_buf_fd = %d", buf_attr_ioctl.fd);
  2014. return PTR_ERR(dmabuf);
  2015. }
  2016. ret = ubwcp_set_buf_attrs(dmabuf, &buf_attr_ioctl.attr);
  2017. dma_buf_put(dmabuf);
  2018. return ret;
  2019. }
  2020. static int ioctl_get_hw_ver(struct ubwcp_driver *ubwcp, unsigned long ioctl_param)
  2021. {
  2022. struct ubwcp_ioctl_hw_version hw_ver;
  2023. DBG("IOCTL: GET_HW_VER");
  2024. if (ubwcp_get_hw_version(&hw_ver))
  2025. return -EINVAL;
  2026. if (copy_to_user((void __user *)ioctl_param, &hw_ver, sizeof(hw_ver))) {
  2027. ERR("copy_to_user() failed");
  2028. return -EFAULT;
  2029. }
  2030. return 0;
  2031. }
  2032. static int ioctl_get_stride_align(struct ubwcp_driver *ubwcp, unsigned long ioctl_param)
  2033. {
  2034. struct ubwcp_ioctl_stride_align stride_align_ioctl;
  2035. enum ubwcp_std_image_format format;
  2036. DBG("IOCTL: GET_STRIDE_ALIGN");
  2037. if (copy_from_user(&stride_align_ioctl, (const void __user *) ioctl_param,
  2038. sizeof(stride_align_ioctl))) {
  2039. ERR("copy_from_user() failed");
  2040. return -EFAULT;
  2041. }
  2042. if (stride_align_ioctl.unused != 0) {
  2043. ERR("unused values must be set to 0");
  2044. return -EINVAL;
  2045. }
  2046. if (!ioctl_format_is_valid(stride_align_ioctl.image_format)) {
  2047. ERR("invalid image format: %d", stride_align_ioctl.image_format);
  2048. return -EINVAL;
  2049. }
  2050. if (stride_align_ioctl.image_format == UBWCP_LINEAR) {
  2051. ERR("not supported for LINEAR format");
  2052. return -EINVAL;
  2053. }
  2054. if (to_std_format(stride_align_ioctl.image_format, &format)) {
  2055. ERR("Unable to map ioctl image format to std image format");
  2056. return -EINVAL;
  2057. }
  2058. if (get_stride_alignment(format, &stride_align_ioctl.stride_align)) {
  2059. ERR("failed for format: %d", format);
  2060. return -EFAULT;
  2061. }
  2062. if (copy_to_user((void __user *)ioctl_param, &stride_align_ioctl,
  2063. sizeof(stride_align_ioctl))) {
  2064. ERR("copy_to_user() failed");
  2065. return -EFAULT;
  2066. }
  2067. return 0;
  2068. }
  2069. static int ioctl_validate_stride(struct ubwcp_driver *ubwcp, unsigned long ioctl_param)
  2070. {
  2071. struct ubwcp_ioctl_validate_stride validate_stride_ioctl;
  2072. enum ubwcp_std_image_format format;
  2073. DBG("IOCTL: VALIDATE_STRIDE");
  2074. if (copy_from_user(&validate_stride_ioctl, (const void __user *) ioctl_param,
  2075. sizeof(validate_stride_ioctl))) {
  2076. ERR("copy_from_user() failed");
  2077. return -EFAULT;
  2078. }
  2079. if (validate_stride_ioctl.unused1 || validate_stride_ioctl.unused2) {
  2080. ERR("unused values must be set to 0");
  2081. return -EINVAL;
  2082. }
  2083. if (!ioctl_format_is_valid(validate_stride_ioctl.image_format)) {
  2084. ERR("not supported for LINEAR format");
  2085. return -EINVAL;
  2086. }
  2087. if (validate_stride_ioctl.image_format == UBWCP_LINEAR) {
  2088. ERR("not supported for LINEAR format");
  2089. return -EINVAL;
  2090. }
  2091. if (to_std_format(validate_stride_ioctl.image_format, &format)) {
  2092. ERR("Unable to map ioctl image format to std image format");
  2093. return -EINVAL;
  2094. }
  2095. validate_stride_ioctl.valid = stride_is_valid(ubwcp, format, validate_stride_ioctl.width,
  2096. validate_stride_ioctl.stride);
  2097. if (copy_to_user((void __user *)ioctl_param, &validate_stride_ioctl,
  2098. sizeof(validate_stride_ioctl))) {
  2099. ERR("copy_to_user() failed");
  2100. return -EFAULT;
  2101. }
  2102. return 0;
  2103. }
  2104. /* handle IOCTLs */
  2105. static long ubwcp_ioctl(struct file *file, unsigned int ioctl_num, unsigned long ioctl_param)
  2106. {
  2107. struct ubwcp_driver *ubwcp;
  2108. ubwcp = ubwcp_get_driver();
  2109. if (!ubwcp)
  2110. return -EINVAL;
  2111. if (ubwcp->state != UBWCP_STATE_READY) {
  2112. ERR("driver in invalid state: %d", ubwcp->state);
  2113. return -EPERM;
  2114. }
  2115. switch (ioctl_num) {
  2116. case UBWCP_IOCTL_SET_BUF_ATTR:
  2117. return ioctl_set_buf_attr(ubwcp, ioctl_param);
  2118. case UBWCP_IOCTL_GET_HW_VER:
  2119. return ioctl_get_hw_ver(ubwcp, ioctl_param);
  2120. case UBWCP_IOCTL_GET_STRIDE_ALIGN:
  2121. return ioctl_get_stride_align(ubwcp, ioctl_param);
  2122. case UBWCP_IOCTL_VALIDATE_STRIDE:
  2123. return ioctl_validate_stride(ubwcp, ioctl_param);
  2124. default:
  2125. ERR("Invalid ioctl_num = %d", ioctl_num);
  2126. return -EINVAL;
  2127. }
  2128. return 0;
  2129. }
  2130. static const struct file_operations ubwcp_fops = {
  2131. .owner = THIS_MODULE,
  2132. .open = ubwcp_open,
  2133. .release = ubwcp_close,
  2134. .unlocked_ioctl = ubwcp_ioctl,
  2135. };
  2136. static int read_err_r_op(void *data, u64 *value)
  2137. {
  2138. struct ubwcp_driver *ubwcp = data;
  2139. *value = ubwcp->read_err_irq_en;
  2140. return 0;
  2141. }
  2142. static int read_err_w_op(void *data, u64 value)
  2143. {
  2144. struct ubwcp_driver *ubwcp = data;
  2145. if (ubwcp->state != UBWCP_STATE_READY)
  2146. return -EPERM;
  2147. if (ubwcp_power(ubwcp, true))
  2148. goto err;
  2149. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, value);
  2150. ubwcp->read_err_irq_en = value;
  2151. if (ubwcp_power(ubwcp, false))
  2152. goto err;
  2153. return 0;
  2154. err:
  2155. ubwcp->state = UBWCP_STATE_FAULT;
  2156. ERR("state set to fault");
  2157. return -1;
  2158. }
  2159. static int write_err_r_op(void *data, u64 *value)
  2160. {
  2161. struct ubwcp_driver *ubwcp = data;
  2162. if (ubwcp->state != UBWCP_STATE_READY)
  2163. return -EPERM;
  2164. *value = ubwcp->write_err_irq_en;
  2165. return 0;
  2166. }
  2167. static int write_err_w_op(void *data, u64 value)
  2168. {
  2169. struct ubwcp_driver *ubwcp = data;
  2170. if (ubwcp->state != UBWCP_STATE_READY)
  2171. return -EPERM;
  2172. if (ubwcp_power(ubwcp, true))
  2173. goto err;
  2174. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, value);
  2175. ubwcp->write_err_irq_en = value;
  2176. if (ubwcp_power(ubwcp, false))
  2177. goto err;
  2178. return 0;
  2179. err:
  2180. ubwcp->state = UBWCP_STATE_FAULT;
  2181. ERR("state set to fault");
  2182. return -1;
  2183. }
  2184. static int decode_err_r_op(void *data, u64 *value)
  2185. {
  2186. struct ubwcp_driver *ubwcp = data;
  2187. if (ubwcp->state != UBWCP_STATE_READY)
  2188. return -EPERM;
  2189. *value = ubwcp->decode_err_irq_en;
  2190. return 0;
  2191. }
  2192. static int decode_err_w_op(void *data, u64 value)
  2193. {
  2194. struct ubwcp_driver *ubwcp = data;
  2195. if (ubwcp->state != UBWCP_STATE_READY)
  2196. return -EPERM;
  2197. if (ubwcp_power(ubwcp, true))
  2198. goto err;
  2199. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, value);
  2200. ubwcp->decode_err_irq_en = value;
  2201. if (ubwcp_power(ubwcp, false))
  2202. goto err;
  2203. return 0;
  2204. err:
  2205. ubwcp->state = UBWCP_STATE_FAULT;
  2206. ERR("state set to fault");
  2207. return -1;
  2208. }
  2209. static int encode_err_r_op(void *data, u64 *value)
  2210. {
  2211. struct ubwcp_driver *ubwcp = data;
  2212. if (ubwcp->state != UBWCP_STATE_READY)
  2213. return -EPERM;
  2214. *value = ubwcp->encode_err_irq_en;
  2215. return 0;
  2216. }
  2217. static int encode_err_w_op(void *data, u64 value)
  2218. {
  2219. struct ubwcp_driver *ubwcp = data;
  2220. if (ubwcp->state != UBWCP_STATE_READY)
  2221. return -EPERM;
  2222. if (ubwcp_power(ubwcp, true))
  2223. goto err;
  2224. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, value);
  2225. ubwcp->encode_err_irq_en = value;
  2226. if (ubwcp_power(ubwcp, false))
  2227. goto err;
  2228. return 0;
  2229. err:
  2230. ubwcp->state = UBWCP_STATE_FAULT;
  2231. ERR("state set to fault");
  2232. return -1;
  2233. }
  2234. static int reg_rw_trace_w_op(void *data, u64 value)
  2235. {
  2236. struct ubwcp_driver *ubwcp = data;
  2237. if (ubwcp->state != UBWCP_STATE_READY)
  2238. return -EPERM;
  2239. ubwcp_hw_trace_set(value);
  2240. return 0;
  2241. }
  2242. static int reg_rw_trace_r_op(void *data, u64 *value)
  2243. {
  2244. struct ubwcp_driver *ubwcp = data;
  2245. bool trace_status;
  2246. if (ubwcp->state != UBWCP_STATE_READY)
  2247. return -EPERM;
  2248. ubwcp_hw_trace_get(&trace_status);
  2249. *value = trace_status;
  2250. return 0;
  2251. }
  2252. static int single_tile_r_op(void *data, u64 *value)
  2253. {
  2254. struct ubwcp_driver *ubwcp = data;
  2255. if (ubwcp->state != UBWCP_STATE_READY)
  2256. return -EPERM;
  2257. *value = ubwcp->single_tile_en;
  2258. return 0;
  2259. }
  2260. static int single_tile_w_op(void *data, u64 value)
  2261. {
  2262. struct ubwcp_driver *ubwcp = data;
  2263. if (ubwcp->state != UBWCP_STATE_READY)
  2264. return -EPERM;
  2265. if (ubwcp_power(ubwcp, true))
  2266. goto err;
  2267. ubwcp_hw_single_tile(ubwcp->base, value);
  2268. ubwcp->single_tile_en = value;
  2269. if (ubwcp_power(ubwcp, false))
  2270. goto err;
  2271. return 0;
  2272. err:
  2273. ubwcp->state = UBWCP_STATE_FAULT;
  2274. ERR("state set to fault");
  2275. return -1;
  2276. }
  2277. DEFINE_DEBUGFS_ATTRIBUTE(read_err_fops, read_err_r_op, read_err_w_op, "%d\n");
  2278. DEFINE_DEBUGFS_ATTRIBUTE(decode_err_fops, decode_err_r_op, decode_err_w_op, "%d\n");
  2279. DEFINE_DEBUGFS_ATTRIBUTE(write_err_fops, write_err_r_op, write_err_w_op, "%d\n");
  2280. DEFINE_DEBUGFS_ATTRIBUTE(encode_err_fops, encode_err_r_op, encode_err_w_op, "%d\n");
  2281. DEFINE_DEBUGFS_ATTRIBUTE(reg_rw_trace_fops, reg_rw_trace_r_op, reg_rw_trace_w_op, "%d\n");
  2282. DEFINE_DEBUGFS_ATTRIBUTE(single_tile_fops, single_tile_r_op, single_tile_w_op, "%d\n");
  2283. static void ubwcp_debugfs_init(struct ubwcp_driver *ubwcp)
  2284. {
  2285. struct dentry *debugfs_root;
  2286. struct dentry *dfile;
  2287. debugfs_root = debugfs_create_dir("ubwcp", NULL);
  2288. if (IS_ERR_OR_NULL(debugfs_root)) {
  2289. ERR("Failed to create debugfs for ubwcp\n");
  2290. return;
  2291. }
  2292. debugfs_create_u32("debug_trace_enable", 0644, debugfs_root, &ubwcp_debug_trace_enable);
  2293. dfile = debugfs_create_file("reg_rw_trace_en", 0644, debugfs_root, ubwcp, &reg_rw_trace_fops);
  2294. if (IS_ERR_OR_NULL(dfile)) {
  2295. ERR("failed to create reg_rw_trace_en debugfs file");
  2296. goto err;
  2297. }
  2298. dfile = debugfs_create_file("read_err_irq_en", 0644, debugfs_root, ubwcp, &read_err_fops);
  2299. if (IS_ERR_OR_NULL(dfile)) {
  2300. ERR("failed to create read_err_irq debugfs file");
  2301. goto err;
  2302. }
  2303. dfile = debugfs_create_file("write_err_irq_en", 0644, debugfs_root, ubwcp, &write_err_fops);
  2304. if (IS_ERR_OR_NULL(dfile)) {
  2305. ERR("failed to create write_err_irq debugfs file");
  2306. goto err;
  2307. }
  2308. dfile = debugfs_create_file("decode_err_irq_en", 0644, debugfs_root, ubwcp,
  2309. &decode_err_fops);
  2310. if (IS_ERR_OR_NULL(dfile)) {
  2311. ERR("failed to create decode_err_irq debugfs file");
  2312. goto err;
  2313. }
  2314. dfile = debugfs_create_file("encode_err_irq_en", 0644, debugfs_root, ubwcp,
  2315. &encode_err_fops);
  2316. if (IS_ERR_OR_NULL(dfile)) {
  2317. ERR("failed to create encode_err_irq debugfs file");
  2318. goto err;
  2319. }
  2320. dfile = debugfs_create_file("single_tile_en", 0644, debugfs_root, ubwcp, &single_tile_fops);
  2321. if (IS_ERR_OR_NULL(dfile)) {
  2322. ERR("failed to create write_err_irq debugfs file");
  2323. goto err;
  2324. }
  2325. ubwcp->debugfs_root = debugfs_root;
  2326. return;
  2327. err:
  2328. debugfs_remove_recursive(ubwcp->debugfs_root);
  2329. ubwcp->debugfs_root = NULL;
  2330. }
  2331. static void ubwcp_debugfs_deinit(struct ubwcp_driver *ubwcp)
  2332. {
  2333. debugfs_remove_recursive(ubwcp->debugfs_root);
  2334. }
  2335. /* ubwcp char device initialization */
  2336. static int ubwcp_cdev_init(struct ubwcp_driver *ubwcp)
  2337. {
  2338. int ret;
  2339. dev_t devt;
  2340. struct class *dev_class;
  2341. struct device *dev_sys;
  2342. /* allocate major device number (/proc/devices -> major_num ubwcp) */
  2343. ret = alloc_chrdev_region(&devt, 0, UBWCP_NUM_DEVICES, UBWCP_DEVICE_NAME);
  2344. if (ret) {
  2345. ERR("alloc_chrdev_region() failed: %d", ret);
  2346. return ret;
  2347. }
  2348. /* create device class (/sys/class/ubwcp_class) */
  2349. dev_class = class_create(THIS_MODULE, "ubwcp_class");
  2350. if (IS_ERR(dev_class)) {
  2351. ret = PTR_ERR(dev_class);
  2352. ERR("class_create() failed, ret: %d", ret);
  2353. goto err;
  2354. }
  2355. /* Create device and register with sysfs
  2356. * (/sys/class/ubwcp_class/ubwcp/... -> dev/power/subsystem/uevent)
  2357. */
  2358. dev_sys = device_create(dev_class, NULL, devt, NULL,
  2359. UBWCP_DEVICE_NAME);
  2360. if (IS_ERR(dev_sys)) {
  2361. ret = PTR_ERR(dev_sys);
  2362. ERR("device_create() failed, ret: %d", ret);
  2363. goto err_device_create;
  2364. }
  2365. /* register file operations and get cdev */
  2366. cdev_init(&ubwcp->cdev, &ubwcp_fops);
  2367. /* associate cdev and device major/minor with file system
  2368. * can do file ops on /dev/ubwcp after this
  2369. */
  2370. ret = cdev_add(&ubwcp->cdev, devt, 1);
  2371. if (ret) {
  2372. ERR("cdev_add() failed, ret: %d", ret);
  2373. goto err_cdev_add;
  2374. }
  2375. ubwcp->devt = devt;
  2376. ubwcp->dev_class = dev_class;
  2377. ubwcp->dev_sys = dev_sys;
  2378. return 0;
  2379. err_cdev_add:
  2380. device_destroy(dev_class, devt);
  2381. err_device_create:
  2382. class_destroy(dev_class);
  2383. err:
  2384. unregister_chrdev_region(devt, UBWCP_NUM_DEVICES);
  2385. return ret;
  2386. }
  2387. static void ubwcp_cdev_deinit(struct ubwcp_driver *ubwcp)
  2388. {
  2389. device_destroy(ubwcp->dev_class, ubwcp->devt);
  2390. class_destroy(ubwcp->dev_class);
  2391. cdev_del(&ubwcp->cdev);
  2392. unregister_chrdev_region(ubwcp->devt, UBWCP_NUM_DEVICES);
  2393. }
  2394. struct handler_node {
  2395. struct list_head list;
  2396. u32 client_id;
  2397. ubwcp_error_handler_t handler;
  2398. void *data;
  2399. };
  2400. int ubwcp_register_error_handler(u32 client_id, ubwcp_error_handler_t handler,
  2401. void *data)
  2402. {
  2403. struct handler_node *node;
  2404. unsigned long flags;
  2405. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2406. if (!ubwcp)
  2407. return -EINVAL;
  2408. if (client_id != -1)
  2409. return -EINVAL;
  2410. if (!handler)
  2411. return -EINVAL;
  2412. if (ubwcp->state != UBWCP_STATE_READY)
  2413. return -EPERM;
  2414. node = kzalloc(sizeof(*node), GFP_KERNEL);
  2415. if (!node)
  2416. return -ENOMEM;
  2417. node->client_id = client_id;
  2418. node->handler = handler;
  2419. node->data = data;
  2420. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  2421. list_add_tail(&node->list, &ubwcp->err_handler_list);
  2422. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  2423. return 0;
  2424. }
  2425. EXPORT_SYMBOL(ubwcp_register_error_handler);
  2426. static void ubwcp_notify_error_handlers(struct ubwcp_err_info *err)
  2427. {
  2428. struct handler_node *node;
  2429. unsigned long flags;
  2430. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2431. if (!ubwcp)
  2432. return;
  2433. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  2434. list_for_each_entry(node, &ubwcp->err_handler_list, list)
  2435. node->handler(err, node->data);
  2436. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  2437. }
  2438. int ubwcp_unregister_error_handler(u32 client_id)
  2439. {
  2440. int ret = -EINVAL;
  2441. struct handler_node *node;
  2442. unsigned long flags;
  2443. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2444. if (!ubwcp)
  2445. return -EINVAL;
  2446. if (ubwcp->state != UBWCP_STATE_INVALID)
  2447. return -EPERM;
  2448. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  2449. list_for_each_entry(node, &ubwcp->err_handler_list, list)
  2450. if (node->client_id == client_id) {
  2451. list_del(&node->list);
  2452. kfree(node);
  2453. ret = 0;
  2454. break;
  2455. }
  2456. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  2457. return ret;
  2458. }
  2459. EXPORT_SYMBOL(ubwcp_unregister_error_handler);
  2460. /* get ubwcp_buf corresponding to the ULA PA*/
  2461. static struct dma_buf *get_dma_buf_from_ulapa(phys_addr_t addr)
  2462. {
  2463. struct ubwcp_buf *buf = NULL;
  2464. struct dma_buf *ret_buf = NULL;
  2465. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2466. unsigned long flags;
  2467. u32 i;
  2468. if (!ubwcp)
  2469. return NULL;
  2470. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  2471. hash_for_each(ubwcp->buf_table, i, buf, hnode) {
  2472. if (buf->ula_pa <= addr && addr < buf->ula_pa + buf->ula_size) {
  2473. ret_buf = buf->dma_buf;
  2474. break;
  2475. }
  2476. }
  2477. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  2478. return ret_buf;
  2479. }
  2480. /* get ubwcp_buf corresponding to the IOVA*/
  2481. static struct dma_buf *get_dma_buf_from_iova(unsigned long addr)
  2482. {
  2483. struct ubwcp_buf *buf = NULL;
  2484. struct dma_buf *ret_buf = NULL;
  2485. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2486. unsigned long flags;
  2487. u32 i;
  2488. if (!ubwcp)
  2489. return NULL;
  2490. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  2491. hash_for_each(ubwcp->buf_table, i, buf, hnode) {
  2492. unsigned long iova_base;
  2493. unsigned int iova_size;
  2494. if (!buf->sgt)
  2495. continue;
  2496. iova_base = sg_dma_address(buf->sgt->sgl);
  2497. iova_size = sg_dma_len(buf->sgt->sgl);
  2498. if (iova_base <= addr && addr < iova_base + iova_size) {
  2499. ret_buf = buf->dma_buf;
  2500. break;
  2501. }
  2502. }
  2503. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  2504. return ret_buf;
  2505. }
  2506. int ubwcp_iommu_fault_handler(struct iommu_domain *domain, struct device *dev,
  2507. unsigned long iova, int flags, void *data)
  2508. {
  2509. int ret = 0;
  2510. struct ubwcp_err_info err;
  2511. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2512. struct device *cb_dev = (struct device *)data;
  2513. if (!ubwcp) {
  2514. ret = -EINVAL;
  2515. goto err;
  2516. }
  2517. err.err_code = UBWCP_SMMU_FAULT;
  2518. if (cb_dev == ubwcp->dev_desc_cb)
  2519. err.smmu_err.iommu_dev_id = UBWCP_DESC_CB_ID;
  2520. else if (cb_dev == ubwcp->dev_buf_cb)
  2521. err.smmu_err.iommu_dev_id = UBWCP_BUF_CB_ID;
  2522. else
  2523. err.smmu_err.iommu_dev_id = UBWCP_UNKNOWN_CB_ID;
  2524. err.smmu_err.dmabuf = get_dma_buf_from_iova(iova);
  2525. err.smmu_err.iova = iova;
  2526. err.smmu_err.iommu_fault_flags = flags;
  2527. ERR("ubwcp_err: err code: %d (smmu), iommu_dev_id: %d, iova: 0x%llx, flags: 0x%x",
  2528. err.err_code, err.smmu_err.iommu_dev_id, err.smmu_err.iova,
  2529. err.smmu_err.iommu_fault_flags);
  2530. ubwcp_notify_error_handlers(&err);
  2531. err:
  2532. return ret;
  2533. }
  2534. static irqreturn_t ubwcp_irq_handler(int irq, void *ptr)
  2535. {
  2536. struct ubwcp_driver *ubwcp;
  2537. void __iomem *base;
  2538. phys_addr_t addr;
  2539. struct ubwcp_err_info err;
  2540. ubwcp = (struct ubwcp_driver *) ptr;
  2541. base = ubwcp->base;
  2542. if (irq == ubwcp->irq_range_ck_rd) {
  2543. addr = ubwcp_hw_interrupt_src_address(base, 0) << 6;
  2544. err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
  2545. err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2546. err.translation_err.ula_pa = addr;
  2547. err.translation_err.read = true;
  2548. ERR("err_code: %d (range read), dmabuf: 0x%llx, read: %d, addr: 0x%llx",
  2549. err.err_code, err.translation_err.dmabuf, err.translation_err.read, addr);
  2550. ubwcp_notify_error_handlers(&err);
  2551. ubwcp_hw_interrupt_clear(ubwcp->base, 0);
  2552. } else if (irq == ubwcp->irq_range_ck_wr) {
  2553. addr = ubwcp_hw_interrupt_src_address(base, 1) << 6;
  2554. err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
  2555. err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2556. err.translation_err.ula_pa = addr;
  2557. err.translation_err.read = false;
  2558. ERR("err_code: %d (range write), dmabuf: 0x%llx, read: %d, addr: 0x%llx",
  2559. err.err_code, err.translation_err.dmabuf, err.translation_err.read, addr);
  2560. ubwcp_notify_error_handlers(&err);
  2561. ubwcp_hw_interrupt_clear(ubwcp->base, 1);
  2562. } else if (irq == ubwcp->irq_encode) {
  2563. addr = ubwcp_hw_interrupt_src_address(base, 3) << 6;
  2564. err.err_code = UBWCP_ENCODE_ERROR;
  2565. err.enc_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2566. err.enc_err.ula_pa = addr;
  2567. ERR("err_code: %d (encode), dmabuf: 0x%llx, addr: 0x%llx",
  2568. err.err_code, err.enc_err.dmabuf, addr);
  2569. ubwcp_notify_error_handlers(&err);
  2570. ubwcp_hw_interrupt_clear(ubwcp->base, 3);
  2571. } else if (irq == ubwcp->irq_decode) {
  2572. addr = ubwcp_hw_interrupt_src_address(base, 2) << 6;
  2573. err.err_code = UBWCP_DECODE_ERROR;
  2574. err.dec_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2575. err.dec_err.ula_pa = addr;
  2576. ERR("err_code: %d (decode), dmabuf: 0x%llx, addr: 0x%llx",
  2577. err.err_code, err.enc_err.dmabuf, addr);
  2578. ubwcp_notify_error_handlers(&err);
  2579. ubwcp_hw_interrupt_clear(ubwcp->base, 2);
  2580. } else {
  2581. ERR("unknown irq: %d", irq);
  2582. return IRQ_NONE;
  2583. }
  2584. return IRQ_HANDLED;
  2585. }
  2586. static int ubwcp_interrupt_register(struct platform_device *pdev, struct ubwcp_driver *ubwcp)
  2587. {
  2588. int ret = 0;
  2589. struct device *dev = &pdev->dev;
  2590. ubwcp->irq_range_ck_rd = platform_get_irq(pdev, 0);
  2591. if (ubwcp->irq_range_ck_rd < 0)
  2592. return ubwcp->irq_range_ck_rd;
  2593. ubwcp->irq_range_ck_wr = platform_get_irq(pdev, 1);
  2594. if (ubwcp->irq_range_ck_wr < 0)
  2595. return ubwcp->irq_range_ck_wr;
  2596. ubwcp->irq_encode = platform_get_irq(pdev, 2);
  2597. if (ubwcp->irq_encode < 0)
  2598. return ubwcp->irq_encode;
  2599. ubwcp->irq_decode = platform_get_irq(pdev, 3);
  2600. if (ubwcp->irq_decode < 0)
  2601. return ubwcp->irq_decode;
  2602. DBG("got irqs: %d %d %d %d", ubwcp->irq_range_ck_rd,
  2603. ubwcp->irq_range_ck_wr,
  2604. ubwcp->irq_encode,
  2605. ubwcp->irq_decode);
  2606. ret = devm_request_irq(dev, ubwcp->irq_range_ck_rd, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2607. if (ret) {
  2608. ERR("request_irq() failed. irq: %d ret: %d",
  2609. ubwcp->irq_range_ck_rd, ret);
  2610. return ret;
  2611. }
  2612. ret = devm_request_irq(dev, ubwcp->irq_range_ck_wr, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2613. if (ret) {
  2614. ERR("request_irq() failed. irq: %d ret: %d",
  2615. ubwcp->irq_range_ck_wr, ret);
  2616. return ret;
  2617. }
  2618. ret = devm_request_irq(dev, ubwcp->irq_encode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2619. if (ret) {
  2620. ERR("request_irq() failed. irq: %d ret: %d",
  2621. ubwcp->irq_encode, ret);
  2622. return ret;
  2623. }
  2624. ret = devm_request_irq(dev, ubwcp->irq_decode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2625. if (ret) {
  2626. ERR("request_irq() failed. irq: %d ret: %d",
  2627. ubwcp->irq_decode, ret);
  2628. return ret;
  2629. }
  2630. return ret;
  2631. }
  2632. /* ubwcp device probe */
  2633. static int qcom_ubwcp_probe(struct platform_device *pdev)
  2634. {
  2635. int ret = 0;
  2636. struct ubwcp_driver *ubwcp;
  2637. struct device *ubwcp_dev = &pdev->dev;
  2638. ubwcp = devm_kzalloc(ubwcp_dev, sizeof(*ubwcp), GFP_KERNEL);
  2639. if (!ubwcp) {
  2640. ERR("devm_kzalloc() failed");
  2641. return -ENOMEM;
  2642. }
  2643. ubwcp->dev = &pdev->dev;
  2644. ret = dma_set_mask_and_coherent(ubwcp->dev, DMA_BIT_MASK(64));
  2645. ubwcp->base = devm_platform_ioremap_resource(pdev, 0);
  2646. if (IS_ERR(ubwcp->base)) {
  2647. ERR("devm ioremap() failed: %d", PTR_ERR(ubwcp->base));
  2648. return PTR_ERR(ubwcp->base);
  2649. }
  2650. DBG("ubwcp->base: %p", ubwcp->base);
  2651. ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 0, &ubwcp->ula_pool_base);
  2652. if (ret) {
  2653. ERR("failed reading ula_range (base): %d", ret);
  2654. return ret;
  2655. }
  2656. DBG("ubwcp: ula_range: base = 0x%lx", ubwcp->ula_pool_base);
  2657. ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 1, &ubwcp->ula_pool_size);
  2658. if (ret) {
  2659. ERR("failed reading ula_range (size): %d", ret);
  2660. return ret;
  2661. }
  2662. DBG("ubwcp: ula_range: size = 0x%lx", ubwcp->ula_pool_size);
  2663. INIT_LIST_HEAD(&ubwcp->err_handler_list);
  2664. /* driver initial state */
  2665. ubwcp->state = UBWCP_STATE_INVALID;
  2666. atomic_set(&ubwcp->num_non_lin_buffers, 0);
  2667. ubwcp->mem_online = false;
  2668. mutex_init(&ubwcp->desc_lock);
  2669. spin_lock_init(&ubwcp->buf_table_lock);
  2670. mutex_init(&ubwcp->mem_hotplug_lock);
  2671. mutex_init(&ubwcp->ula_lock);
  2672. mutex_init(&ubwcp->ubwcp_flush_lock);
  2673. mutex_init(&ubwcp->hw_range_ck_lock);
  2674. mutex_init(&ubwcp->power_ctrl_lock);
  2675. spin_lock_init(&ubwcp->err_handler_list_lock);
  2676. /* Regulator */
  2677. ubwcp->vdd = devm_regulator_get(ubwcp_dev, "vdd");
  2678. if (IS_ERR_OR_NULL(ubwcp->vdd)) {
  2679. ret = PTR_ERR(ubwcp->vdd);
  2680. ERR("devm_regulator_get() failed: %d", ret);
  2681. return ret;
  2682. }
  2683. ret = ubwcp_init_clocks(ubwcp, ubwcp_dev);
  2684. if (ret) {
  2685. ERR("failed to initialize ubwcp clocks err: %d", ret);
  2686. return ret;
  2687. }
  2688. if (ubwcp_power(ubwcp, true))
  2689. return -1;
  2690. if (ubwcp_cdev_init(ubwcp))
  2691. return -1;
  2692. /* disable all interrupts (reset value has some interrupts enabled by default) */
  2693. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2694. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2695. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2696. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2697. if (ubwcp_interrupt_register(pdev, ubwcp))
  2698. return -1;
  2699. ubwcp_debugfs_init(ubwcp);
  2700. /* create ULA pool */
  2701. ubwcp->ula_pool = gen_pool_create(PAGE_SHIFT, -1);
  2702. if (!ubwcp->ula_pool) {
  2703. ERR("failed gen_pool_create()");
  2704. ret = -1;
  2705. goto err_pool_create;
  2706. }
  2707. ret = gen_pool_add(ubwcp->ula_pool, ubwcp->ula_pool_base, ubwcp->ula_pool_size, -1);
  2708. if (ret) {
  2709. ERR("failed gen_pool_add(): %d", ret);
  2710. ret = -1;
  2711. goto err_pool_add;
  2712. }
  2713. /* register the default config mmap function. */
  2714. ubwcp->mmap_config_fptr = msm_ubwcp_dma_buf_configure_mmap;
  2715. hash_init(ubwcp->buf_table);
  2716. ubwcp_buf_desc_list_init(ubwcp);
  2717. image_format_init(ubwcp);
  2718. /* one time hw init */
  2719. ubwcp_hw_one_time_init(ubwcp->base);
  2720. ubwcp_hw_version(ubwcp->base, &ubwcp->hw_ver_major, &ubwcp->hw_ver_minor);
  2721. pr_err("ubwcp: hw version: major %d, minor %d\n", ubwcp->hw_ver_major, ubwcp->hw_ver_minor);
  2722. if (ubwcp->hw_ver_major == 0) {
  2723. ERR("Failed to read HW version");
  2724. ret = -1;
  2725. goto err_pool_add;
  2726. }
  2727. /* set pdev->dev->driver_data = ubwcp */
  2728. platform_set_drvdata(pdev, ubwcp);
  2729. /* enable interrupts */
  2730. if (ubwcp->read_err_irq_en)
  2731. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, true);
  2732. if (ubwcp->write_err_irq_en)
  2733. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, true);
  2734. if (ubwcp->decode_err_irq_en)
  2735. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, true);
  2736. if (ubwcp->encode_err_irq_en)
  2737. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, true);
  2738. /* Turn OFF until buffers are allocated */
  2739. if (ubwcp_power(ubwcp, false)) {
  2740. ret = -1;
  2741. goto err_power_off;
  2742. }
  2743. ret = msm_ubwcp_set_ops(ubwcp_init_buffer, ubwcp_free_buffer, ubwcp_lock, ubwcp_unlock);
  2744. if (ret) {
  2745. ERR("msm_ubwcp_set_ops() failed: %d", ret);
  2746. goto err_power_off;
  2747. } else {
  2748. DBG("msm_ubwcp_set_ops(): success"); }
  2749. me = ubwcp;
  2750. return ret;
  2751. err_power_off:
  2752. if (!ubwcp_power(ubwcp, true)) {
  2753. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2754. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2755. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2756. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2757. ubwcp_power(ubwcp, false);
  2758. }
  2759. err_pool_add:
  2760. gen_pool_destroy(ubwcp->ula_pool);
  2761. err_pool_create:
  2762. ubwcp_debugfs_deinit(ubwcp);
  2763. ubwcp_cdev_deinit(ubwcp);
  2764. return ret;
  2765. }
  2766. /* buffer context bank device probe */
  2767. static int ubwcp_probe_cb_buf(struct platform_device *pdev)
  2768. {
  2769. struct ubwcp_driver *ubwcp;
  2770. struct iommu_domain *domain = NULL;
  2771. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2772. if (!ubwcp) {
  2773. ERR("failed to get ubwcp ptr");
  2774. return -EINVAL;
  2775. }
  2776. ubwcp->dev_buf_cb = &pdev->dev;
  2777. domain = iommu_get_domain_for_dev(ubwcp->dev_buf_cb);
  2778. if (domain)
  2779. iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_buf_cb);
  2780. if (ubwcp->dev_desc_cb)
  2781. ubwcp->state = UBWCP_STATE_READY;
  2782. return 0;
  2783. }
  2784. /* descriptor context bank device probe */
  2785. static int ubwcp_probe_cb_desc(struct platform_device *pdev)
  2786. {
  2787. int ret = 0;
  2788. struct ubwcp_driver *ubwcp;
  2789. struct iommu_domain *domain = NULL;
  2790. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2791. if (!ubwcp) {
  2792. ERR("failed to get ubwcp ptr");
  2793. return -EINVAL;
  2794. }
  2795. ubwcp->buffer_desc_size = UBWCP_BUFFER_DESC_OFFSET *
  2796. UBWCP_BUFFER_DESC_COUNT;
  2797. ubwcp->dev_desc_cb = &pdev->dev;
  2798. dma_set_max_seg_size(ubwcp->dev_desc_cb, DMA_BIT_MASK(32));
  2799. dma_set_seg_boundary(ubwcp->dev_desc_cb, (unsigned long)DMA_BIT_MASK(64));
  2800. /* Allocate buffer descriptors. UBWCP is iocoherent device.
  2801. * Thus we don't need to flush after updates to buffer descriptors.
  2802. */
  2803. ubwcp->buffer_desc_base = dma_alloc_coherent(ubwcp->dev_desc_cb,
  2804. ubwcp->buffer_desc_size,
  2805. &ubwcp->buffer_desc_dma_handle,
  2806. GFP_KERNEL);
  2807. if (!ubwcp->buffer_desc_base) {
  2808. ERR("failed to allocate desc buffer");
  2809. return -ENOMEM;
  2810. }
  2811. DBG("desc_base = %p size = %zu", ubwcp->buffer_desc_base,
  2812. ubwcp->buffer_desc_size);
  2813. ret = ubwcp_power(ubwcp, true);
  2814. if (ret) {
  2815. ERR("failed to power on");
  2816. goto err;
  2817. }
  2818. ubwcp_hw_set_buf_desc(ubwcp->base, (u64) ubwcp->buffer_desc_dma_handle,
  2819. UBWCP_BUFFER_DESC_OFFSET);
  2820. ret = ubwcp_power(ubwcp, false);
  2821. if (ret) {
  2822. ERR("failed to power off");
  2823. goto err;
  2824. }
  2825. domain = iommu_get_domain_for_dev(ubwcp->dev_desc_cb);
  2826. if (domain)
  2827. iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_desc_cb);
  2828. if (ubwcp->dev_buf_cb)
  2829. ubwcp->state = UBWCP_STATE_READY;
  2830. return ret;
  2831. err:
  2832. dma_free_coherent(ubwcp->dev_desc_cb,
  2833. ubwcp->buffer_desc_size,
  2834. ubwcp->buffer_desc_base,
  2835. ubwcp->buffer_desc_dma_handle);
  2836. ubwcp->buffer_desc_base = NULL;
  2837. ubwcp->buffer_desc_dma_handle = 0;
  2838. ubwcp->dev_desc_cb = NULL;
  2839. return -1;
  2840. }
  2841. /* buffer context bank device remove */
  2842. static int ubwcp_remove_cb_buf(struct platform_device *pdev)
  2843. {
  2844. struct ubwcp_driver *ubwcp;
  2845. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2846. if (!ubwcp) {
  2847. ERR("failed to get ubwcp ptr");
  2848. return -EINVAL;
  2849. }
  2850. ubwcp->state = UBWCP_STATE_INVALID;
  2851. ubwcp->dev_buf_cb = NULL;
  2852. return 0;
  2853. }
  2854. /* descriptor context bank device remove */
  2855. static int ubwcp_remove_cb_desc(struct platform_device *pdev)
  2856. {
  2857. struct ubwcp_driver *ubwcp;
  2858. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2859. if (!ubwcp) {
  2860. ERR("failed to get ubwcp ptr");
  2861. return -EINVAL;
  2862. }
  2863. if (!ubwcp->dev_desc_cb) {
  2864. ERR("ubwcp->dev_desc_cb == NULL");
  2865. return -1;
  2866. }
  2867. if (!ubwcp_power(ubwcp, true)) {
  2868. ubwcp_hw_set_buf_desc(ubwcp->base, 0x0, 0x0);
  2869. ubwcp_power(ubwcp, false);
  2870. }
  2871. ubwcp->state = UBWCP_STATE_INVALID;
  2872. dma_free_coherent(ubwcp->dev_desc_cb,
  2873. ubwcp->buffer_desc_size,
  2874. ubwcp->buffer_desc_base,
  2875. ubwcp->buffer_desc_dma_handle);
  2876. ubwcp->buffer_desc_base = NULL;
  2877. ubwcp->buffer_desc_dma_handle = 0;
  2878. return 0;
  2879. }
  2880. /* ubwcp device remove */
  2881. static int qcom_ubwcp_remove(struct platform_device *pdev)
  2882. {
  2883. size_t avail;
  2884. size_t psize;
  2885. struct ubwcp_driver *ubwcp;
  2886. /* get pdev->dev->driver_data = ubwcp */
  2887. ubwcp = platform_get_drvdata(pdev);
  2888. if (!ubwcp) {
  2889. ERR("ubwcp == NULL");
  2890. return -1;
  2891. }
  2892. if (!ubwcp_power(ubwcp, true)) {
  2893. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2894. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2895. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2896. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2897. ubwcp_power(ubwcp, false);
  2898. }
  2899. ubwcp->state = UBWCP_STATE_INVALID;
  2900. /* before destroying, make sure pool is empty. otherwise pool_destroy() panics. */
  2901. avail = gen_pool_avail(ubwcp->ula_pool);
  2902. psize = gen_pool_size(ubwcp->ula_pool);
  2903. if (psize != avail) {
  2904. ERR("gen_pool is not empty! avail: %zx size: %zx", avail, psize);
  2905. ERR("skipping pool destroy....cause it will PANIC. Fix this!!!!");
  2906. } else {
  2907. gen_pool_destroy(ubwcp->ula_pool);
  2908. }
  2909. ubwcp_debugfs_deinit(ubwcp);
  2910. ubwcp_cdev_deinit(ubwcp);
  2911. return 0;
  2912. }
  2913. /* top level ubwcp device probe function */
  2914. static int ubwcp_probe(struct platform_device *pdev)
  2915. {
  2916. const char *compatible = "";
  2917. trace_ubwcp_probe(pdev);
  2918. if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
  2919. return qcom_ubwcp_probe(pdev);
  2920. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
  2921. return ubwcp_probe_cb_desc(pdev);
  2922. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
  2923. return ubwcp_probe_cb_buf(pdev);
  2924. of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
  2925. ERR("unknown device: %s", compatible);
  2926. return -EINVAL;
  2927. }
  2928. /* top level ubwcp device remove function */
  2929. static int ubwcp_remove(struct platform_device *pdev)
  2930. {
  2931. const char *compatible = "";
  2932. trace_ubwcp_remove(pdev);
  2933. /* TBD: what if buffers are still allocated? locked? etc.
  2934. * also should turn off power?
  2935. */
  2936. if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
  2937. return qcom_ubwcp_remove(pdev);
  2938. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
  2939. return ubwcp_remove_cb_desc(pdev);
  2940. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
  2941. return ubwcp_remove_cb_buf(pdev);
  2942. of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
  2943. ERR("unknown device: %s", compatible);
  2944. return -EINVAL;
  2945. }
  2946. static const struct of_device_id ubwcp_dt_match[] = {
  2947. {.compatible = "qcom,ubwcp"},
  2948. {.compatible = "qcom,ubwcp-context-bank-desc"},
  2949. {.compatible = "qcom,ubwcp-context-bank-buf"},
  2950. {}
  2951. };
  2952. struct platform_driver ubwcp_platform_driver = {
  2953. .probe = ubwcp_probe,
  2954. .remove = ubwcp_remove,
  2955. .driver = {
  2956. .name = "qcom,ubwcp",
  2957. .of_match_table = ubwcp_dt_match,
  2958. },
  2959. };
  2960. int ubwcp_init(void)
  2961. {
  2962. int ret = 0;
  2963. DBG("+++++++++++");
  2964. ret = platform_driver_register(&ubwcp_platform_driver);
  2965. if (ret)
  2966. ERR("platform_driver_register() failed: %d", ret);
  2967. return ret;
  2968. }
  2969. void ubwcp_exit(void)
  2970. {
  2971. platform_driver_unregister(&ubwcp_platform_driver);
  2972. DBG("-----------");
  2973. }
  2974. module_init(ubwcp_init);
  2975. module_exit(ubwcp_exit);
  2976. MODULE_LICENSE("GPL");