ubwcp_main.c 85 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/kernel.h>
  7. #include <linux/dma-buf.h>
  8. #include <linux/slab.h>
  9. #include <linux/cdev.h>
  10. #include <linux/hashtable.h>
  11. #include <linux/scatterlist.h>
  12. #include <linux/types.h>
  13. #include <linux/platform_device.h>
  14. #include <linux/of.h>
  15. #include <linux/of_platform.h>
  16. #include <linux/of_address.h>
  17. #include <linux/genalloc.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/regulator/consumer.h>
  20. #include <linux/numa.h>
  21. #include <linux/memory_hotplug.h>
  22. #include <asm/page.h>
  23. #include <linux/delay.h>
  24. #include <linux/ubwcp_dma_heap.h>
  25. #include <linux/debugfs.h>
  26. #include <linux/clk.h>
  27. #include <linux/iommu.h>
  28. #include <linux/set_memory.h>
  29. #include <linux/range.h>
  30. MODULE_IMPORT_NS(DMA_BUF);
  31. #include "include/kernel/ubwcp.h"
  32. #include "ubwcp_hw.h"
  33. #include "include/uapi/ubwcp_ioctl.h"
  34. #define CREATE_TRACE_POINTS
  35. #include "ubwcp_trace.h"
  36. #define UBWCP_NUM_DEVICES 1
  37. #define UBWCP_DEVICE_NAME "ubwcp"
  38. #define UBWCP_BUFFER_DESC_OFFSET 64
  39. #define UBWCP_BUFFER_DESC_COUNT 256
  40. #define CACHE_ADDR(x) ((x) >> 6)
  41. #define PAGE_ADDR(x) ((x) >> 12)
  42. #define UBWCP_ALIGN(_x, _y) ((((_x) + (_y) - 1)/(_y))*(_y))
  43. #define DBG_BUF_ATTR(fmt, args...) do { if (ubwcp_debug_trace_enable) \
  44. pr_err("ubwcp: %s(): " fmt "\n", __func__, ##args); \
  45. } while (0)
  46. #define DBG(fmt, args...) do { if (ubwcp_debug_trace_enable) \
  47. pr_err("ubwcp: %s(): " fmt "\n", __func__, ##args); \
  48. } while (0)
  49. #define ERR(fmt, args...) pr_err("ubwcp: %s(): ~~~ERROR~~~: " fmt "\n", __func__, ##args)
  50. #define ERR_RATE_LIMIT(fmt, args...) pr_err_ratelimited("ubwcp: %s(): ~~~ERROR~~~: " fmt "\n",\
  51. __func__, ##args)
  52. #define FENTRY() DBG("")
  53. #define META_DATA_PITCH_ALIGN 64
  54. #define META_DATA_HEIGHT_ALIGN 16
  55. #define META_DATA_SIZE_ALIGN 4096
  56. #define PIXEL_DATA_SIZE_ALIGN 4096
  57. #define UBWCP_SYNC_GRANULE 0x4000000L /* 64 MB */
  58. struct ubwcp_desc {
  59. int idx;
  60. void *ptr;
  61. };
  62. /* TBD: confirm size of width/height */
  63. struct ubwcp_dimension {
  64. u16 width;
  65. u16 height;
  66. };
  67. struct ubwcp_plane_info {
  68. u16 pixel_bytes;
  69. u16 per_pixel;
  70. struct ubwcp_dimension tilesize_p; /* pixels */
  71. struct ubwcp_dimension macrotilesize_p; /* pixels */
  72. };
  73. struct ubwcp_image_format_info {
  74. u16 planes;
  75. struct ubwcp_plane_info p_info[2];
  76. };
  77. enum ubwcp_std_image_format {
  78. RGBA = 0,
  79. NV12 = 1,
  80. NV124R = 2,
  81. P010 = 3,
  82. TP10 = 4,
  83. P016 = 5,
  84. INFO_FORMAT_LIST_SIZE,
  85. STD_IMAGE_FORMAT_INVALID = 0xFF
  86. };
  87. struct ubwcp_driver {
  88. /* cdev related */
  89. dev_t devt;
  90. struct class *dev_class; //sysfs dev class
  91. struct device *dev_sys; //sysfs dev
  92. struct cdev cdev; //char dev
  93. /* debugfs */
  94. struct dentry *debugfs_root;
  95. bool read_err_irq_en;
  96. bool write_err_irq_en;
  97. bool decode_err_irq_en;
  98. bool encode_err_irq_en;
  99. /* ubwcp devices */
  100. struct device *dev; //ubwcp device
  101. struct device *dev_desc_cb; //smmu dev for descriptors
  102. struct device *dev_buf_cb; //smmu dev for ubwcp buffers
  103. void __iomem *base; //ubwcp base address
  104. struct regulator *vdd;
  105. struct clk **clocks;
  106. int num_clocks;
  107. /* interrupts */
  108. int irq_range_ck_rd;
  109. int irq_range_ck_wr;
  110. int irq_encode;
  111. int irq_decode;
  112. /* ula address pool */
  113. u64 ula_pool_base;
  114. u64 ula_pool_size;
  115. struct gen_pool *ula_pool;
  116. configure_mmap mmap_config_fptr;
  117. /* HW version */
  118. u32 hw_ver_major;
  119. u32 hw_ver_minor;
  120. /* keep track of all potential buffers.
  121. * hash table index'ed using dma_buf ptr.
  122. * 2**13 = 8192 hash values
  123. */
  124. DECLARE_HASHTABLE(buf_table, 13);
  125. /* buffer descriptor */
  126. void *buffer_desc_base; /* CPU address */
  127. dma_addr_t buffer_desc_dma_handle; /* dma address */
  128. size_t buffer_desc_size;
  129. struct ubwcp_desc desc_list[UBWCP_BUFFER_DESC_COUNT];
  130. struct ubwcp_image_format_info format_info[INFO_FORMAT_LIST_SIZE];
  131. atomic_t num_non_lin_buffers;
  132. bool mem_online;
  133. struct mutex desc_lock; /* allocate/free descriptors */
  134. spinlock_t buf_table_lock; /* add/remove dma_buf into list of managed bufffers */
  135. struct mutex mem_hotplug_lock; /* memory hotplug lock */
  136. struct mutex ula_lock; /* allocate/free ula */
  137. struct mutex ubwcp_flush_lock; /* ubwcp flush */
  138. struct mutex hw_range_ck_lock; /* range ck */
  139. struct list_head err_handler_list; /* error handler list */
  140. spinlock_t err_handler_list_lock; /* err_handler_list lock */
  141. struct dev_pagemap pgmap;
  142. };
  143. struct ubwcp_buf {
  144. struct hlist_node hnode;
  145. struct ubwcp_driver *ubwcp;
  146. struct ubwcp_buffer_attrs buf_attr;
  147. bool perm;
  148. struct ubwcp_desc *desc;
  149. bool buf_attr_set;
  150. bool locked;
  151. enum dma_data_direction lock_dir;
  152. int lock_count;
  153. /* dma_buf info */
  154. struct dma_buf *dma_buf;
  155. struct dma_buf_attachment *attachment;
  156. struct sg_table *sgt;
  157. /* ula info */
  158. phys_addr_t ula_pa;
  159. size_t ula_size;
  160. /* meta metadata */
  161. struct ubwcp_hw_meta_metadata mmdata;
  162. struct mutex lock;
  163. };
  164. static struct ubwcp_driver *me;
  165. static u32 ubwcp_debug_trace_enable;
  166. static struct ubwcp_driver *ubwcp_get_driver(void)
  167. {
  168. if (!me)
  169. WARN(1, "ubwcp: driver ptr requested but driver not initialized");
  170. return me;
  171. }
  172. static void image_format_init(struct ubwcp_driver *ubwcp)
  173. { /* planes, bytes/p, Tp , MTp */
  174. ubwcp->format_info[RGBA] = (struct ubwcp_image_format_info)
  175. {1, {{4, 1, {16, 4}, {64, 16}}}};
  176. ubwcp->format_info[NV12] = (struct ubwcp_image_format_info)
  177. {2, {{1, 1, {32, 8}, {128, 32}},
  178. {2, 1, {16, 8}, { 64, 32}}}};
  179. ubwcp->format_info[NV124R] = (struct ubwcp_image_format_info)
  180. {2, {{1, 1, {64, 4}, {256, 16}},
  181. {2, 1, {32, 4}, {128, 16}}}};
  182. ubwcp->format_info[P010] = (struct ubwcp_image_format_info)
  183. {2, {{2, 1, {32, 4}, {128, 16}},
  184. {4, 1, {16, 4}, { 64, 16}}}};
  185. ubwcp->format_info[TP10] = (struct ubwcp_image_format_info)
  186. {2, {{4, 3, {48, 4}, {192, 16}},
  187. {8, 3, {24, 4}, { 96, 16}}}};
  188. ubwcp->format_info[P016] = (struct ubwcp_image_format_info)
  189. {2, {{2, 1, {32, 4}, {128, 16}},
  190. {4, 1, {16, 4}, { 64, 16}}}};
  191. }
  192. static void ubwcp_buf_desc_list_init(struct ubwcp_driver *ubwcp)
  193. {
  194. int idx;
  195. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  196. for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
  197. desc_list[idx].idx = -1;
  198. desc_list[idx].ptr = NULL;
  199. }
  200. }
  201. static int ubwcp_init_clocks(struct ubwcp_driver *ubwcp, struct device *dev)
  202. {
  203. const char *cname;
  204. struct property *prop;
  205. int i;
  206. ubwcp->num_clocks =
  207. of_property_count_strings(dev->of_node, "clock-names");
  208. if (ubwcp->num_clocks < 1) {
  209. ubwcp->num_clocks = 0;
  210. return 0;
  211. }
  212. ubwcp->clocks = devm_kzalloc(dev,
  213. sizeof(*ubwcp->clocks) * ubwcp->num_clocks, GFP_KERNEL);
  214. if (!ubwcp->clocks)
  215. return -ENOMEM;
  216. i = 0;
  217. of_property_for_each_string(dev->of_node, "clock-names",
  218. prop, cname) {
  219. struct clk *c = devm_clk_get(dev, cname);
  220. if (IS_ERR(c)) {
  221. ERR("Couldn't get clock: %s\n", cname);
  222. return PTR_ERR(c);
  223. }
  224. ubwcp->clocks[i] = c;
  225. ++i;
  226. }
  227. return 0;
  228. }
  229. static int ubwcp_enable_clocks(struct ubwcp_driver *ubwcp)
  230. {
  231. int i, ret = 0;
  232. for (i = 0; i < ubwcp->num_clocks; ++i) {
  233. ret = clk_prepare_enable(ubwcp->clocks[i]);
  234. if (ret) {
  235. ERR("Couldn't enable clock #%d\n", i);
  236. while (i--)
  237. clk_disable_unprepare(ubwcp->clocks[i]);
  238. break;
  239. }
  240. }
  241. return ret;
  242. }
  243. static void ubwcp_disable_clocks(struct ubwcp_driver *ubwcp)
  244. {
  245. int i;
  246. for (i = ubwcp->num_clocks; i; --i)
  247. clk_disable_unprepare(ubwcp->clocks[i - 1]);
  248. }
  249. /* UBWCP Power control */
  250. static int ubwcp_power(struct ubwcp_driver *ubwcp, bool enable)
  251. {
  252. int ret = 0;
  253. if (!ubwcp) {
  254. ERR("ubwcp ptr is NULL");
  255. return -1;
  256. }
  257. if (!ubwcp->vdd) {
  258. ERR("vdd is NULL");
  259. return -1;
  260. }
  261. if (enable) {
  262. ret = regulator_enable(ubwcp->vdd);
  263. if (ret < 0) {
  264. ERR("regulator_enable failed: %d", ret);
  265. ret = -1;
  266. } else {
  267. DBG("regulator_enable() success");
  268. }
  269. if (!ret) {
  270. ret = ubwcp_enable_clocks(ubwcp);
  271. if (ret) {
  272. ERR("enable clocks failed: %d", ret);
  273. regulator_disable(ubwcp->vdd);
  274. } else {
  275. DBG("enable clocks success");
  276. }
  277. }
  278. } else {
  279. ret = regulator_disable(ubwcp->vdd);
  280. if (ret < 0) {
  281. ERR("regulator_disable failed: %d", ret);
  282. ret = -1;
  283. } else {
  284. DBG("regulator_disable() success");
  285. }
  286. if (!ret) {
  287. ubwcp_disable_clocks(ubwcp);
  288. DBG("disable clocks success");
  289. }
  290. }
  291. return ret;
  292. }
  293. static int ubwcp_flush(struct ubwcp_driver *ubwcp)
  294. {
  295. int ret = 0;
  296. mutex_lock(&ubwcp->ubwcp_flush_lock);
  297. ret = ubwcp_hw_flush(ubwcp->base);
  298. mutex_unlock(&ubwcp->ubwcp_flush_lock);
  299. if (ret != 0)
  300. WARN(1, "ubwcp_hw_flush() failed!");
  301. return ret;
  302. }
  303. /* get dma_buf ptr for the given dma_buf fd */
  304. struct dma_buf *ubwcp_dma_buf_fd_to_dma_buf(int dma_buf_fd)
  305. {
  306. struct dma_buf *dmabuf;
  307. /* TBD: dma_buf_get() results in taking ref to buf and it won't ever get
  308. * free'ed until ref count goes to 0. So we must reduce the ref count
  309. * immediately after we find our corresponding ubwcp_buf.
  310. */
  311. dmabuf = dma_buf_get(dma_buf_fd);
  312. if (IS_ERR(dmabuf)) {
  313. ERR("dmabuf ptr not found for dma_buf_fd = %d", dma_buf_fd);
  314. return NULL;
  315. }
  316. dma_buf_put(dmabuf);
  317. return dmabuf;
  318. }
  319. EXPORT_SYMBOL(ubwcp_dma_buf_fd_to_dma_buf);
  320. /* get ubwcp_buf corresponding to the given dma_buf */
  321. static struct ubwcp_buf *dma_buf_to_ubwcp_buf(struct dma_buf *dmabuf)
  322. {
  323. struct ubwcp_buf *buf = NULL;
  324. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  325. unsigned long flags;
  326. if (!dmabuf || !ubwcp)
  327. return NULL;
  328. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  329. /* look up ubwcp_buf corresponding to this dma_buf */
  330. hash_for_each_possible(ubwcp->buf_table, buf, hnode, (u64)dmabuf) {
  331. if (buf->dma_buf == dmabuf)
  332. break;
  333. }
  334. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  335. return buf;
  336. }
  337. /* return ubwcp hardware version */
  338. int ubwcp_get_hw_version(struct ubwcp_ioctl_hw_version *ver)
  339. {
  340. struct ubwcp_driver *ubwcp;
  341. FENTRY();
  342. if (!ver) {
  343. ERR("invalid version ptr");
  344. return -EINVAL;
  345. }
  346. ubwcp = ubwcp_get_driver();
  347. if (!ubwcp)
  348. return -1;
  349. ver->major = ubwcp->hw_ver_major;
  350. ver->minor = ubwcp->hw_ver_minor;
  351. return 0;
  352. }
  353. EXPORT_SYMBOL(ubwcp_get_hw_version);
  354. static int add_ula_pa_memory(struct ubwcp_driver *ubwcp)
  355. {
  356. int ret = 0;
  357. int nid;
  358. void *ptr;
  359. nid = memory_add_physaddr_to_nid(ubwcp->ula_pool_base);
  360. DBG("calling memremap_pages()...");
  361. ubwcp->pgmap.type = MEMORY_DEVICE_GENERIC;
  362. ubwcp->pgmap.nr_range = 1;
  363. ubwcp->pgmap.range.start = ubwcp->ula_pool_base;
  364. ubwcp->pgmap.range.end = ubwcp->ula_pool_base + ubwcp->ula_pool_size - 1;
  365. trace_ubwcp_memremap_pages_start(ubwcp->ula_pool_size);
  366. ptr = memremap_pages(&ubwcp->pgmap, nid);
  367. trace_ubwcp_memremap_pages_end(ubwcp->ula_pool_size);
  368. if (IS_ERR(ptr)) {
  369. ret = IS_ERR(ptr);
  370. ERR("memremap_pages() failed st:0x%lx sz:0x%lx err: %d",
  371. ubwcp->ula_pool_base,
  372. ubwcp->ula_pool_size,
  373. ret);
  374. /* Fix to put driver in invalid state */
  375. } else {
  376. DBG("memremap_pages() ula_pool_base:0x%llx, size:0x%zx, kernel addr:0x%p",
  377. ubwcp->ula_pool_base,
  378. ubwcp->ula_pool_size,
  379. page_to_virt(pfn_to_page(PFN_DOWN(ubwcp->ula_pool_base))));
  380. }
  381. return ret;
  382. }
  383. static int inc_num_non_lin_buffers(struct ubwcp_driver *ubwcp)
  384. {
  385. int ret = 0;
  386. atomic_inc(&ubwcp->num_non_lin_buffers);
  387. mutex_lock(&ubwcp->mem_hotplug_lock);
  388. if (!ubwcp->mem_online) {
  389. if (atomic_read(&ubwcp->num_non_lin_buffers) == 0) {
  390. ret = -EINVAL;
  391. ERR("Bad state: num_non_lin_buffers should not be 0");
  392. /* Fix to put driver in invalid state */
  393. goto err_power_on;
  394. }
  395. ret = ubwcp_power(ubwcp, true);
  396. if (ret)
  397. goto err_power_on;
  398. ret = add_ula_pa_memory(ubwcp);
  399. if (ret)
  400. goto err_add_memory;
  401. ubwcp->mem_online = true;
  402. }
  403. mutex_unlock(&ubwcp->mem_hotplug_lock);
  404. return 0;
  405. err_add_memory:
  406. ubwcp_power(ubwcp, false);
  407. err_power_on:
  408. atomic_dec(&ubwcp->num_non_lin_buffers);
  409. mutex_unlock(&ubwcp->mem_hotplug_lock);
  410. return ret;
  411. }
  412. static int dec_num_non_lin_buffers(struct ubwcp_driver *ubwcp)
  413. {
  414. int ret = 0;
  415. atomic_dec(&ubwcp->num_non_lin_buffers);
  416. mutex_lock(&ubwcp->mem_hotplug_lock);
  417. /* If this is the last buffer being freed, power off ubwcp */
  418. if (atomic_read(&ubwcp->num_non_lin_buffers) == 0) {
  419. unsigned long sync_remain = 0;
  420. unsigned long sync_offset = 0;
  421. unsigned long sync_size = 0;
  422. unsigned long sync_granule = UBWCP_SYNC_GRANULE;
  423. DBG("last buffer: ~~~~~~~~~~~");
  424. if (!ubwcp->mem_online) {
  425. ret = -EINVAL;
  426. ERR("Bad state: mem_online should not be false");
  427. /* Fix to put driver in invalid state */
  428. goto err_remove_mem;
  429. }
  430. DBG("set_direct_map_range_uncached() for ULA PA pool st:0x%lx num pages:%lu",
  431. ubwcp->ula_pool_base, ubwcp->ula_pool_size >> PAGE_SHIFT);
  432. trace_ubwcp_set_direct_map_range_uncached_start(ubwcp->ula_pool_size);
  433. ret = set_direct_map_range_uncached((unsigned long)phys_to_virt(
  434. ubwcp->ula_pool_base), ubwcp->ula_pool_size >> PAGE_SHIFT);
  435. trace_ubwcp_set_direct_map_range_uncached_end(ubwcp->ula_pool_size);
  436. if (ret) {
  437. ERR("set_direct_map_range_uncached failed st:0x%lx num pages:%lu err: %d",
  438. ubwcp->ula_pool_base,
  439. ubwcp->ula_pool_size >> PAGE_SHIFT, ret);
  440. goto err_remove_mem;
  441. } else {
  442. DBG("DONE: calling set_direct_map_range_uncached() for ULA PA pool");
  443. }
  444. DBG("Calling dma_sync_single_for_cpu() for ULA PA pool");
  445. trace_ubwcp_offline_sync_start(ubwcp->ula_pool_size);
  446. sync_remain = ubwcp->ula_pool_size;
  447. sync_offset = 0;
  448. while (sync_remain > 0) {
  449. if (atomic_read(&ubwcp->num_non_lin_buffers) > 0) {
  450. trace_ubwcp_offline_sync_end(ubwcp->ula_pool_size);
  451. DBG("Cancel memory offlining");
  452. DBG("Calling memunmap_pages() for ULA PA pool");
  453. trace_ubwcp_memunmap_pages_start(ubwcp->ula_pool_size);
  454. memunmap_pages(&ubwcp->pgmap);
  455. trace_ubwcp_memunmap_pages_end(ubwcp->ula_pool_size);
  456. ret = add_ula_pa_memory(ubwcp);
  457. if (ret) {
  458. ERR("Bad state: failed to add back memory");
  459. /* Fix to put driver in invalid state */
  460. ubwcp->mem_online = false;
  461. }
  462. mutex_unlock(&ubwcp->mem_hotplug_lock);
  463. return ret;
  464. }
  465. if (sync_granule > sync_remain) {
  466. sync_size = sync_remain;
  467. sync_remain = 0;
  468. } else {
  469. sync_size = sync_granule;
  470. sync_remain -= sync_granule;
  471. }
  472. DBG("Partial sync offset:0x%lx size:0x%lx", sync_offset, sync_size);
  473. trace_ubwcp_dma_sync_single_for_cpu_start(sync_size);
  474. dma_sync_single_for_cpu(ubwcp->dev, ubwcp->ula_pool_base + sync_offset,
  475. sync_size, DMA_BIDIRECTIONAL);
  476. trace_ubwcp_dma_sync_single_for_cpu_end(sync_size);
  477. sync_offset += sync_size;
  478. }
  479. trace_ubwcp_offline_sync_end(ubwcp->ula_pool_size);
  480. DBG("Calling memunmap_pages() for ULA PA pool");
  481. trace_ubwcp_memunmap_pages_start(ubwcp->ula_pool_size);
  482. memunmap_pages(&ubwcp->pgmap);
  483. trace_ubwcp_memunmap_pages_end(ubwcp->ula_pool_size);
  484. DBG("Calling power OFF ...");
  485. ubwcp_power(ubwcp, false);
  486. ubwcp->mem_online = false;
  487. }
  488. mutex_unlock(&ubwcp->mem_hotplug_lock);
  489. return 0;
  490. err_remove_mem:
  491. atomic_inc(&ubwcp->num_non_lin_buffers);
  492. mutex_unlock(&ubwcp->mem_hotplug_lock);
  493. DBG("returning error: %d", ret);
  494. return ret;
  495. }
  496. /**
  497. *
  498. * Initialize ubwcp buffer for the given dma_buf. This
  499. * initializes ubwcp internal data structures and possibly hw to
  500. * use ubwcp for this buffer.
  501. *
  502. * @param dmabuf : ptr to the buffer to be configured for ubwcp
  503. *
  504. * @return int : 0 on success, otherwise error code
  505. */
  506. static int ubwcp_init_buffer(struct dma_buf *dmabuf)
  507. {
  508. struct ubwcp_buf *buf;
  509. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  510. unsigned long flags;
  511. FENTRY();
  512. trace_ubwcp_init_buffer_start(dmabuf);
  513. if (!ubwcp) {
  514. trace_ubwcp_init_buffer_end(dmabuf);
  515. return -1;
  516. }
  517. if (!dmabuf) {
  518. ERR("NULL dmabuf input ptr");
  519. trace_ubwcp_init_buffer_end(dmabuf);
  520. return -EINVAL;
  521. }
  522. if (dma_buf_to_ubwcp_buf(dmabuf)) {
  523. ERR("dma_buf already initialized for ubwcp");
  524. trace_ubwcp_init_buffer_end(dmabuf);
  525. return -EEXIST;
  526. }
  527. buf = kzalloc(sizeof(*buf), GFP_KERNEL);
  528. if (!buf) {
  529. ERR("failed to alloc for new ubwcp_buf");
  530. trace_ubwcp_init_buffer_end(dmabuf);
  531. return -ENOMEM;
  532. }
  533. mutex_init(&buf->lock);
  534. buf->dma_buf = dmabuf;
  535. buf->ubwcp = ubwcp;
  536. buf->buf_attr.image_format = UBWCP_LINEAR;
  537. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  538. hash_add(ubwcp->buf_table, &buf->hnode, (u64)buf->dma_buf);
  539. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  540. trace_ubwcp_init_buffer_end(dmabuf);
  541. return 0;
  542. }
  543. static void dump_attributes(struct ubwcp_buffer_attrs *attr)
  544. {
  545. DBG_BUF_ATTR("");
  546. DBG_BUF_ATTR("image_format: %d", attr->image_format);
  547. DBG_BUF_ATTR("major_ubwc_ver: %d", attr->major_ubwc_ver);
  548. DBG_BUF_ATTR("minor_ubwc_ver: %d", attr->minor_ubwc_ver);
  549. DBG_BUF_ATTR("compression_type: %d", attr->compression_type);
  550. DBG_BUF_ATTR("lossy_params: %llu", attr->lossy_params);
  551. DBG_BUF_ATTR("width: %d", attr->width);
  552. DBG_BUF_ATTR("height: %d", attr->height);
  553. DBG_BUF_ATTR("stride: %d", attr->stride);
  554. DBG_BUF_ATTR("scanlines: %d", attr->scanlines);
  555. DBG_BUF_ATTR("planar_padding: %d", attr->planar_padding);
  556. DBG_BUF_ATTR("subsample: %d", attr->subsample);
  557. DBG_BUF_ATTR("sub_system_target: %d", attr->sub_system_target);
  558. DBG_BUF_ATTR("y_offset: %d", attr->y_offset);
  559. DBG_BUF_ATTR("batch_size: %d", attr->batch_size);
  560. DBG_BUF_ATTR("");
  561. }
  562. static enum ubwcp_std_image_format to_std_format(u16 ioctl_image_format)
  563. {
  564. switch (ioctl_image_format) {
  565. case UBWCP_RGBA8888:
  566. return RGBA;
  567. case UBWCP_NV12:
  568. case UBWCP_NV12_Y:
  569. case UBWCP_NV12_UV:
  570. return NV12;
  571. case UBWCP_NV124R:
  572. case UBWCP_NV124R_Y:
  573. case UBWCP_NV124R_UV:
  574. return NV124R;
  575. case UBWCP_TP10:
  576. case UBWCP_TP10_Y:
  577. case UBWCP_TP10_UV:
  578. return TP10;
  579. case UBWCP_P010:
  580. case UBWCP_P010_Y:
  581. case UBWCP_P010_UV:
  582. return P010;
  583. case UBWCP_P016:
  584. case UBWCP_P016_Y:
  585. case UBWCP_P016_UV:
  586. return P016;
  587. default:
  588. WARN(1, "Fix this!!!");
  589. return STD_IMAGE_FORMAT_INVALID;
  590. }
  591. }
  592. static int get_stride_alignment(enum ubwcp_std_image_format format, u16 *align)
  593. {
  594. switch (format) {
  595. case TP10:
  596. *align = 64;
  597. return 0;
  598. case NV12:
  599. *align = 128;
  600. return 0;
  601. case RGBA:
  602. case NV124R:
  603. case P010:
  604. case P016:
  605. *align = 256;
  606. return 0;
  607. default:
  608. return -1;
  609. }
  610. }
  611. /* returns stride of compressed image */
  612. static u32 get_compressed_stride(struct ubwcp_driver *ubwcp,
  613. enum ubwcp_std_image_format format, u32 width)
  614. {
  615. struct ubwcp_plane_info p_info;
  616. u16 macro_tile_width_p;
  617. u16 pixel_bytes;
  618. u16 per_pixel;
  619. p_info = ubwcp->format_info[format].p_info[0];
  620. macro_tile_width_p = p_info.macrotilesize_p.width;
  621. pixel_bytes = p_info.pixel_bytes;
  622. per_pixel = p_info.per_pixel;
  623. return UBWCP_ALIGN(width, macro_tile_width_p)*pixel_bytes/per_pixel;
  624. }
  625. /* check if linear stride conforms to hw limitations
  626. * always returns false for linear image
  627. */
  628. static bool stride_is_valid(struct ubwcp_driver *ubwcp,
  629. u16 ioctl_img_fmt, u32 width, u32 lin_stride)
  630. {
  631. u32 compressed_stride;
  632. enum ubwcp_std_image_format format = to_std_format(ioctl_img_fmt);
  633. if (format == STD_IMAGE_FORMAT_INVALID)
  634. return false;
  635. if ((lin_stride < width) || (lin_stride > 64*1024)) {
  636. ERR("stride is not valid (width <= stride <= 64K): %d", lin_stride);
  637. return false;
  638. }
  639. if (format == TP10) {
  640. if(!IS_ALIGNED(lin_stride, 64)) {
  641. ERR("stride must be aligned to 64: %d", lin_stride);
  642. return false;
  643. }
  644. } else {
  645. compressed_stride = get_compressed_stride(ubwcp, format, width);
  646. if (lin_stride != compressed_stride) {
  647. ERR("linear stride: %d must be same as compressed stride: %d",
  648. lin_stride, compressed_stride);
  649. return false;
  650. }
  651. }
  652. return true;
  653. }
  654. static bool ioctl_format_is_valid(u16 ioctl_image_format)
  655. {
  656. switch (ioctl_image_format) {
  657. case UBWCP_LINEAR:
  658. case UBWCP_RGBA8888:
  659. case UBWCP_NV12:
  660. case UBWCP_NV12_Y:
  661. case UBWCP_NV12_UV:
  662. case UBWCP_NV124R:
  663. case UBWCP_NV124R_Y:
  664. case UBWCP_NV124R_UV:
  665. case UBWCP_TP10:
  666. case UBWCP_TP10_Y:
  667. case UBWCP_TP10_UV:
  668. case UBWCP_P010:
  669. case UBWCP_P010_Y:
  670. case UBWCP_P010_UV:
  671. case UBWCP_P016:
  672. case UBWCP_P016_Y:
  673. case UBWCP_P016_UV:
  674. return true;
  675. default:
  676. return false;
  677. }
  678. }
  679. /* validate buffer attributes */
  680. static bool ubwcp_buf_attrs_valid(struct ubwcp_driver *ubwcp, struct ubwcp_buffer_attrs *attr)
  681. {
  682. if (!ioctl_format_is_valid(attr->image_format)) {
  683. ERR("invalid image format: %d", attr->image_format);
  684. goto err;
  685. }
  686. if (attr->major_ubwc_ver || attr->minor_ubwc_ver) {
  687. ERR("major/minor ubwc ver must be 0. major: %d minor: %d",
  688. attr->major_ubwc_ver, attr->minor_ubwc_ver);
  689. goto err;
  690. }
  691. if (attr->compression_type != UBWCP_COMPRESSION_LOSSLESS) {
  692. ERR("compression_type is not valid: %d",
  693. attr->compression_type);
  694. goto err;
  695. }
  696. if (attr->lossy_params != 0) {
  697. ERR("lossy_params is not valid: %d", attr->lossy_params);
  698. goto err;
  699. }
  700. //TBD: some upper limit for width?
  701. if (attr->width > 10*1024) {
  702. ERR("width is invalid (above upper limit): %d", attr->width);
  703. goto err;
  704. }
  705. //TBD: some upper limit for height?
  706. if (attr->height > 10*1024) {
  707. ERR("height is invalid (above upper limit): %d", attr->height);
  708. goto err;
  709. }
  710. if (attr->image_format != UBWCP_LINEAR)
  711. if(!stride_is_valid(ubwcp, attr->image_format, attr->width, attr->stride)) {
  712. ERR("stride is invalid: %d", attr->stride);
  713. goto err;
  714. }
  715. if ((attr->scanlines < attr->height) ||
  716. (attr->scanlines > attr->height + 32*1024)) {
  717. ERR("scanlines is not valid - height: %d scanlines: %d",
  718. attr->height, attr->scanlines);
  719. goto err;
  720. }
  721. if (attr->planar_padding > 4096) {
  722. ERR("planar_padding is not valid. (<= 4096): %d",
  723. attr->planar_padding);
  724. goto err;
  725. }
  726. if (attr->subsample != UBWCP_SUBSAMPLE_4_2_0) {
  727. ERR("subsample is not valid: %d", attr->subsample);
  728. goto err;
  729. }
  730. if (attr->sub_system_target & ~UBWCP_SUBSYSTEM_TARGET_CPU) {
  731. ERR("sub_system_target other that CPU is not supported: %d",
  732. attr->sub_system_target);
  733. goto err;
  734. }
  735. if (!(attr->sub_system_target & UBWCP_SUBSYSTEM_TARGET_CPU)) {
  736. ERR("sub_system_target is not set to CPU: %d",
  737. attr->sub_system_target);
  738. goto err;
  739. }
  740. if (attr->y_offset != 0) {
  741. ERR("y_offset is not valid: %d", attr->y_offset);
  742. goto err;
  743. }
  744. if (attr->batch_size != 1) {
  745. ERR("batch_size is not valid: %d", attr->batch_size);
  746. goto err;
  747. }
  748. dump_attributes(attr);
  749. return true;
  750. err:
  751. dump_attributes(attr);
  752. return false;
  753. }
  754. /* return true if image format has only Y plane*/
  755. bool ubwcp_image_y_only(u16 format)
  756. {
  757. switch (format) {
  758. case UBWCP_NV12_Y:
  759. case UBWCP_NV124R_Y:
  760. case UBWCP_TP10_Y:
  761. case UBWCP_P010_Y:
  762. case UBWCP_P016_Y:
  763. return true;
  764. default:
  765. return false;
  766. }
  767. }
  768. /* return true if image format has only UV plane*/
  769. bool ubwcp_image_uv_only(u16 format)
  770. {
  771. switch (format) {
  772. case UBWCP_NV12_UV:
  773. case UBWCP_NV124R_UV:
  774. case UBWCP_TP10_UV:
  775. case UBWCP_P010_UV:
  776. case UBWCP_P016_UV:
  777. return true;
  778. default:
  779. return false;
  780. }
  781. }
  782. /* calculate and return metadata buffer size for a given plane
  783. * and buffer attributes
  784. * NOTE: in this function, we will only pass in NV12 format.
  785. * NOT NV12_Y or NV12_UV etc.
  786. * the Y or UV information is in the "plane"
  787. * "format" here purely means "encoding format" and no information
  788. * if some plane data is missing.
  789. */
  790. static size_t metadata_buf_sz(struct ubwcp_driver *ubwcp,
  791. enum ubwcp_std_image_format format,
  792. u32 width, u32 height, u8 plane)
  793. {
  794. size_t size;
  795. u64 pitch;
  796. u64 lines;
  797. u64 tile_width;
  798. u32 tile_height;
  799. struct ubwcp_image_format_info f_info;
  800. struct ubwcp_plane_info p_info;
  801. f_info = ubwcp->format_info[format];
  802. DBG_BUF_ATTR("");
  803. DBG_BUF_ATTR("");
  804. DBG_BUF_ATTR("Calculating metadata buffer size: format = %d, plane = %d", format, plane);
  805. if (plane >= f_info.planes) {
  806. ERR("Format does not have requested plane info: format: %d, plane: %d",
  807. format, plane);
  808. WARN(1, "Fix this!!!!!");
  809. return 0;
  810. }
  811. p_info = f_info.p_info[plane];
  812. /* UV plane */
  813. if (plane == 1) {
  814. width = width/2;
  815. height = height/2;
  816. }
  817. tile_width = p_info.tilesize_p.width;
  818. tile_height = p_info.tilesize_p.height;
  819. /* pitch: # of tiles in a row
  820. * lines: # of tile rows
  821. */
  822. pitch = UBWCP_ALIGN((width + tile_width - 1)/tile_width, META_DATA_PITCH_ALIGN);
  823. lines = UBWCP_ALIGN((height + tile_height - 1)/tile_height, META_DATA_HEIGHT_ALIGN);
  824. DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
  825. DBG_BUF_ATTR("tile params : %d x %d (pixels)", tile_width, tile_height);
  826. DBG_BUF_ATTR("pitch : %d (%d)", pitch, width/tile_width);
  827. DBG_BUF_ATTR("lines : %d (%d)", lines, height);
  828. DBG_BUF_ATTR("size (p*l*bytes) : %d", pitch*lines*1);
  829. /* x1 below is only to clarify that we are multiplying by 1 bytes/tile */
  830. size = UBWCP_ALIGN(pitch*lines*1, META_DATA_SIZE_ALIGN);
  831. DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", size, size);
  832. return size;
  833. }
  834. /* calculate and return size of pixel data buffer for a given plane
  835. * and buffer attributes
  836. */
  837. static size_t pixeldata_buf_sz(struct ubwcp_driver *ubwcp,
  838. u16 format, u32 width,
  839. u32 height, u8 plane)
  840. {
  841. size_t size;
  842. u64 pitch;
  843. u64 lines;
  844. u16 pixel_bytes;
  845. u16 per_pixel;
  846. u64 macro_tile_width_p;
  847. u64 macro_tile_height_p;
  848. struct ubwcp_image_format_info f_info;
  849. struct ubwcp_plane_info p_info;
  850. f_info = ubwcp->format_info[format];
  851. DBG_BUF_ATTR("");
  852. DBG_BUF_ATTR("");
  853. DBG_BUF_ATTR("Calculating Pixeldata buffer size: format = %d, plane = %d", format, plane);
  854. if (plane >= f_info.planes) {
  855. ERR("Format does not have requested plane info: format: %d, plane: %d",
  856. format, plane);
  857. WARN(1, "Fix this!!!!!");
  858. return 0;
  859. }
  860. p_info = f_info.p_info[plane];
  861. pixel_bytes = p_info.pixel_bytes;
  862. per_pixel = p_info.per_pixel;
  863. /* UV plane */
  864. if (plane == 1) {
  865. width = width/2;
  866. height = height/2;
  867. }
  868. macro_tile_width_p = p_info.macrotilesize_p.width;
  869. macro_tile_height_p = p_info.macrotilesize_p.height;
  870. /* align pixel width and height macro tile width and height */
  871. pitch = UBWCP_ALIGN(width, macro_tile_width_p);
  872. lines = UBWCP_ALIGN(height, macro_tile_height_p);
  873. DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
  874. DBG_BUF_ATTR("macro tile params: %d x %d (pixels)", macro_tile_width_p,
  875. macro_tile_height_p);
  876. DBG_BUF_ATTR("bytes_per_pixel : %d/%d", pixel_bytes, per_pixel);
  877. DBG_BUF_ATTR("pitch : %d", pitch);
  878. DBG_BUF_ATTR("lines : %d", lines);
  879. DBG_BUF_ATTR("size (p*l*bytes) : %d", (pitch*lines*pixel_bytes)/per_pixel);
  880. size = UBWCP_ALIGN((pitch*lines*pixel_bytes)/per_pixel, PIXEL_DATA_SIZE_ALIGN);
  881. DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", size, size);
  882. return size;
  883. }
  884. static int get_tile_height(struct ubwcp_driver *ubwcp, enum ubwcp_std_image_format format,
  885. u8 plane)
  886. {
  887. struct ubwcp_image_format_info f_info;
  888. struct ubwcp_plane_info p_info;
  889. f_info = ubwcp->format_info[format];
  890. p_info = f_info.p_info[plane];
  891. return p_info.tilesize_p.height;
  892. }
  893. /*
  894. * plane: must be 0 or 1 (1st plane == 0, 2nd plane == 1)
  895. */
  896. static size_t ubwcp_ula_size(struct ubwcp_driver *ubwcp, u16 format,
  897. u32 stride_b, u32 scanlines, u8 plane,
  898. bool add_tile_pad)
  899. {
  900. size_t size;
  901. DBG_BUF_ATTR("%s(format = %d, plane = %d)", __func__, format, plane);
  902. /* UV plane */
  903. if (plane == 1)
  904. scanlines = scanlines/2;
  905. if (add_tile_pad) {
  906. int tile_height = get_tile_height(ubwcp, format, plane);
  907. /* Align plane size to plane tile height */
  908. scanlines = ((scanlines + tile_height - 1) / tile_height) * tile_height;
  909. }
  910. size = stride_b*scanlines;
  911. DBG_BUF_ATTR("Size of plane-%u: (%u * %u) = %zu (0x%zx)",
  912. plane, stride_b, scanlines, size, size);
  913. return size;
  914. }
  915. int missing_plane_from_format(u16 ioctl_image_format)
  916. {
  917. int missing_plane;
  918. switch (ioctl_image_format) {
  919. case UBWCP_NV12_Y:
  920. missing_plane = 2;
  921. break;
  922. case UBWCP_NV12_UV:
  923. missing_plane = 1;
  924. break;
  925. case UBWCP_NV124R_Y:
  926. missing_plane = 2;
  927. break;
  928. case UBWCP_NV124R_UV:
  929. missing_plane = 1;
  930. break;
  931. case UBWCP_TP10_Y:
  932. missing_plane = 2;
  933. break;
  934. case UBWCP_TP10_UV:
  935. missing_plane = 1;
  936. break;
  937. case UBWCP_P010_Y:
  938. missing_plane = 2;
  939. break;
  940. case UBWCP_P010_UV:
  941. missing_plane = 1;
  942. break;
  943. case UBWCP_P016_Y:
  944. missing_plane = 2;
  945. break;
  946. case UBWCP_P016_UV:
  947. missing_plane = 1;
  948. break;
  949. default:
  950. missing_plane = 0;
  951. }
  952. return missing_plane;
  953. }
  954. int planes_in_format(enum ubwcp_std_image_format format)
  955. {
  956. if (format == RGBA)
  957. return 1;
  958. else
  959. return 2;
  960. }
  961. unsigned int ubwcp_get_hw_image_format_value(u16 ioctl_image_format)
  962. {
  963. enum ubwcp_std_image_format format;
  964. format = to_std_format(ioctl_image_format);
  965. switch (format) {
  966. case RGBA:
  967. return HW_BUFFER_FORMAT_RGBA;
  968. case NV12:
  969. return HW_BUFFER_FORMAT_NV12;
  970. case NV124R:
  971. return HW_BUFFER_FORMAT_NV124R;
  972. case P010:
  973. return HW_BUFFER_FORMAT_P010;
  974. case TP10:
  975. return HW_BUFFER_FORMAT_TP10;
  976. case P016:
  977. return HW_BUFFER_FORMAT_P016;
  978. default:
  979. WARN(1, "Fix this!!!!!");
  980. return 0;
  981. }
  982. }
  983. static int ubwcp_validate_uv_align(struct ubwcp_driver *ubwcp,
  984. struct ubwcp_buffer_attrs *attr,
  985. size_t ula_y_plane_size,
  986. size_t uv_start_offset)
  987. {
  988. int ret = 0;
  989. size_t ula_y_plane_size_align;
  990. size_t y_tile_align_bytes;
  991. int y_tile_height;
  992. int planes;
  993. /* Only validate UV align if there is both a Y and UV plane */
  994. planes = planes_in_format(to_std_format(attr->image_format));
  995. if (planes != 2)
  996. return 0;
  997. /* Check it is cache line size aligned */
  998. if ((uv_start_offset % 64) != 0) {
  999. ret = -EINVAL;
  1000. ERR("uv_start_offset %zu not cache line aligned",
  1001. uv_start_offset);
  1002. goto err;
  1003. }
  1004. /*
  1005. * Check that UV plane does not overlap with any of the Y plane’s tiles
  1006. */
  1007. y_tile_height = get_tile_height(ubwcp, to_std_format(attr->image_format), 0);
  1008. y_tile_align_bytes = y_tile_height * attr->stride;
  1009. ula_y_plane_size_align = ((ula_y_plane_size + y_tile_align_bytes - 1) /
  1010. y_tile_align_bytes) * y_tile_align_bytes;
  1011. if (uv_start_offset < ula_y_plane_size_align) {
  1012. ret = -EINVAL;
  1013. ERR("uv offset %zu less than y plane align %zu for y plane size %zu",
  1014. uv_start_offset, ula_y_plane_size_align,
  1015. ula_y_plane_size);
  1016. goto err;
  1017. }
  1018. return 0;
  1019. err:
  1020. return ret;
  1021. }
  1022. /* calculate ULA buffer parms
  1023. * TBD: how do we make sure uv_start address (not the offset)
  1024. * is aligned per requirement: cache line
  1025. */
  1026. static int ubwcp_calc_ula_params(struct ubwcp_driver *ubwcp,
  1027. struct ubwcp_buffer_attrs *attr,
  1028. size_t *ula_size,
  1029. size_t *ula_y_plane_size,
  1030. size_t *uv_start_offset)
  1031. {
  1032. size_t size;
  1033. enum ubwcp_std_image_format format;
  1034. int planes;
  1035. int missing_plane;
  1036. u32 stride;
  1037. u32 scanlines;
  1038. u32 planar_padding;
  1039. stride = attr->stride;
  1040. scanlines = attr->scanlines;
  1041. planar_padding = attr->planar_padding;
  1042. /* convert ioctl image format to standard image format */
  1043. format = to_std_format(attr->image_format);
  1044. /* Number of "expected" planes in "the standard defined" image format */
  1045. planes = planes_in_format(format);
  1046. /* any plane missing?
  1047. * valid missing_plane values:
  1048. * 0 == no plane missing
  1049. * 1 == 1st plane missing
  1050. * 2 == 2nd plane missing
  1051. */
  1052. missing_plane = missing_plane_from_format(attr->image_format);
  1053. DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
  1054. DBG_BUF_ATTR("planes_in_format : %d", planes);
  1055. DBG_BUF_ATTR("missing_plane : %d", missing_plane);
  1056. DBG_BUF_ATTR("Planar Padding : %d", planar_padding);
  1057. if (planes == 1) {
  1058. /* uv_start beyond ULA range */
  1059. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
  1060. *uv_start_offset = size;
  1061. *ula_y_plane_size = size;
  1062. } else {
  1063. if (!missing_plane) {
  1064. /* size for both planes and padding */
  1065. /* Don't pad out Y plane as client would not expect this padding */
  1066. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, false);
  1067. *ula_y_plane_size = size;
  1068. size += planar_padding;
  1069. *uv_start_offset = size;
  1070. size += ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
  1071. } else {
  1072. if (missing_plane == 2) {
  1073. /* Y-only image, set uv_start beyond ULA range */
  1074. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
  1075. *uv_start_offset = size;
  1076. *ula_y_plane_size = size;
  1077. } else {
  1078. /* first plane data is not there */
  1079. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
  1080. *uv_start_offset = 0; /* uv data is at the beginning */
  1081. *ula_y_plane_size = 0;
  1082. }
  1083. }
  1084. }
  1085. //TBD: cleanup
  1086. *ula_size = size;
  1087. DBG_BUF_ATTR("Before page align: Total ULA_Size: %d (0x%x) (planes + planar padding)",
  1088. *ula_size, *ula_size);
  1089. *ula_size = UBWCP_ALIGN(size, 4096);
  1090. DBG_BUF_ATTR("After page align : Total ULA_Size: %d (0x%x) (planes + planar padding)",
  1091. *ula_size, *ula_size);
  1092. return 0;
  1093. }
  1094. /* calculate UBWCP buffer parms */
  1095. static int ubwcp_calc_ubwcp_buf_params(struct ubwcp_driver *ubwcp,
  1096. struct ubwcp_buffer_attrs *attr,
  1097. size_t *md_p0, size_t *pd_p0,
  1098. size_t *md_p1, size_t *pd_p1,
  1099. size_t *stride_tp10_b)
  1100. {
  1101. int planes;
  1102. int missing_plane;
  1103. enum ubwcp_std_image_format format;
  1104. size_t stride_tp10_p;
  1105. FENTRY();
  1106. /* convert ioctl image format to standard image format */
  1107. format = to_std_format(attr->image_format);
  1108. missing_plane = missing_plane_from_format(attr->image_format);
  1109. planes = planes_in_format(format); //pass in 0 (RGB) should return 1
  1110. DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
  1111. DBG_BUF_ATTR("planes_in_format : %d", planes);
  1112. DBG_BUF_ATTR("missing_plane : %d", missing_plane);
  1113. if (!missing_plane) {
  1114. *md_p0 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  1115. *pd_p0 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  1116. if (planes == 2) {
  1117. *md_p1 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  1118. *pd_p1 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  1119. }
  1120. } else {
  1121. if (missing_plane == 1) {
  1122. *md_p0 = 0;
  1123. *pd_p0 = 0;
  1124. *md_p1 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  1125. *pd_p1 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  1126. } else {
  1127. *md_p0 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  1128. *pd_p0 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  1129. *md_p1 = 0;
  1130. *pd_p1 = 0;
  1131. }
  1132. }
  1133. if (format == TP10) {
  1134. stride_tp10_p = UBWCP_ALIGN(attr->width, 192);
  1135. *stride_tp10_b = (stride_tp10_p/3) + stride_tp10_p;
  1136. } else {
  1137. *stride_tp10_b = 0;
  1138. }
  1139. return 0;
  1140. }
  1141. /* reserve ULA address space of the given size */
  1142. static phys_addr_t ubwcp_ula_alloc(struct ubwcp_driver *ubwcp, size_t size)
  1143. {
  1144. phys_addr_t pa;
  1145. mutex_lock(&ubwcp->ula_lock);
  1146. pa = gen_pool_alloc(ubwcp->ula_pool, size);
  1147. DBG("addr: %p, size: %zx", pa, size);
  1148. mutex_unlock(&ubwcp->ula_lock);
  1149. return pa;
  1150. }
  1151. /* free ULA address space of the given address and size */
  1152. static void ubwcp_ula_free(struct ubwcp_driver *ubwcp, phys_addr_t pa, size_t size)
  1153. {
  1154. mutex_lock(&ubwcp->ula_lock);
  1155. if (!gen_pool_has_addr(ubwcp->ula_pool, pa, size)) {
  1156. ERR("Attempt to free mem not from gen_pool: pa: %p, size: %zx", pa, size);
  1157. goto err;
  1158. }
  1159. DBG("addr: %p, size: %zx", pa, size);
  1160. gen_pool_free(ubwcp->ula_pool, pa, size);
  1161. mutex_unlock(&ubwcp->ula_lock);
  1162. return;
  1163. err:
  1164. mutex_unlock(&ubwcp->ula_lock);
  1165. }
  1166. /* free up or expand current_pa and return the new pa */
  1167. static phys_addr_t ubwcp_ula_realloc(struct ubwcp_driver *ubwcp,
  1168. phys_addr_t pa,
  1169. size_t size,
  1170. size_t new_size)
  1171. {
  1172. if (size == new_size)
  1173. return pa;
  1174. if (pa)
  1175. ubwcp_ula_free(ubwcp, pa, size);
  1176. return ubwcp_ula_alloc(ubwcp, new_size);
  1177. }
  1178. /* unmap dma buf */
  1179. static void ubwcp_dma_unmap(struct ubwcp_buf *buf)
  1180. {
  1181. FENTRY();
  1182. if (buf->dma_buf && buf->attachment) {
  1183. DBG("Calling dma_buf_unmap_attachment()");
  1184. dma_buf_unmap_attachment(buf->attachment, buf->sgt, DMA_BIDIRECTIONAL);
  1185. buf->sgt = NULL;
  1186. dma_buf_detach(buf->dma_buf, buf->attachment);
  1187. buf->attachment = NULL;
  1188. }
  1189. }
  1190. /* dma map ubwcp buffer */
  1191. static int ubwcp_dma_map(struct ubwcp_buf *buf,
  1192. struct device *dev,
  1193. size_t iova_min_size,
  1194. dma_addr_t *iova)
  1195. {
  1196. int ret = 0;
  1197. struct dma_buf *dma_buf = buf->dma_buf;
  1198. struct dma_buf_attachment *attachment;
  1199. struct sg_table *sgt;
  1200. size_t dma_len;
  1201. /* Map buffer to SMMU and get IOVA */
  1202. attachment = dma_buf_attach(dma_buf, dev);
  1203. if (IS_ERR(attachment)) {
  1204. ret = PTR_ERR(attachment);
  1205. ERR("dma_buf_attach() failed: %d", ret);
  1206. goto err;
  1207. }
  1208. dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
  1209. dma_set_seg_boundary(dev, (unsigned long)DMA_BIT_MASK(64));
  1210. sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
  1211. if (IS_ERR_OR_NULL(sgt)) {
  1212. ret = PTR_ERR(sgt);
  1213. ERR("dma_buf_map_attachment() failed: %d", ret);
  1214. goto err_detach;
  1215. }
  1216. if (sgt->nents != 1) {
  1217. ERR("nents = %d", sgt->nents);
  1218. goto err_unmap;
  1219. }
  1220. /* ensure that dma_buf is big enough for the new attrs */
  1221. dma_len = sg_dma_len(sgt->sgl);
  1222. if (dma_len < iova_min_size) {
  1223. ERR("dma len: %d is less than min ubwcp buffer size: %d",
  1224. dma_len, iova_min_size);
  1225. goto err_unmap;
  1226. }
  1227. *iova = sg_dma_address(sgt->sgl);
  1228. buf->attachment = attachment;
  1229. buf->sgt = sgt;
  1230. return ret;
  1231. err_unmap:
  1232. dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
  1233. err_detach:
  1234. dma_buf_detach(dma_buf, attachment);
  1235. err:
  1236. if (!ret)
  1237. ret = -1;
  1238. return ret;
  1239. }
  1240. static void
  1241. ubwcp_pixel_to_bytes(struct ubwcp_driver *ubwcp,
  1242. enum ubwcp_std_image_format format,
  1243. u32 width_p, u32 height_p,
  1244. u32 *width_b, u32 *height_b)
  1245. {
  1246. u16 pixel_bytes;
  1247. u16 per_pixel;
  1248. struct ubwcp_image_format_info f_info;
  1249. struct ubwcp_plane_info p_info;
  1250. f_info = ubwcp->format_info[format];
  1251. p_info = f_info.p_info[0];
  1252. pixel_bytes = p_info.pixel_bytes;
  1253. per_pixel = p_info.per_pixel;
  1254. *width_b = (width_p*pixel_bytes)/per_pixel;
  1255. *height_b = (height_p*pixel_bytes)/per_pixel;
  1256. }
  1257. static void reset_buf_attrs(struct ubwcp_buf *buf)
  1258. {
  1259. struct ubwcp_hw_meta_metadata *mmdata;
  1260. struct ubwcp_driver *ubwcp;
  1261. ubwcp = buf->ubwcp;
  1262. mmdata = &buf->mmdata;
  1263. ubwcp_dma_unmap(buf);
  1264. /* reset ula params */
  1265. if (buf->ula_size) {
  1266. ubwcp_ula_free(ubwcp, buf->ula_pa, buf->ula_size);
  1267. buf->ula_size = 0;
  1268. buf->ula_pa = 0;
  1269. }
  1270. /* reset ubwcp params */
  1271. memset(mmdata, 0, sizeof(*mmdata));
  1272. buf->buf_attr_set = false;
  1273. buf->buf_attr.image_format = UBWCP_LINEAR;
  1274. }
  1275. static void print_mmdata_desc(struct ubwcp_hw_meta_metadata *mmdata)
  1276. {
  1277. DBG_BUF_ATTR("");
  1278. DBG_BUF_ATTR("--------MM_DATA DESC ---------");
  1279. DBG_BUF_ATTR("uv_start_addr : 0x%08llx (cache addr) (actual: 0x%llx)",
  1280. mmdata->uv_start_addr, mmdata->uv_start_addr << 6);
  1281. DBG_BUF_ATTR("format : 0x%08x", mmdata->format);
  1282. DBG_BUF_ATTR("stride : 0x%08x (cache addr) (actual: 0x%x)",
  1283. mmdata->stride, mmdata->stride << 6);
  1284. DBG_BUF_ATTR("stride_ubwcp : 0x%08x (cache addr) (actual: 0x%zx)",
  1285. mmdata->stride_ubwcp, mmdata->stride_ubwcp << 6);
  1286. DBG_BUF_ATTR("metadata_base_y : 0x%08x (page addr) (actual: 0x%llx)",
  1287. mmdata->metadata_base_y, mmdata->metadata_base_y << 12);
  1288. DBG_BUF_ATTR("metadata_base_uv: 0x%08x (page addr) (actual: 0x%zx)",
  1289. mmdata->metadata_base_uv, mmdata->metadata_base_uv << 12);
  1290. DBG_BUF_ATTR("buffer_y_offset : 0x%08x (page addr) (actual: 0x%zx)",
  1291. mmdata->buffer_y_offset, mmdata->buffer_y_offset << 12);
  1292. DBG_BUF_ATTR("buffer_uv_offset: 0x%08x (page addr) (actual: 0x%zx)",
  1293. mmdata->buffer_uv_offset, mmdata->buffer_uv_offset << 12);
  1294. DBG_BUF_ATTR("width_height : 0x%08x (width: 0x%x height: 0x%x)",
  1295. mmdata->width_height, mmdata->width_height >> 16, mmdata->width_height & 0xFFFF);
  1296. DBG_BUF_ATTR("");
  1297. }
  1298. /* set buffer attributes:
  1299. * Failure:
  1300. * If a call to ubwcp_set_buf_attrs() fails, any attributes set from a previously
  1301. * successful ubwcp_set_buf_attrs() will be also removed. Thus,
  1302. * ubwcp_set_buf_attrs() implicitly does "unset previous attributes" and
  1303. * then "try to set these new attributes".
  1304. *
  1305. * The result of a failed call to ubwcp_set_buf_attrs() will leave the buffer
  1306. * in a linear mode, NOT with attributes from earlier successful call.
  1307. */
  1308. int ubwcp_set_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
  1309. {
  1310. int ret = 0;
  1311. size_t ula_size = 0;
  1312. size_t uv_start_offset = 0;
  1313. size_t ula_y_plane_size = 0;
  1314. phys_addr_t ula_pa = 0x0;
  1315. struct ubwcp_buf *buf;
  1316. struct ubwcp_driver *ubwcp;
  1317. size_t metadata_p0;
  1318. size_t pixeldata_p0;
  1319. size_t metadata_p1;
  1320. size_t pixeldata_p1;
  1321. size_t iova_min_size;
  1322. size_t stride_tp10_b;
  1323. dma_addr_t iova_base;
  1324. struct ubwcp_hw_meta_metadata *mmdata;
  1325. u64 uv_start;
  1326. u32 stride_b;
  1327. u32 width_b;
  1328. u32 height_b;
  1329. enum ubwcp_std_image_format std_image_format;
  1330. bool is_non_lin_buf;
  1331. FENTRY();
  1332. trace_ubwcp_set_buf_attrs_start(dmabuf);
  1333. if (!dmabuf) {
  1334. ERR("NULL dmabuf input ptr");
  1335. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1336. return -EINVAL;
  1337. }
  1338. if (!attr) {
  1339. ERR("NULL attr ptr");
  1340. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1341. return -EINVAL;
  1342. }
  1343. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1344. if (!buf) {
  1345. ERR("No corresponding ubwcp_buf for the passed in dma_buf");
  1346. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1347. return -EINVAL;
  1348. }
  1349. mutex_lock(&buf->lock);
  1350. if (buf->locked) {
  1351. ERR("Cannot set attr when buffer is locked");
  1352. ret = -EBUSY;
  1353. goto unlock;
  1354. }
  1355. ubwcp = buf->ubwcp;
  1356. mmdata = &buf->mmdata;
  1357. is_non_lin_buf = (buf->buf_attr.image_format != UBWCP_LINEAR);
  1358. //TBD: now that we have single exit point for all errors,
  1359. //we can limit this call to error only?
  1360. //also see if this can be part of reset_buf_attrs()
  1361. DBG_BUF_ATTR("resetting mmap to linear");
  1362. /* remove any earlier dma buf mmap configuration */
  1363. ret = ubwcp->mmap_config_fptr(buf->dma_buf, true, 0, 0);
  1364. if (ret) {
  1365. ERR("dma_buf_mmap_config() failed: %d", ret);
  1366. goto unlock;
  1367. }
  1368. if (!ubwcp_buf_attrs_valid(ubwcp, attr)) {
  1369. ERR("Invalid buf attrs");
  1370. goto err;
  1371. }
  1372. if (attr->image_format == UBWCP_LINEAR) {
  1373. DBG_BUF_ATTR("Linear format requested");
  1374. /* linear format request with permanent range xlation doesn't
  1375. * make sense. need to define behavior if this happens.
  1376. * note: with perm set, desc is allocated to this buffer.
  1377. */
  1378. //TBD: UBWCP_ASSERT(!buf->perm);
  1379. if (buf->buf_attr_set)
  1380. reset_buf_attrs(buf);
  1381. if (is_non_lin_buf) {
  1382. /*
  1383. * Changing buffer from ubwc to linear so decrement
  1384. * number of ubwc buffers
  1385. */
  1386. ret = dec_num_non_lin_buffers(ubwcp);
  1387. }
  1388. mutex_unlock(&buf->lock);
  1389. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1390. return ret;
  1391. }
  1392. std_image_format = to_std_format(attr->image_format);
  1393. if (std_image_format == STD_IMAGE_FORMAT_INVALID) {
  1394. ERR("Unable to map ioctl image format to std image format");
  1395. goto err;
  1396. }
  1397. /* Calculate uncompressed-buffer size. */
  1398. DBG_BUF_ATTR("");
  1399. DBG_BUF_ATTR("");
  1400. DBG_BUF_ATTR("Calculating ula params -->");
  1401. ret = ubwcp_calc_ula_params(ubwcp, attr, &ula_size, &ula_y_plane_size, &uv_start_offset);
  1402. if (ret) {
  1403. ERR("ubwcp_calc_ula_params() failed: %d", ret);
  1404. goto err;
  1405. }
  1406. ret = ubwcp_validate_uv_align(ubwcp, attr, ula_y_plane_size, uv_start_offset);
  1407. if (ret) {
  1408. ERR("ubwcp_validate_uv_align() failed: %d", ret);
  1409. goto err;
  1410. }
  1411. DBG_BUF_ATTR("");
  1412. DBG_BUF_ATTR("");
  1413. DBG_BUF_ATTR("Calculating ubwcp params -->");
  1414. ret = ubwcp_calc_ubwcp_buf_params(ubwcp, attr,
  1415. &metadata_p0, &pixeldata_p0,
  1416. &metadata_p1, &pixeldata_p1,
  1417. &stride_tp10_b);
  1418. if (ret) {
  1419. ERR("ubwcp_calc_buf_params() failed: %d", ret);
  1420. goto err;
  1421. }
  1422. iova_min_size = metadata_p0 + pixeldata_p0 + metadata_p1 + pixeldata_p1;
  1423. DBG_BUF_ATTR("");
  1424. DBG_BUF_ATTR("");
  1425. DBG_BUF_ATTR("------Summary ULA Calculated Params ------");
  1426. DBG_BUF_ATTR("ULA Size : %8zu (0x%8zx)", ula_size, ula_size);
  1427. DBG_BUF_ATTR("UV Start Offset : %8zu (0x%8zx)", uv_start_offset, uv_start_offset);
  1428. DBG_BUF_ATTR("------Summary UBCP Calculated Params ------");
  1429. DBG_BUF_ATTR("metadata_p0 : %8d (0x%8zx)", metadata_p0, metadata_p0);
  1430. DBG_BUF_ATTR("pixeldata_p0 : %8d (0x%8zx)", pixeldata_p0, pixeldata_p0);
  1431. DBG_BUF_ATTR("metadata_p1 : %8d (0x%8zx)", metadata_p1, metadata_p1);
  1432. DBG_BUF_ATTR("pixeldata_p1 : %8d (0x%8zx)", pixeldata_p1, pixeldata_p1);
  1433. DBG_BUF_ATTR("stride_tp10 : %8d (0x%8zx)", stride_tp10_b, stride_tp10_b);
  1434. DBG_BUF_ATTR("iova_min_size : %8d (0x%8zx)", iova_min_size, iova_min_size);
  1435. DBG_BUF_ATTR("");
  1436. if (buf->buf_attr_set) {
  1437. /* if buf attr were previously set, these must not be 0 */
  1438. /* TBD: do we need this check in production code? */
  1439. if (!buf->ula_pa) {
  1440. WARN(1, "ula_pa cannot be 0 if buf_attr_set is true!!!");
  1441. goto err;
  1442. }
  1443. if (!buf->ula_size) {
  1444. WARN(1, "ula_size cannot be 0 if buf_attr_set is true!!!");
  1445. goto err;
  1446. }
  1447. }
  1448. /* assign ULA PA with uncompressed-size range */
  1449. ula_pa = ubwcp_ula_realloc(ubwcp, buf->ula_pa, buf->ula_size, ula_size);
  1450. if (!ula_pa) {
  1451. ERR("ubwcp_ula_alloc/realloc() failed. running out of ULA PA space?");
  1452. goto err;
  1453. }
  1454. buf->ula_size = ula_size;
  1455. buf->ula_pa = ula_pa;
  1456. DBG_BUF_ATTR("Allocated ULA_PA: 0x%p of size: 0x%zx", ula_pa, ula_size);
  1457. DBG_BUF_ATTR("");
  1458. /* inform ULA-PA to dma-heap: needed for dma-heap to do CMOs later on */
  1459. DBG_BUF_ATTR("Calling mmap_config(): ULA_PA: 0x%p size: 0x%zx", ula_pa, ula_size);
  1460. ret = ubwcp->mmap_config_fptr(buf->dma_buf, false, buf->ula_pa,
  1461. buf->ula_size);
  1462. if (ret) {
  1463. ERR("dma_buf_mmap_config() failed: %d", ret);
  1464. goto err;
  1465. }
  1466. /* dma map only the first time attribute is set */
  1467. if (!buf->buf_attr_set) {
  1468. /* linear -> ubwcp. map ubwcp buffer */
  1469. ret = ubwcp_dma_map(buf, ubwcp->dev_buf_cb, iova_min_size, &iova_base);
  1470. if (ret) {
  1471. ERR("ubwcp_dma_map() failed: %d", ret);
  1472. goto err;
  1473. }
  1474. DBG_BUF_ATTR("dma_buf IOVA range: 0x%llx + min_size (0x%zx): 0x%llx",
  1475. iova_base, iova_min_size, iova_base + iova_min_size);
  1476. }
  1477. uv_start = ula_pa + uv_start_offset;
  1478. if (!IS_ALIGNED(uv_start, 64)) {
  1479. ERR("ERROR: uv_start is NOT aligned to cache line");
  1480. goto err;
  1481. }
  1482. /* Convert height and width to bytes for writing to mmdata */
  1483. if (std_image_format != TP10) {
  1484. ubwcp_pixel_to_bytes(ubwcp, std_image_format, attr->width,
  1485. attr->height, &width_b, &height_b);
  1486. } else {
  1487. /* for tp10 image compression, we need to program p010 width/height */
  1488. ubwcp_pixel_to_bytes(ubwcp, P010, attr->width,
  1489. attr->height, &width_b, &height_b);
  1490. }
  1491. stride_b = attr->stride;
  1492. /* create the mmdata descriptor */
  1493. memset(mmdata, 0, sizeof(*mmdata));
  1494. mmdata->uv_start_addr = CACHE_ADDR(uv_start);
  1495. mmdata->format = ubwcp_get_hw_image_format_value(attr->image_format);
  1496. if (std_image_format != TP10) {
  1497. mmdata->stride = CACHE_ADDR(stride_b); /* uncompressed stride */
  1498. } else {
  1499. mmdata->stride = CACHE_ADDR(stride_tp10_b); /* compressed stride */
  1500. mmdata->stride_ubwcp = CACHE_ADDR(stride_b); /* uncompressed stride */
  1501. }
  1502. mmdata->metadata_base_y = PAGE_ADDR(iova_base);
  1503. mmdata->metadata_base_uv = PAGE_ADDR(iova_base + metadata_p0 + pixeldata_p0);
  1504. mmdata->buffer_y_offset = PAGE_ADDR(metadata_p0);
  1505. mmdata->buffer_uv_offset = PAGE_ADDR(metadata_p1);
  1506. /* NOTE: For version 1.1, both width & height needs to be in bytes.
  1507. * For other versions, width in bytes & height in pixels.
  1508. */
  1509. if ((ubwcp->hw_ver_major == 1) && (ubwcp->hw_ver_minor == 1))
  1510. mmdata->width_height = width_b << 16 | height_b;
  1511. else
  1512. mmdata->width_height = width_b << 16 | attr->height;
  1513. print_mmdata_desc(mmdata);
  1514. if (!is_non_lin_buf) {
  1515. /*
  1516. * Changing buffer from linear to ubwc so increment
  1517. * number of ubwc buffers
  1518. */
  1519. ret = inc_num_non_lin_buffers(ubwcp);
  1520. }
  1521. if (ret) {
  1522. ERR("inc_num_non_lin_buffers failed: %d", ret);
  1523. goto err;
  1524. }
  1525. buf->buf_attr = *attr;
  1526. buf->buf_attr_set = true;
  1527. //TBD: UBWCP_ASSERT(!buf->perm);
  1528. mutex_unlock(&buf->lock);
  1529. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1530. return 0;
  1531. err:
  1532. reset_buf_attrs(buf);
  1533. if (is_non_lin_buf) {
  1534. /*
  1535. * Changing buffer from ubwc to linear so decrement
  1536. * number of ubwc buffers
  1537. */
  1538. dec_num_non_lin_buffers(ubwcp);
  1539. }
  1540. unlock:
  1541. mutex_unlock(&buf->lock);
  1542. if (!ret)
  1543. ret = -1;
  1544. trace_ubwcp_set_buf_attrs_end(dmabuf);
  1545. return ret;
  1546. }
  1547. EXPORT_SYMBOL(ubwcp_set_buf_attrs);
  1548. /* Set buffer attributes ioctl */
  1549. static int ubwcp_set_buf_attrs_ioctl(struct ubwcp_ioctl_buffer_attrs *attr_ioctl)
  1550. {
  1551. struct dma_buf *dmabuf;
  1552. dmabuf = ubwcp_dma_buf_fd_to_dma_buf(attr_ioctl->fd);
  1553. return ubwcp_set_buf_attrs(dmabuf, &attr_ioctl->attr);
  1554. }
  1555. /* Free up the buffer descriptor */
  1556. static void ubwcp_buf_desc_free(struct ubwcp_driver *ubwcp, struct ubwcp_desc *desc)
  1557. {
  1558. int idx = desc->idx;
  1559. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  1560. mutex_lock(&ubwcp->desc_lock);
  1561. desc_list[idx].idx = -1;
  1562. desc_list[idx].ptr = NULL;
  1563. DBG("freed descriptor_id: %d", idx);
  1564. mutex_unlock(&ubwcp->desc_lock);
  1565. }
  1566. /* Allocate next available buffer descriptor. */
  1567. static struct ubwcp_desc *ubwcp_buf_desc_allocate(struct ubwcp_driver *ubwcp)
  1568. {
  1569. int idx;
  1570. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  1571. mutex_lock(&ubwcp->desc_lock);
  1572. for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
  1573. if (desc_list[idx].idx == -1) {
  1574. desc_list[idx].idx = idx;
  1575. desc_list[idx].ptr = ubwcp->buffer_desc_base +
  1576. idx*UBWCP_BUFFER_DESC_OFFSET;
  1577. DBG("allocated descriptor_id: %d", idx);
  1578. mutex_unlock(&ubwcp->desc_lock);
  1579. return &desc_list[idx];
  1580. }
  1581. }
  1582. mutex_unlock(&ubwcp->desc_lock);
  1583. return NULL;
  1584. }
  1585. /**
  1586. * Lock buffer for CPU access. This prepares ubwcp hw to allow
  1587. * CPU access to the compressed buffer. It will perform
  1588. * necessary address translation configuration and cache maintenance ops
  1589. * so that CPU can safely access ubwcp buffer, if this call is
  1590. * successful.
  1591. * Allocate descriptor if not already,
  1592. * perform CMO and then enable range check
  1593. *
  1594. * @param dmabuf : ptr to the dma buf
  1595. * @param direction : direction of access
  1596. *
  1597. * @return int : 0 on success, otherwise error code
  1598. */
  1599. static int ubwcp_lock(struct dma_buf *dmabuf, enum dma_data_direction dir)
  1600. {
  1601. int ret = 0;
  1602. struct ubwcp_buf *buf;
  1603. struct ubwcp_driver *ubwcp;
  1604. FENTRY();
  1605. trace_ubwcp_lock_start(dmabuf);
  1606. if (!dmabuf) {
  1607. ERR("NULL dmabuf input ptr");
  1608. trace_ubwcp_lock_end(dmabuf);
  1609. return -EINVAL;
  1610. }
  1611. if (!valid_dma_direction(dir)) {
  1612. ERR("invalid direction: %d", dir);
  1613. trace_ubwcp_lock_end(dmabuf);
  1614. return -EINVAL;
  1615. }
  1616. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1617. if (!buf) {
  1618. ERR("ubwcp_buf ptr not found");
  1619. trace_ubwcp_lock_end(dmabuf);
  1620. return -1;
  1621. }
  1622. mutex_lock(&buf->lock);
  1623. if (!buf->buf_attr_set) {
  1624. ERR("lock() called on buffer, but attr not set");
  1625. goto err;
  1626. }
  1627. if (buf->buf_attr.image_format == UBWCP_LINEAR) {
  1628. ERR("lock() called on linear buffer");
  1629. goto err;
  1630. }
  1631. if (!buf->locked) {
  1632. DBG("first lock on buffer");
  1633. ubwcp = buf->ubwcp;
  1634. /* buf->desc could already be allocated because of perm range xlation */
  1635. if (!buf->desc) {
  1636. /* allocate a buffer descriptor */
  1637. buf->desc = ubwcp_buf_desc_allocate(buf->ubwcp);
  1638. if (!buf->desc) {
  1639. ERR("ubwcp_allocate_buf_desc() failed");
  1640. goto err;
  1641. }
  1642. memcpy(buf->desc->ptr, &buf->mmdata, sizeof(buf->mmdata));
  1643. /* Flushing of updated mmdata:
  1644. * mmdata is iocoherent and ubwcp will get it from CPU cache -
  1645. * *as long as* it has not cached that itself during previous
  1646. * access to the same descriptor.
  1647. *
  1648. * During unlock of previous use of this descriptor,
  1649. * we do hw flush, which will get rid of this mmdata from
  1650. * ubwcp cache.
  1651. *
  1652. * In addition, we also do a hw flush after enable_range_ck().
  1653. * That will also get rid of any speculative fetch of mmdata
  1654. * by the ubwcp hw. At this time, the assumption is that ubwcp
  1655. * will cache mmdata only for active descriptor. But if ubwcp
  1656. * is speculatively fetching mmdata for all descriptors
  1657. * (irrespetive of enabled or not), the flush during lock
  1658. * will be necessary to make sure ubwcp sees updated mmdata
  1659. * that we just updated
  1660. */
  1661. /* program ULA range for this buffer */
  1662. DBG("setting range check: descriptor_id: %d, addr: %p, size: %zx",
  1663. buf->desc->idx, buf->ula_pa, buf->ula_size);
  1664. ubwcp_hw_set_range_check(ubwcp->base, buf->desc->idx, buf->ula_pa,
  1665. buf->ula_size);
  1666. }
  1667. /* enable range check */
  1668. DBG("enabling range check, descriptor_id: %d", buf->desc->idx);
  1669. mutex_lock(&ubwcp->hw_range_ck_lock);
  1670. ubwcp_hw_enable_range_check(ubwcp->base, buf->desc->idx);
  1671. mutex_unlock(&ubwcp->hw_range_ck_lock);
  1672. /* Flush/invalidate UBWCP caches */
  1673. /* Why: cpu could have done a speculative fetch before
  1674. * enable_range_ck() and ubwcp in process of returning "default" data
  1675. * we don't want that stashing of default data pending.
  1676. * we force completion of that and then we also cpu invalidate which
  1677. * will get rid of that line.
  1678. */
  1679. trace_ubwcp_hw_flush_start(buf->ula_size);
  1680. ubwcp_flush(ubwcp);
  1681. trace_ubwcp_hw_flush_end(buf->ula_size);
  1682. /* Flush/invalidate ULA PA from CPU caches
  1683. * TBD: if (dir == READ or BIDIRECTION) //NOT for write
  1684. * -- Confirm with Chris if this can be skipped for write
  1685. */
  1686. trace_ubwcp_dma_sync_single_for_cpu_start(buf->ula_size);
  1687. dma_sync_single_for_cpu(ubwcp->dev, buf->ula_pa, buf->ula_size, dir);
  1688. trace_ubwcp_dma_sync_single_for_cpu_end(buf->ula_size);
  1689. buf->lock_dir = dir;
  1690. buf->locked = true;
  1691. } else {
  1692. DBG("buf already locked");
  1693. /* TBD: what if new buffer direction is not same as previous?
  1694. * must update the dir.
  1695. */
  1696. }
  1697. buf->lock_count++;
  1698. DBG("new lock_count: %d", buf->lock_count);
  1699. mutex_unlock(&buf->lock);
  1700. trace_ubwcp_lock_end(dmabuf);
  1701. return ret;
  1702. err:
  1703. mutex_unlock(&buf->lock);
  1704. if (!ret)
  1705. ret = -1;
  1706. trace_ubwcp_lock_end(dmabuf);
  1707. return ret;
  1708. }
  1709. /* This can be called as a result of external unlock() call or
  1710. * internally if free() is called without unlock().
  1711. */
  1712. static int unlock_internal(struct ubwcp_buf *buf, enum dma_data_direction dir, bool free_buffer)
  1713. {
  1714. int ret = 0;
  1715. struct ubwcp_driver *ubwcp;
  1716. DBG("current lock_count: %d", buf->lock_count);
  1717. if (free_buffer) {
  1718. buf->lock_count = 0;
  1719. DBG("Forced lock_count: %d", buf->lock_count);
  1720. } else {
  1721. buf->lock_count--;
  1722. DBG("new lock_count: %d", buf->lock_count);
  1723. if (buf->lock_count) {
  1724. DBG("more than 1 lock on buffer. waiting until last unlock");
  1725. return 0;
  1726. }
  1727. }
  1728. ubwcp = buf->ubwcp;
  1729. /* Flush/invalidate ULA PA from CPU caches */
  1730. //TBD: if (dir == WRITE or BIDIRECTION)
  1731. trace_ubwcp_dma_sync_single_for_device_start(buf->ula_size);
  1732. dma_sync_single_for_device(ubwcp->dev, buf->ula_pa, buf->ula_size, dir);
  1733. trace_ubwcp_dma_sync_single_for_device_end(buf->ula_size);
  1734. /* disable range check with ubwcp flush */
  1735. DBG("disabling range check");
  1736. //TBD: could combine these 2 locks into a single lock to make it simpler
  1737. mutex_lock(&ubwcp->ubwcp_flush_lock);
  1738. mutex_lock(&ubwcp->hw_range_ck_lock);
  1739. trace_ubwcp_hw_flush_start(buf->ula_size);
  1740. ret = ubwcp_hw_disable_range_check_with_flush(ubwcp->base, buf->desc->idx);
  1741. trace_ubwcp_hw_flush_end(buf->ula_size);
  1742. if (ret)
  1743. ERR("disable_range_check_with_flush() failed: %d", ret);
  1744. mutex_unlock(&ubwcp->hw_range_ck_lock);
  1745. mutex_unlock(&ubwcp->ubwcp_flush_lock);
  1746. /* release descriptor if perm range xlation is not set */
  1747. if (!buf->perm) {
  1748. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1749. buf->desc = NULL;
  1750. }
  1751. buf->locked = false;
  1752. return ret;
  1753. }
  1754. /**
  1755. * Unlock buffer from CPU access. This prepares ubwcp hw to
  1756. * safely allow for device access to the compressed buffer including any
  1757. * necessary cache maintenance ops. It may also free up certain ubwcp
  1758. * resources that could result in error when accessed by CPU in
  1759. * unlocked state.
  1760. *
  1761. * @param dmabuf : ptr to the dma buf
  1762. * @param direction : direction of access
  1763. *
  1764. * @return int : 0 on success, otherwise error code
  1765. */
  1766. static int ubwcp_unlock(struct dma_buf *dmabuf, enum dma_data_direction dir)
  1767. {
  1768. struct ubwcp_buf *buf;
  1769. int ret;
  1770. FENTRY();
  1771. trace_ubwcp_unlock_start(dmabuf);
  1772. if (!dmabuf) {
  1773. ERR("NULL dmabuf input ptr");
  1774. trace_ubwcp_unlock_end(dmabuf);
  1775. return -EINVAL;
  1776. }
  1777. if (!valid_dma_direction(dir)) {
  1778. ERR("invalid direction: %d", dir);
  1779. trace_ubwcp_unlock_end(dmabuf);
  1780. return -EINVAL;
  1781. }
  1782. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1783. if (!buf) {
  1784. ERR("ubwcp_buf not found");
  1785. trace_ubwcp_unlock_end(dmabuf);
  1786. return -1;
  1787. }
  1788. if (!buf->locked) {
  1789. ERR("unlock() called on buffer which not in locked state");
  1790. trace_ubwcp_unlock_end(dmabuf);
  1791. return -1;
  1792. }
  1793. mutex_lock(&buf->lock);
  1794. ret = unlock_internal(buf, dir, false);
  1795. mutex_unlock(&buf->lock);
  1796. trace_ubwcp_unlock_end(dmabuf);
  1797. return ret;
  1798. }
  1799. /* Return buffer attributes for the given buffer */
  1800. int ubwcp_get_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
  1801. {
  1802. int ret = 0;
  1803. struct ubwcp_buf *buf;
  1804. FENTRY();
  1805. if (!dmabuf) {
  1806. ERR("NULL dmabuf input ptr");
  1807. return -EINVAL;
  1808. }
  1809. if (!attr) {
  1810. ERR("NULL attr ptr");
  1811. return -EINVAL;
  1812. }
  1813. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1814. if (!buf) {
  1815. ERR("ubwcp_buf ptr not found");
  1816. return -1;
  1817. }
  1818. mutex_lock(&buf->lock);
  1819. if (!buf->buf_attr_set) {
  1820. ERR("buffer attributes not set");
  1821. mutex_unlock(&buf->lock);
  1822. return -1;
  1823. }
  1824. *attr = buf->buf_attr;
  1825. mutex_unlock(&buf->lock);
  1826. return ret;
  1827. }
  1828. EXPORT_SYMBOL(ubwcp_get_buf_attrs);
  1829. /* Set permanent range translation.
  1830. * enable: Descriptor will be reserved for this buffer until disabled,
  1831. * making lock/unlock quicker.
  1832. * disable: Descriptor will not be reserved for this buffer. Instead,
  1833. * descriptor will be allocated and released for each lock/unlock.
  1834. * If currently allocated but not being used, descriptor will be
  1835. * released.
  1836. */
  1837. int ubwcp_set_perm_range_translation(struct dma_buf *dmabuf, bool enable)
  1838. {
  1839. int ret = 0;
  1840. struct ubwcp_buf *buf;
  1841. FENTRY();
  1842. if (!dmabuf) {
  1843. ERR("NULL dmabuf input ptr");
  1844. return -EINVAL;
  1845. }
  1846. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1847. if (!buf) {
  1848. ERR("ubwcp_buf not found");
  1849. return -1;
  1850. }
  1851. /* not implemented */
  1852. if (1) {
  1853. ERR("API not implemented yet");
  1854. return -1;
  1855. }
  1856. /* TBD: make sure we acquire buf lock while setting this so there is
  1857. * no race condition with attr_set/lock/unlock
  1858. */
  1859. buf->perm = enable;
  1860. /* if "disable" and we have allocated a desc and it is not being
  1861. * used currently, release it
  1862. */
  1863. if (!enable && buf->desc && !buf->locked) {
  1864. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1865. buf->desc = NULL;
  1866. /* Flush/invalidate UBWCP caches */
  1867. //TBD: need to do anything?
  1868. }
  1869. return ret;
  1870. }
  1871. EXPORT_SYMBOL(ubwcp_set_perm_range_translation);
  1872. /**
  1873. * Free up ubwcp resources for this buffer.
  1874. *
  1875. * @param dmabuf : ptr to the dma buf
  1876. *
  1877. * @return int : 0 on success, otherwise error code
  1878. */
  1879. static int ubwcp_free_buffer(struct dma_buf *dmabuf)
  1880. {
  1881. int ret = 0;
  1882. struct ubwcp_buf *buf;
  1883. struct ubwcp_driver *ubwcp;
  1884. unsigned long flags;
  1885. bool is_non_lin_buf;
  1886. FENTRY();
  1887. trace_ubwcp_free_buffer_start(dmabuf);
  1888. if (!dmabuf) {
  1889. ERR("NULL dmabuf input ptr");
  1890. trace_ubwcp_free_buffer_end(dmabuf);
  1891. return -EINVAL;
  1892. }
  1893. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1894. if (!buf) {
  1895. ERR("ubwcp_buf ptr not found");
  1896. trace_ubwcp_free_buffer_end(dmabuf);
  1897. return -1;
  1898. }
  1899. mutex_lock(&buf->lock);
  1900. ubwcp = buf->ubwcp;
  1901. is_non_lin_buf = (buf->buf_attr.image_format != UBWCP_LINEAR);
  1902. if (buf->locked) {
  1903. DBG("free() called without unlock. unlock()'ing first...");
  1904. ret = unlock_internal(buf, buf->lock_dir, true);
  1905. if (ret)
  1906. ERR("unlock_internal(): failed : %d, but continuing free()", ret);
  1907. }
  1908. /* if we are still holding a desc, release it. this can happen only if perm == true */
  1909. if (buf->desc) {
  1910. WARN_ON(!buf->perm); /* TBD: change to BUG() later...*/
  1911. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1912. buf->desc = NULL;
  1913. }
  1914. if (buf->buf_attr_set)
  1915. reset_buf_attrs(buf);
  1916. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  1917. hash_del(&buf->hnode);
  1918. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  1919. kfree(buf);
  1920. if (is_non_lin_buf)
  1921. dec_num_non_lin_buffers(ubwcp);
  1922. trace_ubwcp_free_buffer_end(dmabuf);
  1923. return 0;
  1924. }
  1925. /* file open: TBD: increment ref count? */
  1926. static int ubwcp_open(struct inode *i, struct file *f)
  1927. {
  1928. return 0;
  1929. }
  1930. /* file open: TBD: decrement ref count? */
  1931. static int ubwcp_close(struct inode *i, struct file *f)
  1932. {
  1933. return 0;
  1934. }
  1935. /* handle IOCTLs */
  1936. static long ubwcp_ioctl(struct file *file, unsigned int ioctl_num, unsigned long ioctl_param)
  1937. {
  1938. struct ubwcp_ioctl_buffer_attrs buf_attr_ioctl;
  1939. struct ubwcp_ioctl_hw_version hw_ver;
  1940. struct ubwcp_ioctl_validate_stride validate_stride_ioctl;
  1941. struct ubwcp_ioctl_stride_align stride_align_ioctl;
  1942. enum ubwcp_std_image_format format;
  1943. struct ubwcp_driver *ubwcp;
  1944. switch (ioctl_num) {
  1945. case UBWCP_IOCTL_SET_BUF_ATTR:
  1946. if (copy_from_user(&buf_attr_ioctl, (const void __user *) ioctl_param,
  1947. sizeof(buf_attr_ioctl))) {
  1948. ERR("ERROR: copy_from_user() failed");
  1949. return -EFAULT;
  1950. }
  1951. DBG("IOCTL : SET_BUF_ATTR: fd = %d", buf_attr_ioctl.fd);
  1952. return ubwcp_set_buf_attrs_ioctl(&buf_attr_ioctl);
  1953. case UBWCP_IOCTL_GET_HW_VER:
  1954. DBG("IOCTL : GET_HW_VER");
  1955. ubwcp_get_hw_version(&hw_ver);
  1956. if (copy_to_user((void __user *)ioctl_param, &hw_ver, sizeof(hw_ver))) {
  1957. ERR("ERROR: copy_to_user() failed");
  1958. return -EFAULT;
  1959. }
  1960. break;
  1961. case UBWCP_IOCTL_GET_STRIDE_ALIGN:
  1962. DBG("IOCTL : GET_STRIDE_ALIGN");
  1963. if (copy_from_user(&stride_align_ioctl, (const void __user *) ioctl_param,
  1964. sizeof(stride_align_ioctl))) {
  1965. ERR("ERROR: copy_from_user() failed");
  1966. return -EFAULT;
  1967. }
  1968. format = to_std_format(stride_align_ioctl.image_format);
  1969. if (format == STD_IMAGE_FORMAT_INVALID)
  1970. return -EINVAL;
  1971. if (stride_align_ioctl.unused != 0)
  1972. return -EINVAL;
  1973. if (get_stride_alignment(format, &stride_align_ioctl.stride_align)) {
  1974. ERR("ERROR: copy_to_user() failed");
  1975. return -EFAULT;
  1976. }
  1977. if (copy_to_user((void __user *)ioctl_param, &stride_align_ioctl,
  1978. sizeof(stride_align_ioctl))) {
  1979. ERR("ERROR: copy_to_user() failed");
  1980. return -EFAULT;
  1981. }
  1982. break;
  1983. case UBWCP_IOCTL_VALIDATE_STRIDE:
  1984. DBG("IOCTL : VALIDATE_STRIDE");
  1985. ubwcp = ubwcp_get_driver();
  1986. if (!ubwcp)
  1987. return -EINVAL;
  1988. if (copy_from_user(&validate_stride_ioctl, (const void __user *) ioctl_param,
  1989. sizeof(validate_stride_ioctl))) {
  1990. ERR("ERROR: copy_from_user() failed");
  1991. return -EFAULT;
  1992. }
  1993. format = to_std_format(validate_stride_ioctl.image_format);
  1994. if (format == STD_IMAGE_FORMAT_INVALID) {
  1995. ERR("ERROR: invalid format: %d", validate_stride_ioctl.image_format);
  1996. return -EINVAL;
  1997. }
  1998. if (validate_stride_ioctl.unused1 || validate_stride_ioctl.unused2) {
  1999. ERR("ERROR: unused values must be set to 0");
  2000. return -EINVAL;
  2001. }
  2002. validate_stride_ioctl.valid = stride_is_valid(ubwcp,
  2003. validate_stride_ioctl.image_format,
  2004. validate_stride_ioctl.width,
  2005. validate_stride_ioctl.stride);
  2006. if (copy_to_user((void __user *)ioctl_param, &validate_stride_ioctl,
  2007. sizeof(validate_stride_ioctl))) {
  2008. ERR("ERROR: copy_to_user() failed");
  2009. return -EFAULT;
  2010. }
  2011. break;
  2012. default:
  2013. ERR("Invalid ioctl_num = %d", ioctl_num);
  2014. return -EINVAL;
  2015. }
  2016. return 0;
  2017. }
  2018. static const struct file_operations ubwcp_fops = {
  2019. .owner = THIS_MODULE,
  2020. .open = ubwcp_open,
  2021. .release = ubwcp_close,
  2022. .unlocked_ioctl = ubwcp_ioctl,
  2023. };
  2024. static int read_err_r_op(void *data, u64 *value)
  2025. {
  2026. struct ubwcp_driver *ubwcp = data;
  2027. *value = ubwcp->read_err_irq_en;
  2028. return 0;
  2029. }
  2030. static int read_err_w_op(void *data, u64 value)
  2031. {
  2032. struct ubwcp_driver *ubwcp = data;
  2033. if (ubwcp_power(ubwcp, true))
  2034. return -1;
  2035. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, value);
  2036. ubwcp->read_err_irq_en = value;
  2037. if (ubwcp_power(ubwcp, false))
  2038. return -1;
  2039. return 0;
  2040. }
  2041. static int write_err_r_op(void *data, u64 *value)
  2042. {
  2043. struct ubwcp_driver *ubwcp = data;
  2044. *value = ubwcp->write_err_irq_en;
  2045. return 0;
  2046. }
  2047. static int write_err_w_op(void *data, u64 value)
  2048. {
  2049. struct ubwcp_driver *ubwcp = data;
  2050. if (ubwcp_power(ubwcp, true))
  2051. return -1;
  2052. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, value);
  2053. ubwcp->write_err_irq_en = value;
  2054. if (ubwcp_power(ubwcp, false))
  2055. return -1;
  2056. return 0;
  2057. }
  2058. static int decode_err_r_op(void *data, u64 *value)
  2059. {
  2060. struct ubwcp_driver *ubwcp = data;
  2061. *value = ubwcp->decode_err_irq_en;
  2062. return 0;
  2063. }
  2064. static int decode_err_w_op(void *data, u64 value)
  2065. {
  2066. struct ubwcp_driver *ubwcp = data;
  2067. if (ubwcp_power(ubwcp, true))
  2068. return -1;
  2069. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, value);
  2070. ubwcp->decode_err_irq_en = value;
  2071. if (ubwcp_power(ubwcp, false))
  2072. return -1;
  2073. return 0;
  2074. }
  2075. static int encode_err_r_op(void *data, u64 *value)
  2076. {
  2077. struct ubwcp_driver *ubwcp = data;
  2078. *value = ubwcp->encode_err_irq_en;
  2079. return 0;
  2080. }
  2081. static int encode_err_w_op(void *data, u64 value)
  2082. {
  2083. struct ubwcp_driver *ubwcp = data;
  2084. if (ubwcp_power(ubwcp, true))
  2085. return -1;
  2086. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, value);
  2087. ubwcp->encode_err_irq_en = value;
  2088. if (ubwcp_power(ubwcp, false))
  2089. return -1;
  2090. return 0;
  2091. }
  2092. static int reg_rw_trace_w_op(void *data, u64 value)
  2093. {
  2094. ubwcp_hw_trace_set(value);
  2095. return 0;
  2096. }
  2097. static int reg_rw_trace_r_op(void *data, u64 *value)
  2098. {
  2099. bool trace_status;
  2100. ubwcp_hw_trace_get(&trace_status);
  2101. *value = trace_status;
  2102. return 0;
  2103. }
  2104. DEFINE_DEBUGFS_ATTRIBUTE(read_err_fops, read_err_r_op, read_err_w_op, "%d\n");
  2105. DEFINE_DEBUGFS_ATTRIBUTE(decode_err_fops, decode_err_r_op, decode_err_w_op, "%d\n");
  2106. DEFINE_DEBUGFS_ATTRIBUTE(write_err_fops, write_err_r_op, write_err_w_op, "%d\n");
  2107. DEFINE_DEBUGFS_ATTRIBUTE(encode_err_fops, encode_err_r_op, encode_err_w_op, "%d\n");
  2108. DEFINE_DEBUGFS_ATTRIBUTE(reg_rw_trace_fops, reg_rw_trace_r_op, reg_rw_trace_w_op, "%d\n");
  2109. static void ubwcp_debugfs_init(struct ubwcp_driver *ubwcp)
  2110. {
  2111. struct dentry *debugfs_root;
  2112. struct dentry *dfile;
  2113. debugfs_root = debugfs_create_dir("ubwcp", NULL);
  2114. if (IS_ERR_OR_NULL(debugfs_root)) {
  2115. ERR("Failed to create debugfs for ubwcp\n");
  2116. return;
  2117. }
  2118. debugfs_create_u32("debug_trace_enable", 0644, debugfs_root, &ubwcp_debug_trace_enable);
  2119. dfile = debugfs_create_file("reg_rw_trace_en", 0644, debugfs_root, ubwcp, &reg_rw_trace_fops);
  2120. if (IS_ERR_OR_NULL(dfile)) {
  2121. ERR("failed to create reg_rw_trace_en debugfs file");
  2122. goto err;
  2123. }
  2124. dfile = debugfs_create_file("read_err_irq_en", 0644, debugfs_root, ubwcp, &read_err_fops);
  2125. if (IS_ERR_OR_NULL(dfile)) {
  2126. ERR("failed to create read_err_irq debugfs file");
  2127. goto err;
  2128. }
  2129. dfile = debugfs_create_file("write_err_irq_en", 0644, debugfs_root, ubwcp, &write_err_fops);
  2130. if (IS_ERR_OR_NULL(dfile)) {
  2131. ERR("failed to create write_err_irq debugfs file");
  2132. goto err;
  2133. }
  2134. dfile = debugfs_create_file("decode_err_irq_en", 0644, debugfs_root, ubwcp,
  2135. &decode_err_fops);
  2136. if (IS_ERR_OR_NULL(dfile)) {
  2137. ERR("failed to create decode_err_irq debugfs file");
  2138. goto err;
  2139. }
  2140. dfile = debugfs_create_file("encode_err_irq_en", 0644, debugfs_root, ubwcp,
  2141. &encode_err_fops);
  2142. if (IS_ERR_OR_NULL(dfile)) {
  2143. ERR("failed to create encode_err_irq debugfs file");
  2144. goto err;
  2145. }
  2146. ubwcp->debugfs_root = debugfs_root;
  2147. return;
  2148. err:
  2149. debugfs_remove_recursive(ubwcp->debugfs_root);
  2150. ubwcp->debugfs_root = NULL;
  2151. }
  2152. static void ubwcp_debugfs_deinit(struct ubwcp_driver *ubwcp)
  2153. {
  2154. debugfs_remove_recursive(ubwcp->debugfs_root);
  2155. }
  2156. /* ubwcp char device initialization */
  2157. static int ubwcp_cdev_init(struct ubwcp_driver *ubwcp)
  2158. {
  2159. int ret;
  2160. dev_t devt;
  2161. struct class *dev_class;
  2162. struct device *dev_sys;
  2163. /* allocate major device number (/proc/devices -> major_num ubwcp) */
  2164. ret = alloc_chrdev_region(&devt, 0, UBWCP_NUM_DEVICES, UBWCP_DEVICE_NAME);
  2165. if (ret) {
  2166. ERR("alloc_chrdev_region() failed: %d", ret);
  2167. return ret;
  2168. }
  2169. /* create device class (/sys/class/ubwcp_class) */
  2170. dev_class = class_create(THIS_MODULE, "ubwcp_class");
  2171. if (IS_ERR(dev_class)) {
  2172. ERR("class_create() failed");
  2173. return -1;
  2174. }
  2175. /* Create device and register with sysfs
  2176. * (/sys/class/ubwcp_class/ubwcp/... -> dev/power/subsystem/uevent)
  2177. */
  2178. dev_sys = device_create(dev_class, NULL, devt, NULL,
  2179. UBWCP_DEVICE_NAME);
  2180. if (IS_ERR(dev_sys)) {
  2181. ERR("device_create() failed");
  2182. return -1;
  2183. }
  2184. /* register file operations and get cdev */
  2185. cdev_init(&ubwcp->cdev, &ubwcp_fops);
  2186. /* associate cdev and device major/minor with file system
  2187. * can do file ops on /dev/ubwcp after this
  2188. */
  2189. ret = cdev_add(&ubwcp->cdev, devt, 1);
  2190. if (ret) {
  2191. ERR("cdev_add() failed");
  2192. return -1;
  2193. }
  2194. ubwcp->devt = devt;
  2195. ubwcp->dev_class = dev_class;
  2196. ubwcp->dev_sys = dev_sys;
  2197. return 0;
  2198. }
  2199. static void ubwcp_cdev_deinit(struct ubwcp_driver *ubwcp)
  2200. {
  2201. device_destroy(ubwcp->dev_class, ubwcp->devt);
  2202. class_destroy(ubwcp->dev_class);
  2203. cdev_del(&ubwcp->cdev);
  2204. unregister_chrdev_region(ubwcp->devt, UBWCP_NUM_DEVICES);
  2205. }
  2206. struct handler_node {
  2207. struct list_head list;
  2208. u32 client_id;
  2209. ubwcp_error_handler_t handler;
  2210. void *data;
  2211. };
  2212. int ubwcp_register_error_handler(u32 client_id, ubwcp_error_handler_t handler,
  2213. void *data)
  2214. {
  2215. struct handler_node *node;
  2216. unsigned long flags;
  2217. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2218. if (!ubwcp)
  2219. return -EINVAL;
  2220. if (client_id != -1)
  2221. return -EINVAL;
  2222. if (!handler)
  2223. return -EINVAL;
  2224. node = kzalloc(sizeof(*node), GFP_KERNEL);
  2225. if (!node)
  2226. return -ENOMEM;
  2227. node->client_id = client_id;
  2228. node->handler = handler;
  2229. node->data = data;
  2230. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  2231. list_add_tail(&node->list, &ubwcp->err_handler_list);
  2232. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  2233. return 0;
  2234. }
  2235. EXPORT_SYMBOL(ubwcp_register_error_handler);
  2236. static void ubwcp_notify_error_handlers(struct ubwcp_err_info *err)
  2237. {
  2238. struct handler_node *node;
  2239. unsigned long flags;
  2240. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2241. if (!ubwcp)
  2242. return;
  2243. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  2244. list_for_each_entry(node, &ubwcp->err_handler_list, list)
  2245. node->handler(err, node->data);
  2246. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  2247. }
  2248. int ubwcp_unregister_error_handler(u32 client_id)
  2249. {
  2250. int ret = -EINVAL;
  2251. struct handler_node *node;
  2252. unsigned long flags;
  2253. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2254. if (!ubwcp)
  2255. return -EINVAL;
  2256. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  2257. list_for_each_entry(node, &ubwcp->err_handler_list, list)
  2258. if (node->client_id == client_id) {
  2259. list_del(&node->list);
  2260. kfree(node);
  2261. ret = 0;
  2262. break;
  2263. }
  2264. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  2265. return ret;
  2266. }
  2267. EXPORT_SYMBOL(ubwcp_unregister_error_handler);
  2268. /* get ubwcp_buf corresponding to the ULA PA*/
  2269. static struct dma_buf *get_dma_buf_from_ulapa(phys_addr_t addr)
  2270. {
  2271. struct ubwcp_buf *buf = NULL;
  2272. struct dma_buf *ret_buf = NULL;
  2273. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2274. unsigned long flags;
  2275. u32 i;
  2276. if (!ubwcp)
  2277. return NULL;
  2278. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  2279. hash_for_each(ubwcp->buf_table, i, buf, hnode) {
  2280. if (buf->ula_pa <= addr && addr < buf->ula_pa + buf->ula_size) {
  2281. ret_buf = buf->dma_buf;
  2282. break;
  2283. }
  2284. }
  2285. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  2286. return ret_buf;
  2287. }
  2288. /* get ubwcp_buf corresponding to the IOVA*/
  2289. static struct dma_buf *get_dma_buf_from_iova(unsigned long addr)
  2290. {
  2291. struct ubwcp_buf *buf = NULL;
  2292. struct dma_buf *ret_buf = NULL;
  2293. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2294. unsigned long flags;
  2295. u32 i;
  2296. if (!ubwcp)
  2297. return NULL;
  2298. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  2299. hash_for_each(ubwcp->buf_table, i, buf, hnode) {
  2300. unsigned long iova_base;
  2301. unsigned int iova_size;
  2302. if (!buf->sgt)
  2303. continue;
  2304. iova_base = sg_dma_address(buf->sgt->sgl);
  2305. iova_size = sg_dma_len(buf->sgt->sgl);
  2306. if (iova_base <= addr && addr < iova_base + iova_size) {
  2307. ret_buf = buf->dma_buf;
  2308. break;
  2309. }
  2310. }
  2311. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  2312. return ret_buf;
  2313. }
  2314. int ubwcp_iommu_fault_handler(struct iommu_domain *domain, struct device *dev,
  2315. unsigned long iova, int flags, void *data)
  2316. {
  2317. int ret = 0;
  2318. struct ubwcp_err_info err;
  2319. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  2320. struct device *cb_dev = (struct device *)data;
  2321. if (!ubwcp) {
  2322. ret = -EINVAL;
  2323. goto err;
  2324. }
  2325. err.err_code = UBWCP_SMMU_FAULT;
  2326. if (cb_dev == ubwcp->dev_desc_cb)
  2327. err.smmu_err.iommu_dev_id = UBWCP_DESC_CB_ID;
  2328. else if (cb_dev == ubwcp->dev_buf_cb)
  2329. err.smmu_err.iommu_dev_id = UBWCP_BUF_CB_ID;
  2330. else
  2331. err.smmu_err.iommu_dev_id = UBWCP_UNKNOWN_CB_ID;
  2332. err.smmu_err.dmabuf = get_dma_buf_from_iova(iova);
  2333. err.smmu_err.iova = iova;
  2334. err.smmu_err.iommu_fault_flags = flags;
  2335. ERR_RATE_LIMIT("ubwcp_err: err code: %d (smmu), iommu_dev_id: %d, iova: 0x%llx, flags: 0x%x",
  2336. err.err_code, err.smmu_err.iommu_dev_id, err.smmu_err.iova,
  2337. err.smmu_err.iommu_fault_flags);
  2338. ubwcp_notify_error_handlers(&err);
  2339. err:
  2340. return ret;
  2341. }
  2342. irqreturn_t ubwcp_irq_handler(int irq, void *ptr)
  2343. {
  2344. struct ubwcp_driver *ubwcp;
  2345. void __iomem *base;
  2346. phys_addr_t addr;
  2347. struct ubwcp_err_info err;
  2348. ubwcp = (struct ubwcp_driver *) ptr;
  2349. base = ubwcp->base;
  2350. if (irq == ubwcp->irq_range_ck_rd) {
  2351. addr = ubwcp_hw_interrupt_src_address(base, 0) << 6;
  2352. err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
  2353. err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2354. err.translation_err.ula_pa = addr;
  2355. err.translation_err.read = true;
  2356. ERR_RATE_LIMIT("ubwcp_err: err code: %d (range), dmabuf: 0x%llx, read: %d, addr: 0x%llx",
  2357. err.err_code, err.translation_err.dmabuf, err.translation_err.read, addr);
  2358. ubwcp_notify_error_handlers(&err);
  2359. ubwcp_hw_interrupt_clear(ubwcp->base, 0);
  2360. } else if (irq == ubwcp->irq_range_ck_wr) {
  2361. addr = ubwcp_hw_interrupt_src_address(base, 1) << 6;
  2362. err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
  2363. err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2364. err.translation_err.ula_pa = addr;
  2365. err.translation_err.read = false;
  2366. ERR_RATE_LIMIT("ubwcp_err: err code: %d (range), dmabuf: 0x%llx, read: %d, addr: 0x%llx",
  2367. err.err_code, err.translation_err.dmabuf, err.translation_err.read, addr);
  2368. ubwcp_notify_error_handlers(&err);
  2369. ubwcp_hw_interrupt_clear(ubwcp->base, 1);
  2370. } else if (irq == ubwcp->irq_encode) {
  2371. addr = ubwcp_hw_interrupt_src_address(base, 3) << 6;
  2372. err.err_code = UBWCP_ENCODE_ERROR;
  2373. err.enc_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2374. err.enc_err.ula_pa = addr;
  2375. ERR_RATE_LIMIT("ubwcp_err: err code: %d (encode), dmabuf: 0x%llx, addr: 0x%llx",
  2376. err.err_code, err.enc_err.dmabuf, addr);
  2377. ubwcp_notify_error_handlers(&err);
  2378. ubwcp_hw_interrupt_clear(ubwcp->base, 3);
  2379. } else if (irq == ubwcp->irq_decode) {
  2380. addr = ubwcp_hw_interrupt_src_address(base, 2) << 6;
  2381. err.err_code = UBWCP_DECODE_ERROR;
  2382. err.dec_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2383. err.dec_err.ula_pa = addr;
  2384. ERR_RATE_LIMIT("ubwcp_err: err code: %d (decode), dmabuf: 0x%llx, addr: 0x%llx",
  2385. err.err_code, err.enc_err.dmabuf, addr);
  2386. ubwcp_notify_error_handlers(&err);
  2387. ubwcp_hw_interrupt_clear(ubwcp->base, 2);
  2388. } else {
  2389. ERR("unknown irq: %d", irq);
  2390. return IRQ_NONE;
  2391. }
  2392. return IRQ_HANDLED;
  2393. }
  2394. static int ubwcp_interrupt_register(struct platform_device *pdev, struct ubwcp_driver *ubwcp)
  2395. {
  2396. int ret = 0;
  2397. struct device *dev = &pdev->dev;
  2398. FENTRY();
  2399. ubwcp->irq_range_ck_rd = platform_get_irq(pdev, 0);
  2400. if (ubwcp->irq_range_ck_rd < 0)
  2401. return ubwcp->irq_range_ck_rd;
  2402. ubwcp->irq_range_ck_wr = platform_get_irq(pdev, 1);
  2403. if (ubwcp->irq_range_ck_wr < 0)
  2404. return ubwcp->irq_range_ck_wr;
  2405. ubwcp->irq_encode = platform_get_irq(pdev, 2);
  2406. if (ubwcp->irq_encode < 0)
  2407. return ubwcp->irq_encode;
  2408. ubwcp->irq_decode = platform_get_irq(pdev, 3);
  2409. if (ubwcp->irq_decode < 0)
  2410. return ubwcp->irq_decode;
  2411. DBG("got irqs: %d %d %d %d", ubwcp->irq_range_ck_rd,
  2412. ubwcp->irq_range_ck_wr,
  2413. ubwcp->irq_encode,
  2414. ubwcp->irq_decode);
  2415. ret = devm_request_irq(dev, ubwcp->irq_range_ck_rd, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2416. if (ret) {
  2417. ERR("request_irq() failed. irq: %d ret: %d",
  2418. ubwcp->irq_range_ck_rd, ret);
  2419. return ret;
  2420. }
  2421. ret = devm_request_irq(dev, ubwcp->irq_range_ck_wr, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2422. if (ret) {
  2423. ERR("request_irq() failed. irq: %d ret: %d",
  2424. ubwcp->irq_range_ck_wr, ret);
  2425. return ret;
  2426. }
  2427. ret = devm_request_irq(dev, ubwcp->irq_encode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2428. if (ret) {
  2429. ERR("request_irq() failed. irq: %d ret: %d",
  2430. ubwcp->irq_encode, ret);
  2431. return ret;
  2432. }
  2433. ret = devm_request_irq(dev, ubwcp->irq_decode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2434. if (ret) {
  2435. ERR("request_irq() failed. irq: %d ret: %d",
  2436. ubwcp->irq_decode, ret);
  2437. return ret;
  2438. }
  2439. return ret;
  2440. }
  2441. /* ubwcp device probe */
  2442. static int qcom_ubwcp_probe(struct platform_device *pdev)
  2443. {
  2444. int ret = 0;
  2445. struct ubwcp_driver *ubwcp;
  2446. struct device *ubwcp_dev = &pdev->dev;
  2447. FENTRY();
  2448. ubwcp = devm_kzalloc(ubwcp_dev, sizeof(*ubwcp), GFP_KERNEL);
  2449. if (!ubwcp) {
  2450. ERR("devm_kzalloc() failed");
  2451. return -ENOMEM;
  2452. }
  2453. ubwcp->dev = &pdev->dev;
  2454. ret = dma_set_mask_and_coherent(ubwcp->dev, DMA_BIT_MASK(64));
  2455. #ifdef UBWCP_USE_SMC
  2456. {
  2457. struct resource res;
  2458. of_address_to_resource(ubwcp_dev->of_node, 0, &res);
  2459. ubwcp->base = (void __iomem *) res.start;
  2460. DBG("Using SMC calls. base: %p", ubwcp->base);
  2461. }
  2462. #else
  2463. ubwcp->base = devm_platform_ioremap_resource(pdev, 0);
  2464. if (IS_ERR(ubwcp->base)) {
  2465. ERR("devm ioremap() failed: %d", PTR_ERR(ubwcp->base));
  2466. return PTR_ERR(ubwcp->base);
  2467. }
  2468. DBG("ubwcp->base: %p", ubwcp->base);
  2469. #endif
  2470. ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 0, &ubwcp->ula_pool_base);
  2471. if (ret) {
  2472. ERR("failed reading ula_range (base): %d", ret);
  2473. return ret;
  2474. }
  2475. DBG("ubwcp: ula_range: base = 0x%lx", ubwcp->ula_pool_base);
  2476. ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 1, &ubwcp->ula_pool_size);
  2477. if (ret) {
  2478. ERR("failed reading ula_range (size): %d", ret);
  2479. return ret;
  2480. }
  2481. DBG("ubwcp: ula_range: size = 0x%lx", ubwcp->ula_pool_size);
  2482. INIT_LIST_HEAD(&ubwcp->err_handler_list);
  2483. atomic_set(&ubwcp->num_non_lin_buffers, 0);
  2484. ubwcp->mem_online = false;
  2485. mutex_init(&ubwcp->desc_lock);
  2486. spin_lock_init(&ubwcp->buf_table_lock);
  2487. mutex_init(&ubwcp->mem_hotplug_lock);
  2488. mutex_init(&ubwcp->ula_lock);
  2489. mutex_init(&ubwcp->ubwcp_flush_lock);
  2490. mutex_init(&ubwcp->hw_range_ck_lock);
  2491. spin_lock_init(&ubwcp->err_handler_list_lock);
  2492. /* Regulator */
  2493. ubwcp->vdd = devm_regulator_get(ubwcp_dev, "vdd");
  2494. if (IS_ERR_OR_NULL(ubwcp->vdd)) {
  2495. ret = PTR_ERR(ubwcp->vdd);
  2496. ERR("devm_regulator_get() failed: %d", ret);
  2497. return -1;
  2498. }
  2499. ret = ubwcp_init_clocks(ubwcp, ubwcp_dev);
  2500. if (ret) {
  2501. ERR("failed to initialize ubwcp clocks err: %d", ret);
  2502. return ret;
  2503. }
  2504. if (ubwcp_power(ubwcp, true))
  2505. return -1;
  2506. if (ubwcp_cdev_init(ubwcp))
  2507. return -1;
  2508. /* disable all interrupts (reset value has some interrupts enabled by default) */
  2509. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2510. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2511. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2512. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2513. if (ubwcp_interrupt_register(pdev, ubwcp))
  2514. return -1;
  2515. ubwcp_debugfs_init(ubwcp);
  2516. /* create ULA pool */
  2517. ubwcp->ula_pool = gen_pool_create(12, -1);
  2518. if (!ubwcp->ula_pool) {
  2519. ERR("failed gen_pool_create()");
  2520. ret = -1;
  2521. goto err_pool_create;
  2522. }
  2523. ret = gen_pool_add(ubwcp->ula_pool, ubwcp->ula_pool_base, ubwcp->ula_pool_size, -1);
  2524. if (ret) {
  2525. ERR("failed gen_pool_add(): %d", ret);
  2526. ret = -1;
  2527. goto err_pool_add;
  2528. }
  2529. /* register the default config mmap function. */
  2530. ubwcp->mmap_config_fptr = msm_ubwcp_dma_buf_configure_mmap;
  2531. hash_init(ubwcp->buf_table);
  2532. ubwcp_buf_desc_list_init(ubwcp);
  2533. image_format_init(ubwcp);
  2534. /* one time hw init */
  2535. ubwcp_hw_one_time_init(ubwcp->base);
  2536. ubwcp_hw_version(ubwcp->base, &ubwcp->hw_ver_major, &ubwcp->hw_ver_minor);
  2537. pr_err("ubwcp: hw version: major %d, minor %d\n", ubwcp->hw_ver_major, ubwcp->hw_ver_minor);
  2538. if (ubwcp->hw_ver_major == 0) {
  2539. ERR("Failed to read HW version");
  2540. ret = -1;
  2541. goto err_pool_add;
  2542. }
  2543. /* set pdev->dev->driver_data = ubwcp */
  2544. platform_set_drvdata(pdev, ubwcp);
  2545. /* enable interrupts */
  2546. if (ubwcp->read_err_irq_en)
  2547. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, true);
  2548. if (ubwcp->write_err_irq_en)
  2549. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, true);
  2550. if (ubwcp->decode_err_irq_en)
  2551. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, true);
  2552. if (ubwcp->encode_err_irq_en)
  2553. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, true);
  2554. /* Turn OFF until buffers are allocated */
  2555. if (ubwcp_power(ubwcp, false)) {
  2556. ret = -1;
  2557. goto err_power_off;
  2558. }
  2559. ret = msm_ubwcp_set_ops(ubwcp_init_buffer, ubwcp_free_buffer, ubwcp_lock, ubwcp_unlock);
  2560. if (ret) {
  2561. ERR("msm_ubwcp_set_ops() failed: %d, but IGNORED", ret);
  2562. /* TBD: ignore return error during testing phase.
  2563. * This allows us to rmmod/insmod for faster dev cycle.
  2564. * In final version: return error and de-register driver if set_ops fails.
  2565. */
  2566. ret = 0;
  2567. //goto err_power_off;
  2568. } else {
  2569. DBG("msm_ubwcp_set_ops(): success"); }
  2570. me = ubwcp;
  2571. return ret;
  2572. err_power_off:
  2573. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2574. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2575. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2576. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2577. err_pool_add:
  2578. gen_pool_destroy(ubwcp->ula_pool);
  2579. err_pool_create:
  2580. ubwcp_cdev_deinit(ubwcp);
  2581. return ret;
  2582. }
  2583. /* buffer context bank device probe */
  2584. static int ubwcp_probe_cb_buf(struct platform_device *pdev)
  2585. {
  2586. struct ubwcp_driver *ubwcp;
  2587. struct iommu_domain *domain = NULL;
  2588. FENTRY();
  2589. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2590. if (!ubwcp) {
  2591. ERR("failed to get ubwcp ptr");
  2592. return -EINVAL;
  2593. }
  2594. /* save the buffer cb device */
  2595. ubwcp->dev_buf_cb = &pdev->dev;
  2596. domain = iommu_get_domain_for_dev(ubwcp->dev_buf_cb);
  2597. if (domain)
  2598. iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_buf_cb);
  2599. return 0;
  2600. }
  2601. /* descriptor context bank device probe */
  2602. static int ubwcp_probe_cb_desc(struct platform_device *pdev)
  2603. {
  2604. int ret = 0;
  2605. struct ubwcp_driver *ubwcp;
  2606. struct iommu_domain *domain = NULL;
  2607. FENTRY();
  2608. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2609. if (!ubwcp) {
  2610. ERR("failed to get ubwcp ptr");
  2611. return -EINVAL;
  2612. }
  2613. ubwcp->buffer_desc_size = UBWCP_BUFFER_DESC_OFFSET *
  2614. UBWCP_BUFFER_DESC_COUNT;
  2615. ubwcp->dev_desc_cb = &pdev->dev;
  2616. dma_set_max_seg_size(ubwcp->dev_desc_cb, DMA_BIT_MASK(32));
  2617. dma_set_seg_boundary(ubwcp->dev_desc_cb, (unsigned long)DMA_BIT_MASK(64));
  2618. /* Allocate buffer descriptors. UBWCP is iocoherent device.
  2619. * Thus we don't need to flush after updates to buffer descriptors.
  2620. */
  2621. ubwcp->buffer_desc_base = dma_alloc_coherent(ubwcp->dev_desc_cb,
  2622. ubwcp->buffer_desc_size,
  2623. &ubwcp->buffer_desc_dma_handle,
  2624. GFP_KERNEL);
  2625. if (!ubwcp->buffer_desc_base) {
  2626. ERR("failed to allocate desc buffer");
  2627. return -ENOMEM;
  2628. }
  2629. DBG("desc_base = %p size = %zu", ubwcp->buffer_desc_base,
  2630. ubwcp->buffer_desc_size);
  2631. ret = ubwcp_power(ubwcp, true);
  2632. if (ret) {
  2633. ERR("failed to power on");
  2634. goto err;
  2635. }
  2636. ubwcp_hw_set_buf_desc(ubwcp->base, (u64) ubwcp->buffer_desc_dma_handle,
  2637. UBWCP_BUFFER_DESC_OFFSET);
  2638. ret = ubwcp_power(ubwcp, false);
  2639. if (ret) {
  2640. ERR("failed to power off");
  2641. goto err;
  2642. }
  2643. domain = iommu_get_domain_for_dev(ubwcp->dev_desc_cb);
  2644. if (domain)
  2645. iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_desc_cb);
  2646. return ret;
  2647. err:
  2648. dma_free_coherent(ubwcp->dev_desc_cb,
  2649. ubwcp->buffer_desc_size,
  2650. ubwcp->buffer_desc_base,
  2651. ubwcp->buffer_desc_dma_handle);
  2652. ubwcp->buffer_desc_base = NULL;
  2653. ubwcp->buffer_desc_dma_handle = 0;
  2654. ubwcp->dev_desc_cb = NULL;
  2655. return -1;
  2656. }
  2657. /* buffer context bank device remove */
  2658. static int ubwcp_remove_cb_buf(struct platform_device *pdev)
  2659. {
  2660. struct ubwcp_driver *ubwcp;
  2661. FENTRY();
  2662. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2663. if (!ubwcp) {
  2664. ERR("failed to get ubwcp ptr");
  2665. return -EINVAL;
  2666. }
  2667. /* remove buf_cb reference */
  2668. ubwcp->dev_buf_cb = NULL;
  2669. return 0;
  2670. }
  2671. /* descriptor context bank device remove */
  2672. static int ubwcp_remove_cb_desc(struct platform_device *pdev)
  2673. {
  2674. struct ubwcp_driver *ubwcp;
  2675. FENTRY();
  2676. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2677. if (!ubwcp) {
  2678. ERR("failed to get ubwcp ptr");
  2679. return -EINVAL;
  2680. }
  2681. if (!ubwcp->dev_desc_cb) {
  2682. ERR("ubwcp->dev_desc_cb == NULL");
  2683. return -1;
  2684. }
  2685. ubwcp_power(ubwcp, true);
  2686. ubwcp_hw_set_buf_desc(ubwcp->base, 0x0, 0x0);
  2687. ubwcp_power(ubwcp, false);
  2688. dma_free_coherent(ubwcp->dev_desc_cb,
  2689. ubwcp->buffer_desc_size,
  2690. ubwcp->buffer_desc_base,
  2691. ubwcp->buffer_desc_dma_handle);
  2692. ubwcp->buffer_desc_base = NULL;
  2693. ubwcp->buffer_desc_dma_handle = 0;
  2694. return 0;
  2695. }
  2696. /* ubwcp device remove */
  2697. static int qcom_ubwcp_remove(struct platform_device *pdev)
  2698. {
  2699. size_t avail;
  2700. size_t psize;
  2701. struct ubwcp_driver *ubwcp;
  2702. FENTRY();
  2703. /* get pdev->dev->driver_data = ubwcp */
  2704. ubwcp = platform_get_drvdata(pdev);
  2705. if (!ubwcp) {
  2706. ERR("ubwcp == NULL");
  2707. return -1;
  2708. }
  2709. ubwcp_power(ubwcp, true);
  2710. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2711. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2712. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2713. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2714. ubwcp_power(ubwcp, false);
  2715. /* before destroying, make sure pool is empty. otherwise pool_destroy() panics.
  2716. * TBD: remove this check for production code and let it panic
  2717. */
  2718. avail = gen_pool_avail(ubwcp->ula_pool);
  2719. psize = gen_pool_size(ubwcp->ula_pool);
  2720. if (psize != avail) {
  2721. ERR("gen_pool is not empty! avail: %zx size: %zx", avail, psize);
  2722. ERR("skipping pool destroy....cause it will PANIC. Fix this!!!!");
  2723. WARN(1, "Fix this!");
  2724. } else {
  2725. gen_pool_destroy(ubwcp->ula_pool);
  2726. }
  2727. ubwcp_debugfs_deinit(ubwcp);
  2728. ubwcp_cdev_deinit(ubwcp);
  2729. return 0;
  2730. }
  2731. /* top level ubwcp device probe function */
  2732. static int ubwcp_probe(struct platform_device *pdev)
  2733. {
  2734. const char *compatible = "";
  2735. FENTRY();
  2736. trace_ubwcp_probe(pdev);
  2737. if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
  2738. return qcom_ubwcp_probe(pdev);
  2739. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
  2740. return ubwcp_probe_cb_desc(pdev);
  2741. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
  2742. return ubwcp_probe_cb_buf(pdev);
  2743. of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
  2744. ERR("unknown device: %s", compatible);
  2745. WARN_ON(1);
  2746. return -EINVAL;
  2747. }
  2748. /* top level ubwcp device remove function */
  2749. static int ubwcp_remove(struct platform_device *pdev)
  2750. {
  2751. const char *compatible = "";
  2752. FENTRY();
  2753. trace_ubwcp_remove(pdev);
  2754. /* TBD: what if buffers are still allocated? locked? etc.
  2755. * also should turn off power?
  2756. */
  2757. if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
  2758. return qcom_ubwcp_remove(pdev);
  2759. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
  2760. return ubwcp_remove_cb_desc(pdev);
  2761. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
  2762. return ubwcp_remove_cb_buf(pdev);
  2763. of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
  2764. ERR("unknown device: %s", compatible);
  2765. WARN_ON(1);
  2766. return -EINVAL;
  2767. }
  2768. static const struct of_device_id ubwcp_dt_match[] = {
  2769. {.compatible = "qcom,ubwcp"},
  2770. {.compatible = "qcom,ubwcp-context-bank-desc"},
  2771. {.compatible = "qcom,ubwcp-context-bank-buf"},
  2772. {}
  2773. };
  2774. struct platform_driver ubwcp_platform_driver = {
  2775. .probe = ubwcp_probe,
  2776. .remove = ubwcp_remove,
  2777. .driver = {
  2778. .name = "qcom,ubwcp",
  2779. .of_match_table = ubwcp_dt_match,
  2780. },
  2781. };
  2782. int ubwcp_init(void)
  2783. {
  2784. int ret = 0;
  2785. DBG("+++++++++++");
  2786. ret = platform_driver_register(&ubwcp_platform_driver);
  2787. if (ret)
  2788. ERR("platform_driver_register() failed: %d", ret);
  2789. return ret;
  2790. }
  2791. void ubwcp_exit(void)
  2792. {
  2793. platform_driver_unregister(&ubwcp_platform_driver);
  2794. DBG("-----------");
  2795. }
  2796. module_init(ubwcp_init);
  2797. module_exit(ubwcp_exit);
  2798. MODULE_LICENSE("GPL");