ubwcp_main.c 73 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/kernel.h>
  7. #include <linux/dma-buf.h>
  8. #include <linux/slab.h>
  9. #include <linux/cdev.h>
  10. #include <linux/hashtable.h>
  11. #include <linux/scatterlist.h>
  12. #include <linux/types.h>
  13. #include <linux/platform_device.h>
  14. #include <linux/of.h>
  15. #include <linux/of_platform.h>
  16. #include <linux/of_address.h>
  17. #include <linux/genalloc.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/regulator/consumer.h>
  20. #include <linux/numa.h>
  21. #include <linux/memory_hotplug.h>
  22. #include <asm/page.h>
  23. #include <linux/delay.h>
  24. #include <linux/ubwcp_dma_heap.h>
  25. #include <linux/debugfs.h>
  26. #include <linux/clk.h>
  27. #include <linux/iommu.h>
  28. #include <linux/set_memory.h>
  29. MODULE_IMPORT_NS(DMA_BUF);
  30. #include "include/kernel/ubwcp.h"
  31. #include "ubwcp_hw.h"
  32. #include "include/uapi/ubwcp_ioctl.h"
  33. #define UBWCP_NUM_DEVICES 1
  34. #define UBWCP_DEVICE_NAME "ubwcp"
  35. #define UBWCP_BUFFER_DESC_OFFSET 64
  36. #define UBWCP_BUFFER_DESC_COUNT 256
  37. #define CACHE_ADDR(x) ((x) >> 6)
  38. #define PAGE_ADDR(x) ((x) >> 12)
  39. #define UBWCP_ALIGN(_x, _y) ((((_x) + (_y) - 1)/(_y))*(_y))
  40. //#define DBG(fmt, args...)
  41. //#define DBG_BUF_ATTR(fmt, args...)
  42. #define DBG_BUF_ATTR(fmt, args...) do { if (ubwcp_debug_trace_enable) \
  43. pr_err("ubwcp: %s(): " fmt "\n", __func__, ##args); \
  44. } while (0)
  45. #define DBG(fmt, args...) do { if (ubwcp_debug_trace_enable) \
  46. pr_err("ubwcp: %s(): " fmt "\n", __func__, ##args); \
  47. } while (0)
  48. #define ERR(fmt, args...) pr_err("ubwcp: %s(): ~~~ERROR~~~: " fmt "\n", __func__, ##args)
  49. #define FENTRY() DBG("")
  50. #define META_DATA_PITCH_ALIGN 64
  51. #define META_DATA_HEIGHT_ALIGN 16
  52. #define META_DATA_SIZE_ALIGN 4096
  53. #define PIXEL_DATA_SIZE_ALIGN 4096
  54. struct ubwcp_desc {
  55. int idx;
  56. void *ptr;
  57. };
  58. /* TBD: confirm size of width/height */
  59. struct ubwcp_dimension {
  60. u16 width;
  61. u16 height;
  62. };
  63. struct ubwcp_plane_info {
  64. u16 pixel_bytes;
  65. u16 per_pixel;
  66. struct ubwcp_dimension tilesize_p; /* pixels */
  67. struct ubwcp_dimension macrotilesize_p; /* pixels */
  68. };
  69. struct ubwcp_image_format_info {
  70. u16 planes;
  71. struct ubwcp_plane_info p_info[2];
  72. };
  73. enum ubwcp_std_image_format {
  74. RGBA = 0,
  75. NV12 = 1,
  76. NV124R = 2,
  77. P010 = 3,
  78. TP10 = 4,
  79. P016 = 5,
  80. INFO_FORMAT_LIST_SIZE,
  81. STD_IMAGE_FORMAT_INVALID = 0xFF
  82. };
  83. struct ubwcp_driver {
  84. /* cdev related */
  85. dev_t devt;
  86. struct class *dev_class; //sysfs dev class
  87. struct device *dev_sys; //sysfs dev
  88. struct cdev cdev; //char dev
  89. /* debugfs */
  90. struct dentry *debugfs_root;
  91. /* ubwcp devices */
  92. struct device *dev; //ubwcp device
  93. struct device *dev_desc_cb; //smmu dev for descriptors
  94. struct device *dev_buf_cb; //smmu dev for ubwcp buffers
  95. void __iomem *base; //ubwcp base address
  96. struct regulator *vdd;
  97. struct clk **clocks;
  98. int num_clocks;
  99. /* interrupts */
  100. int irq_range_ck_rd;
  101. int irq_range_ck_wr;
  102. int irq_encode;
  103. int irq_decode;
  104. /* ula address pool */
  105. u64 ula_pool_base;
  106. u64 ula_pool_size;
  107. struct gen_pool *ula_pool;
  108. configure_mmap mmap_config_fptr;
  109. /* HW version */
  110. u32 hw_ver_major;
  111. u32 hw_ver_minor;
  112. /* keep track of all buffers. hash table index'ed using dma_buf ptr.
  113. * 2**8 = 256 hash values
  114. */
  115. DECLARE_HASHTABLE(buf_table, 8);
  116. /* buffer descriptor */
  117. void *buffer_desc_base; /* CPU address */
  118. dma_addr_t buffer_desc_dma_handle; /* dma address */
  119. size_t buffer_desc_size;
  120. struct ubwcp_desc desc_list[UBWCP_BUFFER_DESC_COUNT];
  121. struct ubwcp_image_format_info format_info[INFO_FORMAT_LIST_SIZE];
  122. struct mutex desc_lock; /* allocate/free descriptors */
  123. spinlock_t buf_table_lock; /* add/remove dma_buf into list of managed bufffers */
  124. struct mutex mem_hotplug_lock; /* memory hotplug lock */
  125. struct mutex ula_lock; /* allocate/free ula */
  126. struct mutex ubwcp_flush_lock; /* ubwcp flush */
  127. struct mutex hw_range_ck_lock; /* range ck */
  128. struct list_head err_handler_list; /* error handler list */
  129. spinlock_t err_handler_list_lock; /* err_handler_list lock */
  130. };
  131. struct ubwcp_buf {
  132. struct hlist_node hnode;
  133. struct ubwcp_driver *ubwcp;
  134. struct ubwcp_buffer_attrs buf_attr;
  135. bool perm;
  136. struct ubwcp_desc *desc;
  137. bool buf_attr_set;
  138. bool locked;
  139. enum dma_data_direction lock_dir;
  140. int lock_count;
  141. /* dma_buf info */
  142. struct dma_buf *dma_buf;
  143. struct dma_buf_attachment *attachment;
  144. struct sg_table *sgt;
  145. /* ula info */
  146. phys_addr_t ula_pa;
  147. size_t ula_size;
  148. /* meta metadata */
  149. struct ubwcp_hw_meta_metadata mmdata;
  150. struct mutex lock;
  151. };
  152. static struct ubwcp_driver *me;
  153. static int error_print_count;
  154. u32 ubwcp_debug_trace_enable;
  155. static struct ubwcp_driver *ubwcp_get_driver(void)
  156. {
  157. if (!me)
  158. WARN(1, "ubwcp: driver ptr requested but driver not initialized");
  159. return me;
  160. }
  161. static void image_format_init(struct ubwcp_driver *ubwcp)
  162. { /* planes, bytes/p, Tp , MTp */
  163. ubwcp->format_info[RGBA] = (struct ubwcp_image_format_info)
  164. {1, {{4, 1, {16, 4}, {64, 16}}}};
  165. ubwcp->format_info[NV12] = (struct ubwcp_image_format_info)
  166. {2, {{1, 1, {32, 8}, {128, 32}},
  167. {2, 1, {16, 8}, { 64, 32}}}};
  168. ubwcp->format_info[NV124R] = (struct ubwcp_image_format_info)
  169. {2, {{1, 1, {64, 4}, {256, 16}},
  170. {2, 1, {32, 4}, {128, 16}}}};
  171. ubwcp->format_info[P010] = (struct ubwcp_image_format_info)
  172. {2, {{2, 1, {32, 4}, {128, 16}},
  173. {4, 1, {16, 4}, { 64, 16}}}};
  174. ubwcp->format_info[TP10] = (struct ubwcp_image_format_info)
  175. {2, {{4, 3, {48, 4}, {192, 16}},
  176. {8, 3, {24, 4}, { 96, 16}}}};
  177. ubwcp->format_info[P016] = (struct ubwcp_image_format_info)
  178. {2, {{2, 1, {32, 4}, {128, 16}},
  179. {4, 1, {16, 4}, { 64, 16}}}};
  180. }
  181. static void ubwcp_buf_desc_list_init(struct ubwcp_driver *ubwcp)
  182. {
  183. int idx;
  184. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  185. for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
  186. desc_list[idx].idx = -1;
  187. desc_list[idx].ptr = NULL;
  188. }
  189. }
  190. static int ubwcp_init_clocks(struct ubwcp_driver *ubwcp, struct device *dev)
  191. {
  192. const char *cname;
  193. struct property *prop;
  194. int i;
  195. ubwcp->num_clocks =
  196. of_property_count_strings(dev->of_node, "clock-names");
  197. if (ubwcp->num_clocks < 1) {
  198. ubwcp->num_clocks = 0;
  199. return 0;
  200. }
  201. ubwcp->clocks = devm_kzalloc(dev,
  202. sizeof(*ubwcp->clocks) * ubwcp->num_clocks, GFP_KERNEL);
  203. if (!ubwcp->clocks)
  204. return -ENOMEM;
  205. i = 0;
  206. of_property_for_each_string(dev->of_node, "clock-names",
  207. prop, cname) {
  208. struct clk *c = devm_clk_get(dev, cname);
  209. if (IS_ERR(c)) {
  210. ERR("Couldn't get clock: %s\n", cname);
  211. return PTR_ERR(c);
  212. }
  213. ubwcp->clocks[i] = c;
  214. ++i;
  215. }
  216. return 0;
  217. }
  218. static int ubwcp_enable_clocks(struct ubwcp_driver *ubwcp)
  219. {
  220. int i, ret = 0;
  221. for (i = 0; i < ubwcp->num_clocks; ++i) {
  222. ret = clk_prepare_enable(ubwcp->clocks[i]);
  223. if (ret) {
  224. ERR("Couldn't enable clock #%d\n", i);
  225. while (i--)
  226. clk_disable_unprepare(ubwcp->clocks[i]);
  227. break;
  228. }
  229. }
  230. return ret;
  231. }
  232. static void ubwcp_disable_clocks(struct ubwcp_driver *ubwcp)
  233. {
  234. int i;
  235. for (i = ubwcp->num_clocks; i; --i)
  236. clk_disable_unprepare(ubwcp->clocks[i - 1]);
  237. }
  238. /* UBWCP Power control */
  239. static int ubwcp_power(struct ubwcp_driver *ubwcp, bool enable)
  240. {
  241. int ret = 0;
  242. if (!ubwcp) {
  243. ERR("ubwcp ptr is NULL");
  244. return -1;
  245. }
  246. if (!ubwcp->vdd) {
  247. ERR("vdd is NULL");
  248. return -1;
  249. }
  250. if (enable) {
  251. ret = regulator_enable(ubwcp->vdd);
  252. if (ret < 0) {
  253. ERR("regulator_enable failed: %d", ret);
  254. ret = -1;
  255. } else {
  256. DBG("regulator_enable() success");
  257. }
  258. if (!ret) {
  259. ret = ubwcp_enable_clocks(ubwcp);
  260. if (ret) {
  261. ERR("enable clocks failed: %d", ret);
  262. regulator_disable(ubwcp->vdd);
  263. } else {
  264. DBG("enable clocks success");
  265. }
  266. }
  267. } else {
  268. ret = regulator_disable(ubwcp->vdd);
  269. if (ret < 0) {
  270. ERR("regulator_disable failed: %d", ret);
  271. ret = -1;
  272. } else {
  273. DBG("regulator_disable() success");
  274. }
  275. if (!ret) {
  276. ubwcp_disable_clocks(ubwcp);
  277. DBG("disable clocks success");
  278. }
  279. }
  280. return ret;
  281. }
  282. static int ubwcp_flush(struct ubwcp_driver *ubwcp)
  283. {
  284. int ret = 0;
  285. mutex_lock(&ubwcp->ubwcp_flush_lock);
  286. ret = ubwcp_hw_flush(ubwcp->base);
  287. mutex_unlock(&ubwcp->ubwcp_flush_lock);
  288. if (ret != 0)
  289. WARN(1, "ubwcp_hw_flush() failed!");
  290. return ret;
  291. }
  292. /* get dma_buf ptr for the given dma_buf fd */
  293. struct dma_buf *ubwcp_dma_buf_fd_to_dma_buf(int dma_buf_fd)
  294. {
  295. struct dma_buf *dmabuf;
  296. /* TBD: dma_buf_get() results in taking ref to buf and it won't ever get
  297. * free'ed until ref count goes to 0. So we must reduce the ref count
  298. * immediately after we find our corresponding ubwcp_buf.
  299. */
  300. dmabuf = dma_buf_get(dma_buf_fd);
  301. if (IS_ERR(dmabuf)) {
  302. ERR("dmabuf ptr not found for dma_buf_fd = %d", dma_buf_fd);
  303. return NULL;
  304. }
  305. dma_buf_put(dmabuf);
  306. return dmabuf;
  307. }
  308. EXPORT_SYMBOL(ubwcp_dma_buf_fd_to_dma_buf);
  309. /* get ubwcp_buf corresponding to the given dma_buf */
  310. static struct ubwcp_buf *dma_buf_to_ubwcp_buf(struct dma_buf *dmabuf)
  311. {
  312. struct ubwcp_buf *buf = NULL;
  313. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  314. unsigned long flags;
  315. if (!dmabuf || !ubwcp)
  316. return NULL;
  317. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  318. /* look up ubwcp_buf corresponding to this dma_buf */
  319. hash_for_each_possible(ubwcp->buf_table, buf, hnode, (u64)dmabuf) {
  320. if (buf->dma_buf == dmabuf)
  321. break;
  322. }
  323. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  324. return buf;
  325. }
  326. /* return ubwcp hardware version */
  327. int ubwcp_get_hw_version(struct ubwcp_ioctl_hw_version *ver)
  328. {
  329. struct ubwcp_driver *ubwcp;
  330. FENTRY();
  331. if (!ver) {
  332. ERR("invalid version ptr");
  333. return -EINVAL;
  334. }
  335. ubwcp = ubwcp_get_driver();
  336. if (!ubwcp)
  337. return -1;
  338. ver->major = ubwcp->hw_ver_major;
  339. ver->minor = ubwcp->hw_ver_minor;
  340. return 0;
  341. }
  342. EXPORT_SYMBOL(ubwcp_get_hw_version);
  343. /**
  344. *
  345. * Initialize ubwcp buffer for the given dma_buf. This
  346. * initializes ubwcp internal data structures and possibly hw to
  347. * use ubwcp for this buffer.
  348. *
  349. * @param dmabuf : ptr to the buffer to be configured for ubwcp
  350. *
  351. * @return int : 0 on success, otherwise error code
  352. */
  353. static int ubwcp_init_buffer(struct dma_buf *dmabuf)
  354. {
  355. int ret = 0;
  356. int nid;
  357. struct ubwcp_buf *buf;
  358. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  359. unsigned long flags;
  360. bool table_empty;
  361. FENTRY();
  362. if (!ubwcp)
  363. return -1;
  364. if (!dmabuf) {
  365. ERR("NULL dmabuf input ptr");
  366. return -EINVAL;
  367. }
  368. if (dma_buf_to_ubwcp_buf(dmabuf)) {
  369. ERR("dma_buf already initialized for ubwcp");
  370. return -EEXIST;
  371. }
  372. buf = kzalloc(sizeof(*buf), GFP_KERNEL);
  373. if (!buf) {
  374. ERR("failed to alloc for new ubwcp_buf");
  375. return -ENOMEM;
  376. }
  377. mutex_init(&buf->lock);
  378. buf->dma_buf = dmabuf;
  379. buf->ubwcp = ubwcp;
  380. mutex_lock(&ubwcp->mem_hotplug_lock);
  381. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  382. table_empty = hash_empty(ubwcp->buf_table);
  383. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  384. if (table_empty) {
  385. ret = ubwcp_power(ubwcp, true);
  386. if (ret)
  387. goto err_power_on;
  388. nid = memory_add_physaddr_to_nid(ubwcp->ula_pool_base);
  389. DBG("calling add_memory()...");
  390. ret = add_memory(nid, ubwcp->ula_pool_base, ubwcp->ula_pool_size, MHP_NONE);
  391. if (ret) {
  392. ERR("add_memory() failed st:0x%lx sz:0x%lx err: %d",
  393. ubwcp->ula_pool_base,
  394. ubwcp->ula_pool_size,
  395. ret);
  396. goto err_add_memory;
  397. } else {
  398. DBG("add_memory() ula_pool_base:0x%llx, size:0x%zx, kernel addr:0x%p",
  399. ubwcp->ula_pool_base,
  400. ubwcp->ula_pool_size,
  401. page_to_virt(pfn_to_page(PFN_DOWN(ubwcp->ula_pool_base))));
  402. }
  403. }
  404. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  405. hash_add(ubwcp->buf_table, &buf->hnode, (u64)buf->dma_buf);
  406. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  407. mutex_unlock(&ubwcp->mem_hotplug_lock);
  408. return ret;
  409. err_add_memory:
  410. ubwcp_power(ubwcp, false);
  411. err_power_on:
  412. mutex_unlock(&ubwcp->mem_hotplug_lock);
  413. kfree(buf);
  414. if (!ret)
  415. ret = -1;
  416. return ret;
  417. }
  418. static void dump_attributes(struct ubwcp_buffer_attrs *attr)
  419. {
  420. DBG_BUF_ATTR("");
  421. DBG_BUF_ATTR("image_format: %d", attr->image_format);
  422. DBG_BUF_ATTR("major_ubwc_ver: %d", attr->major_ubwc_ver);
  423. DBG_BUF_ATTR("minor_ubwc_ver: %d", attr->minor_ubwc_ver);
  424. DBG_BUF_ATTR("compression_type: %d", attr->compression_type);
  425. DBG_BUF_ATTR("lossy_params: %llu", attr->lossy_params);
  426. DBG_BUF_ATTR("width: %d", attr->width);
  427. DBG_BUF_ATTR("height: %d", attr->height);
  428. DBG_BUF_ATTR("stride: %d", attr->stride);
  429. DBG_BUF_ATTR("scanlines: %d", attr->scanlines);
  430. DBG_BUF_ATTR("planar_padding: %d", attr->planar_padding);
  431. DBG_BUF_ATTR("subsample: %d", attr->subsample);
  432. DBG_BUF_ATTR("sub_system_target: %d", attr->sub_system_target);
  433. DBG_BUF_ATTR("y_offset: %d", attr->y_offset);
  434. DBG_BUF_ATTR("batch_size: %d", attr->batch_size);
  435. DBG_BUF_ATTR("");
  436. }
  437. /* validate buffer attributes */
  438. static bool ubwcp_buf_attrs_valid(struct ubwcp_buffer_attrs *attr)
  439. {
  440. bool valid_format;
  441. switch (attr->image_format) {
  442. case UBWCP_LINEAR:
  443. case UBWCP_RGBA8888:
  444. case UBWCP_NV12:
  445. case UBWCP_NV12_Y:
  446. case UBWCP_NV12_UV:
  447. case UBWCP_NV124R:
  448. case UBWCP_NV124R_Y:
  449. case UBWCP_NV124R_UV:
  450. case UBWCP_TP10:
  451. case UBWCP_TP10_Y:
  452. case UBWCP_TP10_UV:
  453. case UBWCP_P010:
  454. case UBWCP_P010_Y:
  455. case UBWCP_P010_UV:
  456. case UBWCP_P016:
  457. case UBWCP_P016_Y:
  458. case UBWCP_P016_UV:
  459. valid_format = true;
  460. break;
  461. default:
  462. valid_format = false;
  463. }
  464. if (!valid_format) {
  465. ERR("invalid image format: %d", attr->image_format);
  466. goto err;
  467. }
  468. if (attr->major_ubwc_ver || attr->minor_ubwc_ver) {
  469. ERR("major/minor ubwc ver must be 0. major: %d minor: %d",
  470. attr->major_ubwc_ver, attr->minor_ubwc_ver);
  471. goto err;
  472. }
  473. if (attr->compression_type != UBWCP_COMPRESSION_LOSSLESS) {
  474. ERR("compression_type is not valid: %d",
  475. attr->compression_type);
  476. goto err;
  477. }
  478. if (attr->lossy_params != 0) {
  479. ERR("lossy_params is not valid: %d", attr->lossy_params);
  480. goto err;
  481. }
  482. //TBD: some upper limit for width?
  483. if (attr->width > 10*1024) {
  484. ERR("width is invalid (above upper limit): %d", attr->width);
  485. goto err;
  486. }
  487. //TBD: some upper limit for height?
  488. if (attr->height > 10*1024) {
  489. ERR("height is invalid (above upper limit): %d", attr->height);
  490. goto err;
  491. }
  492. /* TBD: what's the upper limit for stride? 8K is likely too high. */
  493. if (!IS_ALIGNED(attr->stride, 64) ||
  494. (attr->stride < attr->width) ||
  495. (attr->stride > 4*8192)) {
  496. ERR("stride is not valid (aligned to 64 and <= 8192): %d",
  497. attr->stride);
  498. goto err;
  499. }
  500. /* TBD: currently assume height + 10. Replace 10 with right num from camera. */
  501. if ((attr->scanlines < attr->height) ||
  502. (attr->scanlines > attr->height + 10)) {
  503. ERR("scanlines is not valid - height: %d scanlines: %d",
  504. attr->height, attr->scanlines);
  505. goto err;
  506. }
  507. if (attr->planar_padding > 4096) {
  508. ERR("planar_padding is not valid. (<= 4096): %d",
  509. attr->planar_padding);
  510. goto err;
  511. }
  512. if (attr->subsample != UBWCP_SUBSAMPLE_4_2_0) {
  513. ERR("subsample is not valid: %d", attr->subsample);
  514. goto err;
  515. }
  516. if (attr->sub_system_target & ~UBWCP_SUBSYSTEM_TARGET_CPU) {
  517. ERR("sub_system_target other that CPU is not supported: %d",
  518. attr->sub_system_target);
  519. goto err;
  520. }
  521. if (!(attr->sub_system_target & UBWCP_SUBSYSTEM_TARGET_CPU)) {
  522. ERR("sub_system_target is not set to CPU: %d",
  523. attr->sub_system_target);
  524. goto err;
  525. }
  526. if (attr->y_offset != 0) {
  527. ERR("y_offset is not valid: %d", attr->y_offset);
  528. goto err;
  529. }
  530. if (attr->batch_size != 1) {
  531. ERR("batch_size is not valid: %d", attr->batch_size);
  532. goto err;
  533. }
  534. dump_attributes(attr);
  535. return true;
  536. err:
  537. dump_attributes(attr);
  538. return false;
  539. }
  540. /* return true if image format has only Y plane*/
  541. bool ubwcp_image_y_only(u16 format)
  542. {
  543. switch (format) {
  544. case UBWCP_NV12_Y:
  545. case UBWCP_NV124R_Y:
  546. case UBWCP_TP10_Y:
  547. case UBWCP_P010_Y:
  548. case UBWCP_P016_Y:
  549. return true;
  550. default:
  551. return false;
  552. }
  553. }
  554. /* return true if image format has only UV plane*/
  555. bool ubwcp_image_uv_only(u16 format)
  556. {
  557. switch (format) {
  558. case UBWCP_NV12_UV:
  559. case UBWCP_NV124R_UV:
  560. case UBWCP_TP10_UV:
  561. case UBWCP_P010_UV:
  562. case UBWCP_P016_UV:
  563. return true;
  564. default:
  565. return false;
  566. }
  567. }
  568. /* calculate and return metadata buffer size for a given plane
  569. * and buffer attributes
  570. * NOTE: in this function, we will only pass in NV12 format.
  571. * NOT NV12_Y or NV12_UV etc.
  572. * the Y or UV information is in the "plane"
  573. * "format" here purely means "encoding format" and no information
  574. * if some plane data is missing.
  575. */
  576. static size_t metadata_buf_sz(struct ubwcp_driver *ubwcp,
  577. enum ubwcp_std_image_format format,
  578. u32 width, u32 height, u8 plane)
  579. {
  580. size_t size;
  581. u64 pitch;
  582. u64 lines;
  583. u64 tile_width;
  584. u32 tile_height;
  585. struct ubwcp_image_format_info f_info;
  586. struct ubwcp_plane_info p_info;
  587. f_info = ubwcp->format_info[format];
  588. DBG_BUF_ATTR("");
  589. DBG_BUF_ATTR("");
  590. DBG_BUF_ATTR("Calculating metadata buffer size: format = %d, plane = %d", format, plane);
  591. if (plane >= f_info.planes) {
  592. ERR("Format does not have requested plane info: format: %d, plane: %d",
  593. format, plane);
  594. WARN(1, "Fix this!!!!!");
  595. return 0;
  596. }
  597. p_info = f_info.p_info[plane];
  598. /* UV plane */
  599. if (plane == 1) {
  600. width = width/2;
  601. height = height/2;
  602. }
  603. tile_width = p_info.tilesize_p.width;
  604. tile_height = p_info.tilesize_p.height;
  605. /* pitch: # of tiles in a row
  606. * lines: # of tile rows
  607. */
  608. pitch = UBWCP_ALIGN((width + tile_width - 1)/tile_width, META_DATA_PITCH_ALIGN);
  609. lines = UBWCP_ALIGN((height + tile_height - 1)/tile_height, META_DATA_HEIGHT_ALIGN);
  610. DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
  611. DBG_BUF_ATTR("tile params : %d x %d (pixels)", tile_width, tile_height);
  612. DBG_BUF_ATTR("pitch : %d (%d)", pitch, width/tile_width);
  613. DBG_BUF_ATTR("lines : %d (%d)", lines, height);
  614. DBG_BUF_ATTR("size (p*l*bytes) : %d", pitch*lines*1);
  615. /* x1 below is only to clarify that we are multiplying by 1 bytes/tile */
  616. size = UBWCP_ALIGN(pitch*lines*1, META_DATA_SIZE_ALIGN);
  617. DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", size, size);
  618. return size;
  619. }
  620. /* calculate and return size of pixel data buffer for a given plane
  621. * and buffer attributes
  622. */
  623. static size_t pixeldata_buf_sz(struct ubwcp_driver *ubwcp,
  624. u16 format, u32 width,
  625. u32 height, u8 plane)
  626. {
  627. size_t size;
  628. u64 pitch;
  629. u64 lines;
  630. u16 pixel_bytes;
  631. u16 per_pixel;
  632. u64 macro_tile_width_p;
  633. u64 macro_tile_height_p;
  634. struct ubwcp_image_format_info f_info;
  635. struct ubwcp_plane_info p_info;
  636. f_info = ubwcp->format_info[format];
  637. DBG_BUF_ATTR("");
  638. DBG_BUF_ATTR("");
  639. DBG_BUF_ATTR("Calculating Pixeldata buffer size: format = %d, plane = %d", format, plane);
  640. if (plane >= f_info.planes) {
  641. ERR("Format does not have requested plane info: format: %d, plane: %d",
  642. format, plane);
  643. WARN(1, "Fix this!!!!!");
  644. return 0;
  645. }
  646. p_info = f_info.p_info[plane];
  647. pixel_bytes = p_info.pixel_bytes;
  648. per_pixel = p_info.per_pixel;
  649. /* UV plane */
  650. if (plane == 1) {
  651. width = width/2;
  652. height = height/2;
  653. }
  654. macro_tile_width_p = p_info.macrotilesize_p.width;
  655. macro_tile_height_p = p_info.macrotilesize_p.height;
  656. /* align pixel width and height macro tile width and height */
  657. pitch = UBWCP_ALIGN(width, macro_tile_width_p);
  658. lines = UBWCP_ALIGN(height, macro_tile_height_p);
  659. DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
  660. DBG_BUF_ATTR("macro tile params: %d x %d (pixels)", macro_tile_width_p,
  661. macro_tile_height_p);
  662. DBG_BUF_ATTR("bytes_per_pixel : %d/%d", pixel_bytes, per_pixel);
  663. DBG_BUF_ATTR("pitch : %d", pitch);
  664. DBG_BUF_ATTR("lines : %d", lines);
  665. DBG_BUF_ATTR("size (p*l*bytes) : %d", (pitch*lines*pixel_bytes)/per_pixel);
  666. size = UBWCP_ALIGN((pitch*lines*pixel_bytes)/per_pixel, PIXEL_DATA_SIZE_ALIGN);
  667. DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", size, size);
  668. return size;
  669. }
  670. static int get_tile_height(struct ubwcp_driver *ubwcp, enum ubwcp_std_image_format format,
  671. u8 plane)
  672. {
  673. struct ubwcp_image_format_info f_info;
  674. struct ubwcp_plane_info p_info;
  675. f_info = ubwcp->format_info[format];
  676. p_info = f_info.p_info[plane];
  677. return p_info.tilesize_p.height;
  678. }
  679. /*
  680. * plane: must be 0 or 1 (1st plane == 0, 2nd plane == 1)
  681. */
  682. static size_t ubwcp_ula_size(struct ubwcp_driver *ubwcp, u16 format,
  683. u32 stride_b, u32 scanlines, u8 plane,
  684. bool add_tile_pad)
  685. {
  686. size_t size;
  687. DBG_BUF_ATTR("%s(format = %d, plane = %d)", __func__, format, plane);
  688. /* UV plane */
  689. if (plane == 1)
  690. scanlines = scanlines/2;
  691. if (add_tile_pad) {
  692. int tile_height = get_tile_height(ubwcp, format, plane);
  693. /* Align plane size to plane tile height */
  694. scanlines = ((scanlines + tile_height - 1) / tile_height) * tile_height;
  695. }
  696. size = stride_b*scanlines;
  697. DBG_BUF_ATTR("Size of plane-%u: (%u * %u) = %zu (0x%zx)",
  698. plane, stride_b, scanlines, size, size);
  699. return size;
  700. }
  701. int missing_plane_from_format(u16 ioctl_image_format)
  702. {
  703. int missing_plane;
  704. switch (ioctl_image_format) {
  705. case UBWCP_NV12_Y:
  706. missing_plane = 2;
  707. break;
  708. case UBWCP_NV12_UV:
  709. missing_plane = 1;
  710. break;
  711. case UBWCP_NV124R_Y:
  712. missing_plane = 2;
  713. break;
  714. case UBWCP_NV124R_UV:
  715. missing_plane = 1;
  716. break;
  717. case UBWCP_TP10_Y:
  718. missing_plane = 2;
  719. break;
  720. case UBWCP_TP10_UV:
  721. missing_plane = 1;
  722. break;
  723. case UBWCP_P010_Y:
  724. missing_plane = 2;
  725. break;
  726. case UBWCP_P010_UV:
  727. missing_plane = 1;
  728. break;
  729. case UBWCP_P016_Y:
  730. missing_plane = 2;
  731. break;
  732. case UBWCP_P016_UV:
  733. missing_plane = 1;
  734. break;
  735. default:
  736. missing_plane = 0;
  737. }
  738. return missing_plane;
  739. }
  740. int planes_in_format(enum ubwcp_std_image_format format)
  741. {
  742. if (format == RGBA)
  743. return 1;
  744. else
  745. return 2;
  746. }
  747. enum ubwcp_std_image_format to_std_format(u16 ioctl_image_format)
  748. {
  749. switch (ioctl_image_format) {
  750. case UBWCP_RGBA8888:
  751. return RGBA;
  752. case UBWCP_NV12:
  753. case UBWCP_NV12_Y:
  754. case UBWCP_NV12_UV:
  755. return NV12;
  756. case UBWCP_NV124R:
  757. case UBWCP_NV124R_Y:
  758. case UBWCP_NV124R_UV:
  759. return NV124R;
  760. case UBWCP_TP10:
  761. case UBWCP_TP10_Y:
  762. case UBWCP_TP10_UV:
  763. return TP10;
  764. case UBWCP_P010:
  765. case UBWCP_P010_Y:
  766. case UBWCP_P010_UV:
  767. return P010;
  768. case UBWCP_P016:
  769. case UBWCP_P016_Y:
  770. case UBWCP_P016_UV:
  771. return P016;
  772. default:
  773. WARN(1, "Fix this!!!");
  774. return STD_IMAGE_FORMAT_INVALID;
  775. }
  776. }
  777. unsigned int ubwcp_get_hw_image_format_value(u16 ioctl_image_format)
  778. {
  779. enum ubwcp_std_image_format format;
  780. format = to_std_format(ioctl_image_format);
  781. switch (format) {
  782. case RGBA:
  783. return HW_BUFFER_FORMAT_RGBA;
  784. case NV12:
  785. return HW_BUFFER_FORMAT_NV12;
  786. case NV124R:
  787. return HW_BUFFER_FORMAT_NV124R;
  788. case P010:
  789. return HW_BUFFER_FORMAT_P010;
  790. case TP10:
  791. return HW_BUFFER_FORMAT_TP10;
  792. case P016:
  793. return HW_BUFFER_FORMAT_P016;
  794. default:
  795. WARN(1, "Fix this!!!!!");
  796. return 0;
  797. }
  798. }
  799. static int ubwcp_validate_uv_align(struct ubwcp_driver *ubwcp,
  800. struct ubwcp_buffer_attrs *attr,
  801. size_t ula_y_plane_size,
  802. size_t uv_start_offset)
  803. {
  804. int ret = 0;
  805. size_t ula_y_plane_size_align;
  806. size_t y_tile_align_bytes;
  807. int y_tile_height;
  808. int planes;
  809. /* Only validate UV align if there is both a Y and UV plane */
  810. planes = planes_in_format(to_std_format(attr->image_format));
  811. if (planes != 2)
  812. return 0;
  813. /* Check it is cache line size aligned */
  814. if ((uv_start_offset % 64) != 0) {
  815. ret = -EINVAL;
  816. ERR("uv_start_offset %zu not cache line aligned",
  817. uv_start_offset);
  818. goto err;
  819. }
  820. /*
  821. * Check that UV plane does not overlap with any of the Y plane’s tiles
  822. */
  823. y_tile_height = get_tile_height(ubwcp, to_std_format(attr->image_format), 0);
  824. y_tile_align_bytes = y_tile_height * attr->stride;
  825. ula_y_plane_size_align = ((ula_y_plane_size + y_tile_align_bytes - 1) /
  826. y_tile_align_bytes) * y_tile_align_bytes;
  827. if (uv_start_offset < ula_y_plane_size_align) {
  828. ret = -EINVAL;
  829. ERR("uv offset %zu less than y plane align %zu for y plane size %zu",
  830. uv_start_offset, ula_y_plane_size_align,
  831. ula_y_plane_size);
  832. goto err;
  833. }
  834. return 0;
  835. err:
  836. return ret;
  837. }
  838. /* calculate ULA buffer parms
  839. * TBD: how do we make sure uv_start address (not the offset)
  840. * is aligned per requirement: cache line
  841. */
  842. static int ubwcp_calc_ula_params(struct ubwcp_driver *ubwcp,
  843. struct ubwcp_buffer_attrs *attr,
  844. size_t *ula_size,
  845. size_t *ula_y_plane_size,
  846. size_t *uv_start_offset)
  847. {
  848. size_t size;
  849. enum ubwcp_std_image_format format;
  850. int planes;
  851. int missing_plane;
  852. u32 stride;
  853. u32 scanlines;
  854. u32 planar_padding;
  855. stride = attr->stride;
  856. scanlines = attr->scanlines;
  857. planar_padding = attr->planar_padding;
  858. /* convert ioctl image format to standard image format */
  859. format = to_std_format(attr->image_format);
  860. /* Number of "expected" planes in "the standard defined" image format */
  861. planes = planes_in_format(format);
  862. /* any plane missing?
  863. * valid missing_plane values:
  864. * 0 == no plane missing
  865. * 1 == 1st plane missing
  866. * 2 == 2nd plane missing
  867. */
  868. missing_plane = missing_plane_from_format(attr->image_format);
  869. DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
  870. DBG_BUF_ATTR("planes_in_format : %d", planes);
  871. DBG_BUF_ATTR("missing_plane : %d", missing_plane);
  872. DBG_BUF_ATTR("Planar Padding : %d", planar_padding);
  873. if (planes == 1) {
  874. /* uv_start beyond ULA range */
  875. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
  876. *uv_start_offset = size;
  877. *ula_y_plane_size = size;
  878. } else {
  879. if (!missing_plane) {
  880. /* size for both planes and padding */
  881. /* Don't pad out Y plane as client would not expect this padding */
  882. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, false);
  883. *ula_y_plane_size = size;
  884. size += planar_padding;
  885. *uv_start_offset = size;
  886. size += ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
  887. } else {
  888. if (missing_plane == 2) {
  889. /* Y-only image, set uv_start beyond ULA range */
  890. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
  891. *uv_start_offset = size;
  892. *ula_y_plane_size = size;
  893. } else {
  894. /* first plane data is not there */
  895. size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
  896. *uv_start_offset = 0; /* uv data is at the beginning */
  897. *ula_y_plane_size = 0;
  898. }
  899. }
  900. }
  901. //TBD: cleanup
  902. *ula_size = size;
  903. DBG_BUF_ATTR("Before page align: Total ULA_Size: %d (0x%x) (planes + planar padding)",
  904. *ula_size, *ula_size);
  905. *ula_size = UBWCP_ALIGN(size, 4096);
  906. DBG_BUF_ATTR("After page align : Total ULA_Size: %d (0x%x) (planes + planar padding)",
  907. *ula_size, *ula_size);
  908. return 0;
  909. }
  910. /* calculate UBWCP buffer parms */
  911. static int ubwcp_calc_ubwcp_buf_params(struct ubwcp_driver *ubwcp,
  912. struct ubwcp_buffer_attrs *attr,
  913. size_t *md_p0, size_t *pd_p0,
  914. size_t *md_p1, size_t *pd_p1,
  915. size_t *stride_tp10_b)
  916. {
  917. int planes;
  918. int missing_plane;
  919. enum ubwcp_std_image_format format;
  920. size_t stride_tp10_p;
  921. FENTRY();
  922. /* convert ioctl image format to standard image format */
  923. format = to_std_format(attr->image_format);
  924. missing_plane = missing_plane_from_format(attr->image_format);
  925. planes = planes_in_format(format); //pass in 0 (RGB) should return 1
  926. DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
  927. DBG_BUF_ATTR("planes_in_format : %d", planes);
  928. DBG_BUF_ATTR("missing_plane : %d", missing_plane);
  929. if (!missing_plane) {
  930. *md_p0 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  931. *pd_p0 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  932. if (planes == 2) {
  933. *md_p1 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  934. *pd_p1 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  935. }
  936. } else {
  937. if (missing_plane == 1) {
  938. *md_p0 = 0;
  939. *pd_p0 = 0;
  940. *md_p1 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  941. *pd_p1 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 1);
  942. } else {
  943. *md_p0 = metadata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  944. *pd_p0 = pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 0);
  945. *md_p1 = 0;
  946. *pd_p1 = 0;
  947. }
  948. }
  949. if (format == TP10) {
  950. stride_tp10_p = UBWCP_ALIGN(attr->width, 192);
  951. *stride_tp10_b = (stride_tp10_p/3) + stride_tp10_p;
  952. } else {
  953. *stride_tp10_b = 0;
  954. }
  955. return 0;
  956. }
  957. /* reserve ULA address space of the given size */
  958. static phys_addr_t ubwcp_ula_alloc(struct ubwcp_driver *ubwcp, size_t size)
  959. {
  960. phys_addr_t pa;
  961. mutex_lock(&ubwcp->ula_lock);
  962. pa = gen_pool_alloc(ubwcp->ula_pool, size);
  963. DBG("addr: %p, size: %zx", pa, size);
  964. mutex_unlock(&ubwcp->ula_lock);
  965. return pa;
  966. }
  967. /* free ULA address space of the given address and size */
  968. static void ubwcp_ula_free(struct ubwcp_driver *ubwcp, phys_addr_t pa, size_t size)
  969. {
  970. mutex_lock(&ubwcp->ula_lock);
  971. if (!gen_pool_has_addr(ubwcp->ula_pool, pa, size)) {
  972. ERR("Attempt to free mem not from gen_pool: pa: %p, size: %zx", pa, size);
  973. goto err;
  974. }
  975. DBG("addr: %p, size: %zx", pa, size);
  976. gen_pool_free(ubwcp->ula_pool, pa, size);
  977. mutex_unlock(&ubwcp->ula_lock);
  978. return;
  979. err:
  980. mutex_unlock(&ubwcp->ula_lock);
  981. }
  982. /* free up or expand current_pa and return the new pa */
  983. static phys_addr_t ubwcp_ula_realloc(struct ubwcp_driver *ubwcp,
  984. phys_addr_t pa,
  985. size_t size,
  986. size_t new_size)
  987. {
  988. if (size == new_size)
  989. return pa;
  990. if (pa)
  991. ubwcp_ula_free(ubwcp, pa, size);
  992. return ubwcp_ula_alloc(ubwcp, new_size);
  993. }
  994. /* unmap dma buf */
  995. static void ubwcp_dma_unmap(struct ubwcp_buf *buf)
  996. {
  997. FENTRY();
  998. if (buf->dma_buf && buf->attachment) {
  999. DBG("Calling dma_buf_unmap_attachment()");
  1000. dma_buf_unmap_attachment(buf->attachment, buf->sgt, DMA_BIDIRECTIONAL);
  1001. buf->sgt = NULL;
  1002. dma_buf_detach(buf->dma_buf, buf->attachment);
  1003. buf->attachment = NULL;
  1004. }
  1005. }
  1006. /* dma map ubwcp buffer */
  1007. static int ubwcp_dma_map(struct ubwcp_buf *buf,
  1008. struct device *dev,
  1009. size_t iova_min_size,
  1010. dma_addr_t *iova)
  1011. {
  1012. int ret = 0;
  1013. struct dma_buf *dma_buf = buf->dma_buf;
  1014. struct dma_buf_attachment *attachment;
  1015. struct sg_table *sgt;
  1016. size_t dma_len;
  1017. /* Map buffer to SMMU and get IOVA */
  1018. attachment = dma_buf_attach(dma_buf, dev);
  1019. if (IS_ERR(attachment)) {
  1020. ret = PTR_ERR(attachment);
  1021. ERR("dma_buf_attach() failed: %d", ret);
  1022. goto err;
  1023. }
  1024. dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
  1025. dma_set_seg_boundary(dev, (unsigned long)DMA_BIT_MASK(64));
  1026. sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
  1027. if (IS_ERR_OR_NULL(sgt)) {
  1028. ret = PTR_ERR(sgt);
  1029. ERR("dma_buf_map_attachment() failed: %d", ret);
  1030. goto err_detach;
  1031. }
  1032. if (sgt->nents != 1) {
  1033. ERR("nents = %d", sgt->nents);
  1034. goto err_unmap;
  1035. }
  1036. /* ensure that dma_buf is big enough for the new attrs */
  1037. dma_len = sg_dma_len(sgt->sgl);
  1038. if (dma_len < iova_min_size) {
  1039. ERR("dma len: %d is less than min ubwcp buffer size: %d",
  1040. dma_len, iova_min_size);
  1041. goto err_unmap;
  1042. }
  1043. *iova = sg_dma_address(sgt->sgl);
  1044. buf->attachment = attachment;
  1045. buf->sgt = sgt;
  1046. return ret;
  1047. err_unmap:
  1048. dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
  1049. err_detach:
  1050. dma_buf_detach(dma_buf, attachment);
  1051. err:
  1052. if (!ret)
  1053. ret = -1;
  1054. return ret;
  1055. }
  1056. static void
  1057. ubwcp_pixel_to_bytes(struct ubwcp_driver *ubwcp,
  1058. enum ubwcp_std_image_format format,
  1059. u32 width_p, u32 height_p,
  1060. u32 *width_b, u32 *height_b)
  1061. {
  1062. u16 pixel_bytes;
  1063. u16 per_pixel;
  1064. struct ubwcp_image_format_info f_info;
  1065. struct ubwcp_plane_info p_info;
  1066. f_info = ubwcp->format_info[format];
  1067. p_info = f_info.p_info[0];
  1068. pixel_bytes = p_info.pixel_bytes;
  1069. per_pixel = p_info.per_pixel;
  1070. *width_b = (width_p*pixel_bytes)/per_pixel;
  1071. *height_b = (height_p*pixel_bytes)/per_pixel;
  1072. }
  1073. static void reset_buf_attrs(struct ubwcp_buf *buf)
  1074. {
  1075. struct ubwcp_hw_meta_metadata *mmdata;
  1076. struct ubwcp_driver *ubwcp;
  1077. ubwcp = buf->ubwcp;
  1078. mmdata = &buf->mmdata;
  1079. ubwcp_dma_unmap(buf);
  1080. /* reset ula params */
  1081. if (buf->ula_size) {
  1082. ubwcp_ula_free(ubwcp, buf->ula_pa, buf->ula_size);
  1083. buf->ula_size = 0;
  1084. buf->ula_pa = 0;
  1085. }
  1086. /* reset ubwcp params */
  1087. memset(mmdata, 0, sizeof(*mmdata));
  1088. buf->buf_attr_set = false;
  1089. }
  1090. static void print_mmdata_desc(struct ubwcp_hw_meta_metadata *mmdata)
  1091. {
  1092. DBG_BUF_ATTR("");
  1093. DBG_BUF_ATTR("--------MM_DATA DESC ---------");
  1094. DBG_BUF_ATTR("uv_start_addr : 0x%08llx (cache addr) (actual: 0x%llx)",
  1095. mmdata->uv_start_addr, mmdata->uv_start_addr << 6);
  1096. DBG_BUF_ATTR("format : 0x%08x", mmdata->format);
  1097. DBG_BUF_ATTR("stride : 0x%08x (cache addr) (actual: 0x%x)",
  1098. mmdata->stride, mmdata->stride << 6);
  1099. DBG_BUF_ATTR("stride_ubwcp : 0x%08x (cache addr) (actual: 0x%zx)",
  1100. mmdata->stride_ubwcp, mmdata->stride_ubwcp << 6);
  1101. DBG_BUF_ATTR("metadata_base_y : 0x%08x (page addr) (actual: 0x%llx)",
  1102. mmdata->metadata_base_y, mmdata->metadata_base_y << 12);
  1103. DBG_BUF_ATTR("metadata_base_uv: 0x%08x (page addr) (actual: 0x%zx)",
  1104. mmdata->metadata_base_uv, mmdata->metadata_base_uv << 12);
  1105. DBG_BUF_ATTR("buffer_y_offset : 0x%08x (page addr) (actual: 0x%zx)",
  1106. mmdata->buffer_y_offset, mmdata->buffer_y_offset << 12);
  1107. DBG_BUF_ATTR("buffer_uv_offset: 0x%08x (page addr) (actual: 0x%zx)",
  1108. mmdata->buffer_uv_offset, mmdata->buffer_uv_offset << 12);
  1109. DBG_BUF_ATTR("width_height : 0x%08x (width: 0x%x height: 0x%x)",
  1110. mmdata->width_height, mmdata->width_height >> 16, mmdata->width_height & 0xFFFF);
  1111. DBG_BUF_ATTR("");
  1112. }
  1113. /* set buffer attributes:
  1114. * Failure:
  1115. * If a call to ubwcp_set_buf_attrs() fails, any attributes set from a previously
  1116. * successful ubwcp_set_buf_attrs() will be also removed. Thus,
  1117. * ubwcp_set_buf_attrs() implicitly does "unset previous attributes" and
  1118. * then "try to set these new attributes".
  1119. *
  1120. * The result of a failed call to ubwcp_set_buf_attrs() will leave the buffer
  1121. * in a linear mode, NOT with attributes from earlier successful call.
  1122. */
  1123. int ubwcp_set_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
  1124. {
  1125. int ret = 0;
  1126. size_t ula_size = 0;
  1127. size_t uv_start_offset = 0;
  1128. size_t ula_y_plane_size = 0;
  1129. phys_addr_t ula_pa = 0x0;
  1130. struct ubwcp_buf *buf;
  1131. struct ubwcp_driver *ubwcp;
  1132. size_t metadata_p0;
  1133. size_t pixeldata_p0;
  1134. size_t metadata_p1;
  1135. size_t pixeldata_p1;
  1136. size_t iova_min_size;
  1137. size_t stride_tp10_b;
  1138. dma_addr_t iova_base;
  1139. struct ubwcp_hw_meta_metadata *mmdata;
  1140. u64 uv_start;
  1141. u32 stride_b;
  1142. u32 width_b;
  1143. u32 height_b;
  1144. enum ubwcp_std_image_format std_image_format;
  1145. FENTRY();
  1146. if (!dmabuf) {
  1147. ERR("NULL dmabuf input ptr");
  1148. return -EINVAL;
  1149. }
  1150. if (!attr) {
  1151. ERR("NULL attr ptr");
  1152. return -EINVAL;
  1153. }
  1154. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1155. if (!buf) {
  1156. ERR("No corresponding ubwcp_buf for the passed in dma_buf");
  1157. return -EINVAL;
  1158. }
  1159. mutex_lock(&buf->lock);
  1160. if (buf->locked) {
  1161. ERR("Cannot set attr when buffer is locked");
  1162. ret = -EBUSY;
  1163. goto err;
  1164. }
  1165. ubwcp = buf->ubwcp;
  1166. mmdata = &buf->mmdata;
  1167. //TBD: now that we have single exit point for all errors,
  1168. //we can limit this call to error only?
  1169. //also see if this can be part of reset_buf_attrs()
  1170. DBG_BUF_ATTR("resetting mmap to linear");
  1171. /* remove any earlier dma buf mmap configuration */
  1172. ret = ubwcp->mmap_config_fptr(buf->dma_buf, true, 0, 0);
  1173. if (ret) {
  1174. ERR("dma_buf_mmap_config() failed: %d", ret);
  1175. goto err;
  1176. }
  1177. if (!ubwcp_buf_attrs_valid(attr)) {
  1178. ERR("Invalid buf attrs");
  1179. goto err;
  1180. }
  1181. DBG_BUF_ATTR("valid buf attrs");
  1182. if (attr->image_format == UBWCP_LINEAR) {
  1183. DBG_BUF_ATTR("Linear format requested");
  1184. /* linear format request with permanent range xlation doesn't
  1185. * make sense. need to define behavior if this happens.
  1186. * note: with perm set, desc is allocated to this buffer.
  1187. */
  1188. //TBD: UBWCP_ASSERT(!buf->perm);
  1189. if (buf->buf_attr_set)
  1190. reset_buf_attrs(buf);
  1191. mutex_unlock(&buf->lock);
  1192. return 0;
  1193. }
  1194. std_image_format = to_std_format(attr->image_format);
  1195. if (std_image_format == STD_IMAGE_FORMAT_INVALID) {
  1196. ERR("Unable to map ioctl image format to std image format");
  1197. goto err;
  1198. }
  1199. /* Calculate uncompressed-buffer size. */
  1200. DBG_BUF_ATTR("");
  1201. DBG_BUF_ATTR("");
  1202. DBG_BUF_ATTR("Calculating ula params -->");
  1203. ret = ubwcp_calc_ula_params(ubwcp, attr, &ula_size, &ula_y_plane_size, &uv_start_offset);
  1204. if (ret) {
  1205. ERR("ubwcp_calc_ula_params() failed: %d", ret);
  1206. goto err;
  1207. }
  1208. ret = ubwcp_validate_uv_align(ubwcp, attr, ula_y_plane_size, uv_start_offset);
  1209. if (ret) {
  1210. ERR("ubwcp_validate_uv_align() failed: %d", ret);
  1211. goto err;
  1212. }
  1213. DBG_BUF_ATTR("");
  1214. DBG_BUF_ATTR("");
  1215. DBG_BUF_ATTR("Calculating ubwcp params -->");
  1216. ret = ubwcp_calc_ubwcp_buf_params(ubwcp, attr,
  1217. &metadata_p0, &pixeldata_p0,
  1218. &metadata_p1, &pixeldata_p1,
  1219. &stride_tp10_b);
  1220. if (ret) {
  1221. ERR("ubwcp_calc_buf_params() failed: %d", ret);
  1222. goto err;
  1223. }
  1224. iova_min_size = metadata_p0 + pixeldata_p0 + metadata_p1 + pixeldata_p1;
  1225. DBG_BUF_ATTR("");
  1226. DBG_BUF_ATTR("");
  1227. DBG_BUF_ATTR("------Summary ULA Calculated Params ------");
  1228. DBG_BUF_ATTR("ULA Size : %8zu (0x%8zx)", ula_size, ula_size);
  1229. DBG_BUF_ATTR("UV Start Offset : %8zu (0x%8zx)", uv_start_offset, uv_start_offset);
  1230. DBG_BUF_ATTR("------Summary UBCP Calculated Params ------");
  1231. DBG_BUF_ATTR("metadata_p0 : %8d (0x%8zx)", metadata_p0, metadata_p0);
  1232. DBG_BUF_ATTR("pixeldata_p0 : %8d (0x%8zx)", pixeldata_p0, pixeldata_p0);
  1233. DBG_BUF_ATTR("metadata_p1 : %8d (0x%8zx)", metadata_p1, metadata_p1);
  1234. DBG_BUF_ATTR("pixeldata_p1 : %8d (0x%8zx)", pixeldata_p1, pixeldata_p1);
  1235. DBG_BUF_ATTR("stride_tp10 : %8d (0x%8zx)", stride_tp10_b, stride_tp10_b);
  1236. DBG_BUF_ATTR("iova_min_size : %8d (0x%8zx)", iova_min_size, iova_min_size);
  1237. DBG_BUF_ATTR("");
  1238. if (buf->buf_attr_set) {
  1239. /* if buf attr were previously set, these must not be 0 */
  1240. /* TBD: do we need this check in production code? */
  1241. if (!buf->ula_pa) {
  1242. WARN(1, "ula_pa cannot be 0 if buf_attr_set is true!!!");
  1243. goto err;
  1244. }
  1245. if (!buf->ula_size) {
  1246. WARN(1, "ula_size cannot be 0 if buf_attr_set is true!!!");
  1247. goto err;
  1248. }
  1249. }
  1250. /* assign ULA PA with uncompressed-size range */
  1251. ula_pa = ubwcp_ula_realloc(ubwcp, buf->ula_pa, buf->ula_size, ula_size);
  1252. if (!ula_pa) {
  1253. ERR("ubwcp_ula_alloc/realloc() failed. running out of ULA PA space?");
  1254. goto err;
  1255. }
  1256. buf->ula_size = ula_size;
  1257. buf->ula_pa = ula_pa;
  1258. DBG_BUF_ATTR("Allocated ULA_PA: 0x%p of size: 0x%zx", ula_pa, ula_size);
  1259. DBG_BUF_ATTR("");
  1260. /* inform ULA-PA to dma-heap: needed for dma-heap to do CMOs later on */
  1261. DBG_BUF_ATTR("Calling mmap_config(): ULA_PA: 0x%p size: 0x%zx", ula_pa, ula_size);
  1262. ret = ubwcp->mmap_config_fptr(buf->dma_buf, false, buf->ula_pa,
  1263. buf->ula_size);
  1264. if (ret) {
  1265. ERR("dma_buf_mmap_config() failed: %d", ret);
  1266. goto err;
  1267. }
  1268. /* dma map only the first time attribute is set */
  1269. if (!buf->buf_attr_set) {
  1270. /* linear -> ubwcp. map ubwcp buffer */
  1271. ret = ubwcp_dma_map(buf, ubwcp->dev_buf_cb, iova_min_size, &iova_base);
  1272. if (ret) {
  1273. ERR("ubwcp_dma_map() failed: %d", ret);
  1274. goto err;
  1275. }
  1276. DBG_BUF_ATTR("dma_buf IOVA range: 0x%llx + min_size (0x%zx): 0x%llx",
  1277. iova_base, iova_min_size, iova_base + iova_min_size);
  1278. }
  1279. uv_start = ula_pa + uv_start_offset;
  1280. if (!IS_ALIGNED(uv_start, 64)) {
  1281. ERR("ERROR: uv_start is NOT aligned to cache line");
  1282. goto err;
  1283. }
  1284. /* Convert height and width to bytes for writing to mmdata */
  1285. if (std_image_format != TP10) {
  1286. ubwcp_pixel_to_bytes(ubwcp, std_image_format, attr->width,
  1287. attr->height, &width_b, &height_b);
  1288. } else {
  1289. /* for tp10 image compression, we need to program p010 width/height */
  1290. ubwcp_pixel_to_bytes(ubwcp, P010, attr->width,
  1291. attr->height, &width_b, &height_b);
  1292. }
  1293. stride_b = attr->stride;
  1294. /* create the mmdata descriptor */
  1295. memset(mmdata, 0, sizeof(*mmdata));
  1296. mmdata->uv_start_addr = CACHE_ADDR(uv_start);
  1297. mmdata->format = ubwcp_get_hw_image_format_value(attr->image_format);
  1298. if (std_image_format != TP10) {
  1299. mmdata->stride = CACHE_ADDR(stride_b); /* uncompressed stride */
  1300. } else {
  1301. mmdata->stride = CACHE_ADDR(stride_tp10_b); /* compressed stride */
  1302. mmdata->stride_ubwcp = CACHE_ADDR(stride_b); /* uncompressed stride */
  1303. }
  1304. mmdata->metadata_base_y = PAGE_ADDR(iova_base);
  1305. mmdata->metadata_base_uv = PAGE_ADDR(iova_base + metadata_p0 + pixeldata_p0);
  1306. mmdata->buffer_y_offset = PAGE_ADDR(metadata_p0);
  1307. mmdata->buffer_uv_offset = PAGE_ADDR(metadata_p1);
  1308. /* NOTE: For version 1.1, both width & height needs to be in bytes.
  1309. * For other versions, width in bytes & height in pixels.
  1310. */
  1311. if ((ubwcp->hw_ver_major == 1) && (ubwcp->hw_ver_minor == 1))
  1312. mmdata->width_height = width_b << 16 | height_b;
  1313. else
  1314. mmdata->width_height = width_b << 16 | attr->height;
  1315. print_mmdata_desc(mmdata);
  1316. buf->buf_attr = *attr;
  1317. buf->buf_attr_set = true;
  1318. //TBD: UBWCP_ASSERT(!buf->perm);
  1319. mutex_unlock(&buf->lock);
  1320. return 0;
  1321. err:
  1322. reset_buf_attrs(buf);
  1323. mutex_unlock(&buf->lock);
  1324. if (!ret)
  1325. ret = -1;
  1326. return ret;
  1327. }
  1328. EXPORT_SYMBOL(ubwcp_set_buf_attrs);
  1329. /* Set buffer attributes ioctl */
  1330. static int ubwcp_set_buf_attrs_ioctl(struct ubwcp_ioctl_buffer_attrs *attr_ioctl)
  1331. {
  1332. struct dma_buf *dmabuf;
  1333. dmabuf = ubwcp_dma_buf_fd_to_dma_buf(attr_ioctl->fd);
  1334. return ubwcp_set_buf_attrs(dmabuf, &attr_ioctl->attr);
  1335. }
  1336. /* Free up the buffer descriptor */
  1337. static void ubwcp_buf_desc_free(struct ubwcp_driver *ubwcp, struct ubwcp_desc *desc)
  1338. {
  1339. int idx = desc->idx;
  1340. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  1341. mutex_lock(&ubwcp->desc_lock);
  1342. desc_list[idx].idx = -1;
  1343. desc_list[idx].ptr = NULL;
  1344. DBG("freed descriptor_id: %d", idx);
  1345. mutex_unlock(&ubwcp->desc_lock);
  1346. }
  1347. /* Allocate next available buffer descriptor. */
  1348. static struct ubwcp_desc *ubwcp_buf_desc_allocate(struct ubwcp_driver *ubwcp)
  1349. {
  1350. int idx;
  1351. struct ubwcp_desc *desc_list = ubwcp->desc_list;
  1352. mutex_lock(&ubwcp->desc_lock);
  1353. for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
  1354. if (desc_list[idx].idx == -1) {
  1355. desc_list[idx].idx = idx;
  1356. desc_list[idx].ptr = ubwcp->buffer_desc_base +
  1357. idx*UBWCP_BUFFER_DESC_OFFSET;
  1358. DBG("allocated descriptor_id: %d", idx);
  1359. mutex_unlock(&ubwcp->desc_lock);
  1360. return &desc_list[idx];
  1361. }
  1362. }
  1363. mutex_unlock(&ubwcp->desc_lock);
  1364. return NULL;
  1365. }
  1366. /**
  1367. * Lock buffer for CPU access. This prepares ubwcp hw to allow
  1368. * CPU access to the compressed buffer. It will perform
  1369. * necessary address translation configuration and cache maintenance ops
  1370. * so that CPU can safely access ubwcp buffer, if this call is
  1371. * successful.
  1372. * Allocate descriptor if not already,
  1373. * perform CMO and then enable range check
  1374. *
  1375. * @param dmabuf : ptr to the dma buf
  1376. * @param direction : direction of access
  1377. *
  1378. * @return int : 0 on success, otherwise error code
  1379. */
  1380. static int ubwcp_lock(struct dma_buf *dmabuf, enum dma_data_direction dir)
  1381. {
  1382. int ret = 0;
  1383. struct ubwcp_buf *buf;
  1384. struct ubwcp_driver *ubwcp;
  1385. FENTRY();
  1386. if (!dmabuf) {
  1387. ERR("NULL dmabuf input ptr");
  1388. return -EINVAL;
  1389. }
  1390. if (!valid_dma_direction(dir)) {
  1391. ERR("invalid direction: %d", dir);
  1392. return -EINVAL;
  1393. }
  1394. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1395. if (!buf) {
  1396. ERR("ubwcp_buf ptr not found");
  1397. return -1;
  1398. }
  1399. mutex_lock(&buf->lock);
  1400. if (!buf->buf_attr_set) {
  1401. ERR("lock() called on buffer, but attr not set");
  1402. goto err;
  1403. }
  1404. if (buf->buf_attr.image_format == UBWCP_LINEAR) {
  1405. ERR("lock() called on linear buffer");
  1406. goto err;
  1407. }
  1408. if (!buf->locked) {
  1409. DBG("first lock on buffer");
  1410. ubwcp = buf->ubwcp;
  1411. /* buf->desc could already be allocated because of perm range xlation */
  1412. if (!buf->desc) {
  1413. /* allocate a buffer descriptor */
  1414. buf->desc = ubwcp_buf_desc_allocate(buf->ubwcp);
  1415. if (!buf->desc) {
  1416. ERR("ubwcp_allocate_buf_desc() failed");
  1417. goto err;
  1418. }
  1419. memcpy(buf->desc->ptr, &buf->mmdata, sizeof(buf->mmdata));
  1420. /* Flushing of updated mmdata:
  1421. * mmdata is iocoherent and ubwcp will get it from CPU cache -
  1422. * *as long as* it has not cached that itself during previous
  1423. * access to the same descriptor.
  1424. *
  1425. * During unlock of previous use of this descriptor,
  1426. * we do hw flush, which will get rid of this mmdata from
  1427. * ubwcp cache.
  1428. *
  1429. * In addition, we also do a hw flush after enable_range_ck().
  1430. * That will also get rid of any speculative fetch of mmdata
  1431. * by the ubwcp hw. At this time, the assumption is that ubwcp
  1432. * will cache mmdata only for active descriptor. But if ubwcp
  1433. * is speculatively fetching mmdata for all descriptors
  1434. * (irrespetive of enabled or not), the flush during lock
  1435. * will be necessary to make sure ubwcp sees updated mmdata
  1436. * that we just updated
  1437. */
  1438. /* program ULA range for this buffer */
  1439. DBG("setting range check: descriptor_id: %d, addr: %p, size: %zx",
  1440. buf->desc->idx, buf->ula_pa, buf->ula_size);
  1441. ubwcp_hw_set_range_check(ubwcp->base, buf->desc->idx, buf->ula_pa,
  1442. buf->ula_size);
  1443. }
  1444. /* enable range check */
  1445. DBG("enabling range check, descriptor_id: %d", buf->desc->idx);
  1446. mutex_lock(&ubwcp->hw_range_ck_lock);
  1447. ubwcp_hw_enable_range_check(ubwcp->base, buf->desc->idx);
  1448. mutex_unlock(&ubwcp->hw_range_ck_lock);
  1449. /* Flush/invalidate UBWCP caches */
  1450. /* Why: cpu could have done a speculative fetch before
  1451. * enable_range_ck() and ubwcp in process of returning "default" data
  1452. * we don't want that stashing of default data pending.
  1453. * we force completion of that and then we also cpu invalidate which
  1454. * will get rid of that line.
  1455. */
  1456. ubwcp_flush(ubwcp);
  1457. /* Flush/invalidate ULA PA from CPU caches
  1458. * TBD: if (dir == READ or BIDIRECTION) //NOT for write
  1459. * -- Confirm with Chris if this can be skipped for write
  1460. */
  1461. dma_sync_single_for_cpu(ubwcp->dev, buf->ula_pa, buf->ula_size, dir);
  1462. buf->lock_dir = dir;
  1463. buf->locked = true;
  1464. } else {
  1465. DBG("buf already locked");
  1466. /* TBD: what if new buffer direction is not same as previous?
  1467. * must update the dir.
  1468. */
  1469. }
  1470. buf->lock_count++;
  1471. DBG("new lock_count: %d", buf->lock_count);
  1472. mutex_unlock(&buf->lock);
  1473. return ret;
  1474. err:
  1475. mutex_unlock(&buf->lock);
  1476. if (!ret)
  1477. ret = -1;
  1478. return ret;
  1479. }
  1480. /* This can be called as a result of external unlock() call or
  1481. * internally if free() is called without unlock().
  1482. */
  1483. static int unlock_internal(struct ubwcp_buf *buf, enum dma_data_direction dir, bool free_buffer)
  1484. {
  1485. int ret = 0;
  1486. struct ubwcp_driver *ubwcp;
  1487. DBG("current lock_count: %d", buf->lock_count);
  1488. if (free_buffer) {
  1489. buf->lock_count = 0;
  1490. DBG("Forced lock_count: %d", buf->lock_count);
  1491. } else {
  1492. buf->lock_count--;
  1493. DBG("new lock_count: %d", buf->lock_count);
  1494. if (buf->lock_count) {
  1495. DBG("more than 1 lock on buffer. waiting until last unlock");
  1496. return 0;
  1497. }
  1498. }
  1499. ubwcp = buf->ubwcp;
  1500. /* Flush/invalidate ULA PA from CPU caches */
  1501. //TBD: if (dir == WRITE or BIDIRECTION)
  1502. dma_sync_single_for_device(ubwcp->dev, buf->ula_pa, buf->ula_size, dir);
  1503. /* disable range check with ubwcp flush */
  1504. DBG("disabling range check");
  1505. //TBD: could combine these 2 locks into a single lock to make it simpler
  1506. mutex_lock(&ubwcp->ubwcp_flush_lock);
  1507. mutex_lock(&ubwcp->hw_range_ck_lock);
  1508. ret = ubwcp_hw_disable_range_check_with_flush(ubwcp->base, buf->desc->idx);
  1509. if (ret)
  1510. ERR("disable_range_check_with_flush() failed: %d", ret);
  1511. mutex_unlock(&ubwcp->hw_range_ck_lock);
  1512. mutex_unlock(&ubwcp->ubwcp_flush_lock);
  1513. /* release descriptor if perm range xlation is not set */
  1514. if (!buf->perm) {
  1515. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1516. buf->desc = NULL;
  1517. }
  1518. buf->locked = false;
  1519. return ret;
  1520. }
  1521. /**
  1522. * Unlock buffer from CPU access. This prepares ubwcp hw to
  1523. * safely allow for device access to the compressed buffer including any
  1524. * necessary cache maintenance ops. It may also free up certain ubwcp
  1525. * resources that could result in error when accessed by CPU in
  1526. * unlocked state.
  1527. *
  1528. * @param dmabuf : ptr to the dma buf
  1529. * @param direction : direction of access
  1530. *
  1531. * @return int : 0 on success, otherwise error code
  1532. */
  1533. static int ubwcp_unlock(struct dma_buf *dmabuf, enum dma_data_direction dir)
  1534. {
  1535. struct ubwcp_buf *buf;
  1536. int ret;
  1537. FENTRY();
  1538. if (!dmabuf) {
  1539. ERR("NULL dmabuf input ptr");
  1540. return -EINVAL;
  1541. }
  1542. if (!valid_dma_direction(dir)) {
  1543. ERR("invalid direction: %d", dir);
  1544. return -EINVAL;
  1545. }
  1546. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1547. if (!buf) {
  1548. ERR("ubwcp_buf not found");
  1549. return -1;
  1550. }
  1551. if (!buf->locked) {
  1552. ERR("unlock() called on buffer which not in locked state");
  1553. return -1;
  1554. }
  1555. error_print_count = 0;
  1556. mutex_lock(&buf->lock);
  1557. ret = unlock_internal(buf, dir, false);
  1558. mutex_unlock(&buf->lock);
  1559. return ret;
  1560. }
  1561. /* Return buffer attributes for the given buffer */
  1562. int ubwcp_get_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
  1563. {
  1564. int ret = 0;
  1565. struct ubwcp_buf *buf;
  1566. FENTRY();
  1567. if (!dmabuf) {
  1568. ERR("NULL dmabuf input ptr");
  1569. return -EINVAL;
  1570. }
  1571. if (!attr) {
  1572. ERR("NULL attr ptr");
  1573. return -EINVAL;
  1574. }
  1575. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1576. if (!buf) {
  1577. ERR("ubwcp_buf ptr not found");
  1578. return -1;
  1579. }
  1580. mutex_lock(&buf->lock);
  1581. if (!buf->buf_attr_set) {
  1582. ERR("buffer attributes not set");
  1583. mutex_unlock(&buf->lock);
  1584. return -1;
  1585. }
  1586. *attr = buf->buf_attr;
  1587. mutex_unlock(&buf->lock);
  1588. return ret;
  1589. }
  1590. EXPORT_SYMBOL(ubwcp_get_buf_attrs);
  1591. /* Set permanent range translation.
  1592. * enable: Descriptor will be reserved for this buffer until disabled,
  1593. * making lock/unlock quicker.
  1594. * disable: Descriptor will not be reserved for this buffer. Instead,
  1595. * descriptor will be allocated and released for each lock/unlock.
  1596. * If currently allocated but not being used, descriptor will be
  1597. * released.
  1598. */
  1599. int ubwcp_set_perm_range_translation(struct dma_buf *dmabuf, bool enable)
  1600. {
  1601. int ret = 0;
  1602. struct ubwcp_buf *buf;
  1603. FENTRY();
  1604. if (!dmabuf) {
  1605. ERR("NULL dmabuf input ptr");
  1606. return -EINVAL;
  1607. }
  1608. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1609. if (!buf) {
  1610. ERR("ubwcp_buf not found");
  1611. return -1;
  1612. }
  1613. /* not implemented */
  1614. if (1) {
  1615. ERR("API not implemented yet");
  1616. return -1;
  1617. }
  1618. /* TBD: make sure we acquire buf lock while setting this so there is
  1619. * no race condition with attr_set/lock/unlock
  1620. */
  1621. buf->perm = enable;
  1622. /* if "disable" and we have allocated a desc and it is not being
  1623. * used currently, release it
  1624. */
  1625. if (!enable && buf->desc && !buf->locked) {
  1626. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1627. buf->desc = NULL;
  1628. /* Flush/invalidate UBWCP caches */
  1629. //TBD: need to do anything?
  1630. }
  1631. return ret;
  1632. }
  1633. EXPORT_SYMBOL(ubwcp_set_perm_range_translation);
  1634. /**
  1635. * Free up ubwcp resources for this buffer.
  1636. *
  1637. * @param dmabuf : ptr to the dma buf
  1638. *
  1639. * @return int : 0 on success, otherwise error code
  1640. */
  1641. static int ubwcp_free_buffer(struct dma_buf *dmabuf)
  1642. {
  1643. int ret = 0;
  1644. struct ubwcp_buf *buf;
  1645. struct ubwcp_driver *ubwcp;
  1646. bool table_empty;
  1647. unsigned long flags;
  1648. FENTRY();
  1649. if (!dmabuf) {
  1650. ERR("NULL dmabuf input ptr");
  1651. return -EINVAL;
  1652. }
  1653. buf = dma_buf_to_ubwcp_buf(dmabuf);
  1654. if (!buf) {
  1655. ERR("ubwcp_buf ptr not found");
  1656. return -1;
  1657. }
  1658. mutex_lock(&buf->lock);
  1659. ubwcp = buf->ubwcp;
  1660. if (buf->locked) {
  1661. DBG("free() called without unlock. unlock()'ing first...");
  1662. ret = unlock_internal(buf, buf->lock_dir, true);
  1663. if (ret)
  1664. ERR("unlock_internal(): failed : %d, but continuing free()", ret);
  1665. }
  1666. /* if we are still holding a desc, release it. this can happen only if perm == true */
  1667. if (buf->desc) {
  1668. WARN_ON(!buf->perm); /* TBD: change to BUG() later...*/
  1669. ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
  1670. buf->desc = NULL;
  1671. }
  1672. if (buf->buf_attr_set)
  1673. reset_buf_attrs(buf);
  1674. mutex_lock(&ubwcp->mem_hotplug_lock);
  1675. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  1676. hash_del(&buf->hnode);
  1677. table_empty = hash_empty(ubwcp->buf_table);
  1678. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  1679. kfree(buf);
  1680. /* If this is the last buffer being freed, power off ubwcp */
  1681. if (table_empty) {
  1682. DBG("last buffer: ~~~~~~~~~~~");
  1683. /* TBD: If everything is working fine, ubwcp_flush() should not
  1684. * be needed here. Each buffer free logic should be taking
  1685. * care of flush. Just a note for now. Might need to add the
  1686. * flush here for debug purpose.
  1687. */
  1688. DBG("set_direct_map_range_uncached() for ULA PA pool st:0x%lx num pages:%lu",
  1689. ubwcp->ula_pool_base, ubwcp->ula_pool_size >> PAGE_SHIFT);
  1690. ret = set_direct_map_range_uncached((unsigned long)phys_to_virt(
  1691. ubwcp->ula_pool_base), ubwcp->ula_pool_size >> PAGE_SHIFT);
  1692. if (ret) {
  1693. ERR("set_direct_map_range_uncached failed st:0x%lx num pages:%lu err: %d",
  1694. ubwcp->ula_pool_base,
  1695. ubwcp->ula_pool_size >> PAGE_SHIFT, ret);
  1696. goto err_remove_mem;
  1697. } else {
  1698. DBG("DONE: calling set_direct_map_range_uncached() for ULA PA pool");
  1699. }
  1700. DBG("Calling dma_sync_single_for_cpu() for ULA PA pool");
  1701. dma_sync_single_for_cpu(ubwcp->dev, ubwcp->ula_pool_base, ubwcp->ula_pool_size,
  1702. DMA_BIDIRECTIONAL);
  1703. DBG("Calling offline_and_remove_memory() for ULA PA pool");
  1704. ret = offline_and_remove_memory(ubwcp->ula_pool_base,
  1705. ubwcp->ula_pool_size);
  1706. if (ret) {
  1707. ERR("offline_and_remove_memory failed st:0x%lx sz:0x%lx err: %d",
  1708. ubwcp->ula_pool_base,
  1709. ubwcp->ula_pool_size, ret);
  1710. goto err_remove_mem;
  1711. } else {
  1712. DBG("DONE: calling offline_and_remove_memory() for ULA PA pool");
  1713. }
  1714. DBG("Calling power OFF ...");
  1715. ubwcp_power(ubwcp, false);
  1716. }
  1717. mutex_unlock(&ubwcp->mem_hotplug_lock);
  1718. return ret;
  1719. err_remove_mem:
  1720. mutex_unlock(&ubwcp->mem_hotplug_lock);
  1721. if (!ret)
  1722. ret = -1;
  1723. DBG("returning error: %d", ret);
  1724. return ret;
  1725. }
  1726. /* file open: TBD: increment ref count? */
  1727. static int ubwcp_open(struct inode *i, struct file *f)
  1728. {
  1729. return 0;
  1730. }
  1731. /* file open: TBD: decrement ref count? */
  1732. static int ubwcp_close(struct inode *i, struct file *f)
  1733. {
  1734. return 0;
  1735. }
  1736. /* handle IOCTLs */
  1737. static long ubwcp_ioctl(struct file *file, unsigned int ioctl_num, unsigned long ioctl_param)
  1738. {
  1739. struct ubwcp_ioctl_buffer_attrs buf_attr_ioctl;
  1740. struct ubwcp_ioctl_hw_version hw_ver;
  1741. switch (ioctl_num) {
  1742. case UBWCP_IOCTL_SET_BUF_ATTR:
  1743. if (copy_from_user(&buf_attr_ioctl, (const void __user *) ioctl_param,
  1744. sizeof(buf_attr_ioctl))) {
  1745. ERR("ERROR: copy_from_user() failed");
  1746. return -EFAULT;
  1747. }
  1748. DBG("IOCTL : SET_BUF_ATTR: fd = %d", buf_attr_ioctl.fd);
  1749. return ubwcp_set_buf_attrs_ioctl(&buf_attr_ioctl);
  1750. case UBWCP_IOCTL_GET_HW_VER:
  1751. DBG("IOCTL : GET_HW_VER");
  1752. ubwcp_get_hw_version(&hw_ver);
  1753. if (copy_to_user((void __user *)ioctl_param, &hw_ver, sizeof(hw_ver))) {
  1754. ERR("ERROR: copy_to_user() failed");
  1755. return -EFAULT;
  1756. }
  1757. break;
  1758. default:
  1759. ERR("Invalid ioctl_num = %d", ioctl_num);
  1760. return -EINVAL;
  1761. }
  1762. return 0;
  1763. }
  1764. static const struct file_operations ubwcp_fops = {
  1765. .owner = THIS_MODULE,
  1766. .open = ubwcp_open,
  1767. .release = ubwcp_close,
  1768. .unlocked_ioctl = ubwcp_ioctl,
  1769. };
  1770. static int ubwcp_debugfs_init(struct ubwcp_driver *ubwcp)
  1771. {
  1772. struct dentry *debugfs_root;
  1773. debugfs_root = debugfs_create_dir("ubwcp", NULL);
  1774. if (!debugfs_root) {
  1775. pr_warn("Failed to create debugfs for ubwcp\n");
  1776. return -1;
  1777. }
  1778. debugfs_create_u32("debug_trace_enable", 0644, debugfs_root, &ubwcp_debug_trace_enable);
  1779. ubwcp->debugfs_root = debugfs_root;
  1780. return 0;
  1781. }
  1782. static void ubwcp_debugfs_deinit(struct ubwcp_driver *ubwcp)
  1783. {
  1784. debugfs_remove_recursive(ubwcp->debugfs_root);
  1785. }
  1786. /* ubwcp char device initialization */
  1787. static int ubwcp_cdev_init(struct ubwcp_driver *ubwcp)
  1788. {
  1789. int ret;
  1790. dev_t devt;
  1791. struct class *dev_class;
  1792. struct device *dev_sys;
  1793. /* allocate major device number (/proc/devices -> major_num ubwcp) */
  1794. ret = alloc_chrdev_region(&devt, 0, UBWCP_NUM_DEVICES, UBWCP_DEVICE_NAME);
  1795. if (ret) {
  1796. ERR("alloc_chrdev_region() failed: %d", ret);
  1797. return ret;
  1798. }
  1799. /* create device class (/sys/class/ubwcp_class) */
  1800. dev_class = class_create(THIS_MODULE, "ubwcp_class");
  1801. if (IS_ERR(dev_class)) {
  1802. ERR("class_create() failed");
  1803. return -1;
  1804. }
  1805. /* Create device and register with sysfs
  1806. * (/sys/class/ubwcp_class/ubwcp/... -> dev/power/subsystem/uevent)
  1807. */
  1808. dev_sys = device_create(dev_class, NULL, devt, NULL,
  1809. UBWCP_DEVICE_NAME);
  1810. if (IS_ERR(dev_sys)) {
  1811. ERR("device_create() failed");
  1812. return -1;
  1813. }
  1814. /* register file operations and get cdev */
  1815. cdev_init(&ubwcp->cdev, &ubwcp_fops);
  1816. /* associate cdev and device major/minor with file system
  1817. * can do file ops on /dev/ubwcp after this
  1818. */
  1819. ret = cdev_add(&ubwcp->cdev, devt, 1);
  1820. if (ret) {
  1821. ERR("cdev_add() failed");
  1822. return -1;
  1823. }
  1824. ubwcp->devt = devt;
  1825. ubwcp->dev_class = dev_class;
  1826. ubwcp->dev_sys = dev_sys;
  1827. return 0;
  1828. }
  1829. static void ubwcp_cdev_deinit(struct ubwcp_driver *ubwcp)
  1830. {
  1831. device_destroy(ubwcp->dev_class, ubwcp->devt);
  1832. class_destroy(ubwcp->dev_class);
  1833. cdev_del(&ubwcp->cdev);
  1834. unregister_chrdev_region(ubwcp->devt, UBWCP_NUM_DEVICES);
  1835. }
  1836. struct handler_node {
  1837. struct list_head list;
  1838. u32 client_id;
  1839. ubwcp_error_handler_t handler;
  1840. void *data;
  1841. };
  1842. int ubwcp_register_error_handler(u32 client_id, ubwcp_error_handler_t handler,
  1843. void *data)
  1844. {
  1845. struct handler_node *node;
  1846. unsigned long flags;
  1847. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1848. if (!ubwcp)
  1849. return -EINVAL;
  1850. if (client_id != -1)
  1851. return -EINVAL;
  1852. if (!handler)
  1853. return -EINVAL;
  1854. node = kzalloc(sizeof(*node), GFP_KERNEL);
  1855. if (!node)
  1856. return -ENOMEM;
  1857. node->client_id = client_id;
  1858. node->handler = handler;
  1859. node->data = data;
  1860. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  1861. list_add_tail(&node->list, &ubwcp->err_handler_list);
  1862. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  1863. return 0;
  1864. }
  1865. EXPORT_SYMBOL(ubwcp_register_error_handler);
  1866. static void ubwcp_notify_error_handlers(struct unwcp_err_info *err)
  1867. {
  1868. struct handler_node *node;
  1869. unsigned long flags;
  1870. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1871. if (!ubwcp)
  1872. return;
  1873. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  1874. list_for_each_entry(node, &ubwcp->err_handler_list, list)
  1875. node->handler(err, node->data);
  1876. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  1877. }
  1878. int ubwcp_unregister_error_handler(u32 client_id)
  1879. {
  1880. int ret = -EINVAL;
  1881. struct handler_node *node;
  1882. unsigned long flags;
  1883. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1884. if (!ubwcp)
  1885. return -EINVAL;
  1886. spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
  1887. list_for_each_entry(node, &ubwcp->err_handler_list, list)
  1888. if (node->client_id == client_id) {
  1889. list_del(&node->list);
  1890. kfree(node);
  1891. ret = 0;
  1892. break;
  1893. }
  1894. spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
  1895. return ret;
  1896. }
  1897. EXPORT_SYMBOL(ubwcp_unregister_error_handler);
  1898. /* get ubwcp_buf corresponding to the ULA PA*/
  1899. static struct dma_buf *get_dma_buf_from_ulapa(phys_addr_t addr)
  1900. {
  1901. struct ubwcp_buf *buf = NULL;
  1902. struct dma_buf *ret_buf = NULL;
  1903. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1904. unsigned long flags;
  1905. u32 i;
  1906. if (!ubwcp)
  1907. return NULL;
  1908. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  1909. hash_for_each(ubwcp->buf_table, i, buf, hnode) {
  1910. if (buf->ula_pa <= addr && addr < buf->ula_pa + buf->ula_size) {
  1911. ret_buf = buf->dma_buf;
  1912. break;
  1913. }
  1914. }
  1915. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  1916. return ret_buf;
  1917. }
  1918. /* get ubwcp_buf corresponding to the IOVA*/
  1919. static struct dma_buf *get_dma_buf_from_iova(unsigned long addr)
  1920. {
  1921. struct ubwcp_buf *buf = NULL;
  1922. struct dma_buf *ret_buf = NULL;
  1923. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1924. unsigned long flags;
  1925. u32 i;
  1926. if (!ubwcp)
  1927. return NULL;
  1928. spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
  1929. hash_for_each(ubwcp->buf_table, i, buf, hnode) {
  1930. unsigned long iova_base = sg_dma_address(buf->sgt->sgl);
  1931. unsigned int iova_size = sg_dma_len(buf->sgt->sgl);
  1932. if (iova_base <= addr && addr < iova_base + iova_size) {
  1933. ret_buf = buf->dma_buf;
  1934. break;
  1935. }
  1936. }
  1937. spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
  1938. return ret_buf;
  1939. }
  1940. #define ERR_PRINT_COUNT_MAX 21
  1941. /* TBD: use proper rate limit for debug prints */
  1942. int ubwcp_iommu_fault_handler(struct iommu_domain *domain, struct device *dev,
  1943. unsigned long iova, int flags, void *data)
  1944. {
  1945. int ret = 0;
  1946. struct unwcp_err_info err;
  1947. struct ubwcp_driver *ubwcp = ubwcp_get_driver();
  1948. struct device *cb_dev = (struct device *)data;
  1949. if (!ubwcp) {
  1950. ret = -EINVAL;
  1951. goto err;
  1952. }
  1953. error_print_count++;
  1954. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  1955. err.err_code = UBWCP_SMMU_FAULT;
  1956. if (cb_dev == ubwcp->dev_desc_cb)
  1957. err.smmu_err.iommu_dev_id = UBWCP_DESC_CB_ID;
  1958. else if (cb_dev == ubwcp->dev_buf_cb)
  1959. err.smmu_err.iommu_dev_id = UBWCP_BUF_CB_ID;
  1960. else
  1961. err.smmu_err.iommu_dev_id = UBWCP_UNKNOWN_CB_ID;
  1962. ERR("smmu fault error: iommu_dev_id:%d iova 0x%llx flags:0x%x",
  1963. err.smmu_err.iommu_dev_id, iova, flags);
  1964. err.smmu_err.dmabuf = get_dma_buf_from_iova(iova);
  1965. err.smmu_err.iova = iova;
  1966. err.smmu_err.iommu_fault_flags = flags;
  1967. ubwcp_notify_error_handlers(&err);
  1968. }
  1969. err:
  1970. return ret;
  1971. }
  1972. irqreturn_t ubwcp_irq_handler(int irq, void *ptr)
  1973. {
  1974. struct ubwcp_driver *ubwcp;
  1975. void __iomem *base;
  1976. u64 src;
  1977. phys_addr_t addr;
  1978. struct unwcp_err_info err;
  1979. error_print_count++;
  1980. ubwcp = (struct ubwcp_driver *) ptr;
  1981. base = ubwcp->base;
  1982. if (irq == ubwcp->irq_range_ck_rd) {
  1983. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  1984. src = ubwcp_hw_interrupt_src_address(base, 0);
  1985. addr = src << 6;
  1986. ERR("check range read error: src: 0x%llx", addr);
  1987. err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
  1988. err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
  1989. err.translation_err.ula_pa = addr;
  1990. err.translation_err.read = true;
  1991. ubwcp_notify_error_handlers(&err);
  1992. }
  1993. ubwcp_hw_interrupt_clear(ubwcp->base, 0);
  1994. } else if (irq == ubwcp->irq_range_ck_wr) {
  1995. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  1996. src = ubwcp_hw_interrupt_src_address(base, 1);
  1997. addr = src << 6;
  1998. ERR("check range write error: src: 0x%llx", addr);
  1999. err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
  2000. err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2001. err.translation_err.ula_pa = addr;
  2002. err.translation_err.read = false;
  2003. ubwcp_notify_error_handlers(&err);
  2004. }
  2005. ubwcp_hw_interrupt_clear(ubwcp->base, 1);
  2006. } else if (irq == ubwcp->irq_encode) {
  2007. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  2008. src = ubwcp_hw_interrupt_src_address(base, 3);
  2009. addr = src << 6;
  2010. ERR("encode error: src: 0x%llx", addr);
  2011. err.err_code = UBWCP_ENCODE_ERROR;
  2012. err.enc_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2013. err.enc_err.ula_pa = addr;
  2014. ubwcp_notify_error_handlers(&err);
  2015. }
  2016. ubwcp_hw_interrupt_clear(ubwcp->base, 3); //TBD: encode is bit-3 instead of bit-2
  2017. } else if (irq == ubwcp->irq_decode) {
  2018. if (error_print_count < ERR_PRINT_COUNT_MAX) {
  2019. src = ubwcp_hw_interrupt_src_address(base, 2);
  2020. addr = src << 6;
  2021. ERR("decode error: src: 0x%llx", addr);
  2022. err.err_code = UBWCP_DECODE_ERROR;
  2023. err.dec_err.dmabuf = get_dma_buf_from_ulapa(addr);
  2024. err.dec_err.ula_pa = addr;
  2025. ubwcp_notify_error_handlers(&err);
  2026. }
  2027. ubwcp_hw_interrupt_clear(ubwcp->base, 2); //TBD: decode is bit-2 instead of bit-3
  2028. } else {
  2029. ERR("unknown irq: %d", irq);
  2030. return IRQ_NONE;
  2031. }
  2032. return IRQ_HANDLED;
  2033. }
  2034. static int ubwcp_interrupt_register(struct platform_device *pdev, struct ubwcp_driver *ubwcp)
  2035. {
  2036. int ret = 0;
  2037. struct device *dev = &pdev->dev;
  2038. FENTRY();
  2039. ubwcp->irq_range_ck_rd = platform_get_irq(pdev, 0);
  2040. if (ubwcp->irq_range_ck_rd < 0)
  2041. return ubwcp->irq_range_ck_rd;
  2042. ubwcp->irq_range_ck_wr = platform_get_irq(pdev, 1);
  2043. if (ubwcp->irq_range_ck_wr < 0)
  2044. return ubwcp->irq_range_ck_wr;
  2045. ubwcp->irq_encode = platform_get_irq(pdev, 2);
  2046. if (ubwcp->irq_encode < 0)
  2047. return ubwcp->irq_encode;
  2048. ubwcp->irq_decode = platform_get_irq(pdev, 3);
  2049. if (ubwcp->irq_decode < 0)
  2050. return ubwcp->irq_decode;
  2051. DBG("got irqs: %d %d %d %d", ubwcp->irq_range_ck_rd,
  2052. ubwcp->irq_range_ck_wr,
  2053. ubwcp->irq_encode,
  2054. ubwcp->irq_decode);
  2055. ret = devm_request_irq(dev, ubwcp->irq_range_ck_rd, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2056. if (ret) {
  2057. ERR("request_irq() failed. irq: %d ret: %d",
  2058. ubwcp->irq_range_ck_rd, ret);
  2059. return ret;
  2060. }
  2061. ret = devm_request_irq(dev, ubwcp->irq_range_ck_wr, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2062. if (ret) {
  2063. ERR("request_irq() failed. irq: %d ret: %d",
  2064. ubwcp->irq_range_ck_wr, ret);
  2065. return ret;
  2066. }
  2067. ret = devm_request_irq(dev, ubwcp->irq_encode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2068. if (ret) {
  2069. ERR("request_irq() failed. irq: %d ret: %d",
  2070. ubwcp->irq_encode, ret);
  2071. return ret;
  2072. }
  2073. ret = devm_request_irq(dev, ubwcp->irq_decode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
  2074. if (ret) {
  2075. ERR("request_irq() failed. irq: %d ret: %d",
  2076. ubwcp->irq_decode, ret);
  2077. return ret;
  2078. }
  2079. return ret;
  2080. }
  2081. /* ubwcp device probe */
  2082. static int qcom_ubwcp_probe(struct platform_device *pdev)
  2083. {
  2084. int ret = 0;
  2085. struct ubwcp_driver *ubwcp;
  2086. struct device *ubwcp_dev = &pdev->dev;
  2087. FENTRY();
  2088. ubwcp = devm_kzalloc(ubwcp_dev, sizeof(*ubwcp), GFP_KERNEL);
  2089. if (!ubwcp) {
  2090. ERR("devm_kzalloc() failed");
  2091. return -ENOMEM;
  2092. }
  2093. ubwcp->dev = &pdev->dev;
  2094. ret = dma_set_mask_and_coherent(ubwcp->dev, DMA_BIT_MASK(64));
  2095. #ifdef UBWCP_USE_SMC
  2096. {
  2097. struct resource res;
  2098. of_address_to_resource(ubwcp_dev->of_node, 0, &res);
  2099. ubwcp->base = (void __iomem *) res.start;
  2100. DBG("Using SMC calls. base: %p", ubwcp->base);
  2101. }
  2102. #else
  2103. ubwcp->base = devm_platform_ioremap_resource(pdev, 0);
  2104. if (IS_ERR(ubwcp->base)) {
  2105. ERR("devm ioremap() failed: %d", PTR_ERR(ubwcp->base));
  2106. return PTR_ERR(ubwcp->base);
  2107. }
  2108. DBG("ubwcp->base: %p", ubwcp->base);
  2109. #endif
  2110. ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 0, &ubwcp->ula_pool_base);
  2111. if (ret) {
  2112. ERR("failed reading ula_range (base): %d", ret);
  2113. return ret;
  2114. }
  2115. DBG("ubwcp: ula_range: base = 0x%lx", ubwcp->ula_pool_base);
  2116. ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 1, &ubwcp->ula_pool_size);
  2117. if (ret) {
  2118. ERR("failed reading ula_range (size): %d", ret);
  2119. return ret;
  2120. }
  2121. DBG("ubwcp: ula_range: size = 0x%lx", ubwcp->ula_pool_size);
  2122. /*TBD: remove later. reducing size for quick testing...*/
  2123. ubwcp->ula_pool_size = 0x20000000; //500MB instead of 8GB
  2124. INIT_LIST_HEAD(&ubwcp->err_handler_list);
  2125. mutex_init(&ubwcp->desc_lock);
  2126. spin_lock_init(&ubwcp->buf_table_lock);
  2127. mutex_init(&ubwcp->mem_hotplug_lock);
  2128. mutex_init(&ubwcp->ula_lock);
  2129. mutex_init(&ubwcp->ubwcp_flush_lock);
  2130. mutex_init(&ubwcp->hw_range_ck_lock);
  2131. spin_lock_init(&ubwcp->err_handler_list_lock);
  2132. if (ubwcp_interrupt_register(pdev, ubwcp))
  2133. return -1;
  2134. /* Regulator */
  2135. ubwcp->vdd = devm_regulator_get(ubwcp_dev, "vdd");
  2136. if (IS_ERR_OR_NULL(ubwcp->vdd)) {
  2137. ret = PTR_ERR(ubwcp->vdd);
  2138. ERR("devm_regulator_get() failed: %d", ret);
  2139. return -1;
  2140. }
  2141. ret = ubwcp_init_clocks(ubwcp, ubwcp_dev);
  2142. if (ret) {
  2143. ERR("failed to initialize ubwcp clocks err: %d", ret);
  2144. return ret;
  2145. }
  2146. if (ubwcp_power(ubwcp, true))
  2147. return -1;
  2148. if (ubwcp_cdev_init(ubwcp))
  2149. return -1;
  2150. if (ubwcp_debugfs_init(ubwcp))
  2151. return -1;
  2152. /* create ULA pool */
  2153. ubwcp->ula_pool = gen_pool_create(12, -1);
  2154. if (!ubwcp->ula_pool) {
  2155. ERR("failed gen_pool_create()");
  2156. ret = -1;
  2157. goto err_pool_create;
  2158. }
  2159. ret = gen_pool_add(ubwcp->ula_pool, ubwcp->ula_pool_base, ubwcp->ula_pool_size, -1);
  2160. if (ret) {
  2161. ERR("failed gen_pool_add(): %d", ret);
  2162. ret = -1;
  2163. goto err_pool_add;
  2164. }
  2165. /* register the default config mmap function. */
  2166. ubwcp->mmap_config_fptr = msm_ubwcp_dma_buf_configure_mmap;
  2167. hash_init(ubwcp->buf_table);
  2168. ubwcp_buf_desc_list_init(ubwcp);
  2169. image_format_init(ubwcp);
  2170. /* one time hw init */
  2171. ubwcp_hw_one_time_init(ubwcp->base);
  2172. ubwcp_hw_version(ubwcp->base, &ubwcp->hw_ver_major, &ubwcp->hw_ver_minor);
  2173. pr_err("ubwcp: hw version: major %d, minor %d\n", ubwcp->hw_ver_major, ubwcp->hw_ver_minor);
  2174. if (ubwcp->hw_ver_major == 0) {
  2175. ERR("Failed to read HW version");
  2176. ret = -1;
  2177. goto err_pool_add;
  2178. }
  2179. /* set pdev->dev->driver_data = ubwcp */
  2180. platform_set_drvdata(pdev, ubwcp);
  2181. /* enable all 4 interrupts */
  2182. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, true);
  2183. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, true);
  2184. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, true);
  2185. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, true);
  2186. /* Turn OFF until buffers are allocated */
  2187. if (ubwcp_power(ubwcp, false)) {
  2188. ret = -1;
  2189. goto err_power_off;
  2190. }
  2191. ret = msm_ubwcp_set_ops(ubwcp_init_buffer, ubwcp_free_buffer, ubwcp_lock, ubwcp_unlock);
  2192. if (ret) {
  2193. ERR("msm_ubwcp_set_ops() failed: %d, but IGNORED", ret);
  2194. /* TBD: ignore return error during testing phase.
  2195. * This allows us to rmmod/insmod for faster dev cycle.
  2196. * In final version: return error and de-register driver if set_ops fails.
  2197. */
  2198. ret = 0;
  2199. //goto err_power_off;
  2200. } else {
  2201. DBG("msm_ubwcp_set_ops(): success"); }
  2202. me = ubwcp;
  2203. return ret;
  2204. err_power_off:
  2205. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2206. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2207. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2208. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2209. err_pool_add:
  2210. gen_pool_destroy(ubwcp->ula_pool);
  2211. err_pool_create:
  2212. ubwcp_cdev_deinit(ubwcp);
  2213. return ret;
  2214. }
  2215. /* buffer context bank device probe */
  2216. static int ubwcp_probe_cb_buf(struct platform_device *pdev)
  2217. {
  2218. struct ubwcp_driver *ubwcp;
  2219. struct iommu_domain *domain = NULL;
  2220. FENTRY();
  2221. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2222. if (!ubwcp) {
  2223. ERR("failed to get ubwcp ptr");
  2224. return -EINVAL;
  2225. }
  2226. /* save the buffer cb device */
  2227. ubwcp->dev_buf_cb = &pdev->dev;
  2228. domain = iommu_get_domain_for_dev(ubwcp->dev_buf_cb);
  2229. if (domain)
  2230. iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_buf_cb);
  2231. return 0;
  2232. }
  2233. /* descriptor context bank device probe */
  2234. static int ubwcp_probe_cb_desc(struct platform_device *pdev)
  2235. {
  2236. int ret = 0;
  2237. struct ubwcp_driver *ubwcp;
  2238. struct iommu_domain *domain = NULL;
  2239. FENTRY();
  2240. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2241. if (!ubwcp) {
  2242. ERR("failed to get ubwcp ptr");
  2243. return -EINVAL;
  2244. }
  2245. ubwcp->buffer_desc_size = UBWCP_BUFFER_DESC_OFFSET *
  2246. UBWCP_BUFFER_DESC_COUNT;
  2247. ubwcp->dev_desc_cb = &pdev->dev;
  2248. dma_set_max_seg_size(ubwcp->dev_desc_cb, DMA_BIT_MASK(32));
  2249. dma_set_seg_boundary(ubwcp->dev_desc_cb, (unsigned long)DMA_BIT_MASK(64));
  2250. /* Allocate buffer descriptors. UBWCP is iocoherent device.
  2251. * Thus we don't need to flush after updates to buffer descriptors.
  2252. */
  2253. ubwcp->buffer_desc_base = dma_alloc_coherent(ubwcp->dev_desc_cb,
  2254. ubwcp->buffer_desc_size,
  2255. &ubwcp->buffer_desc_dma_handle,
  2256. GFP_KERNEL);
  2257. if (!ubwcp->buffer_desc_base) {
  2258. ERR("failed to allocate desc buffer");
  2259. return -ENOMEM;
  2260. }
  2261. DBG("desc_base = %p size = %zu", ubwcp->buffer_desc_base,
  2262. ubwcp->buffer_desc_size);
  2263. ret = ubwcp_power(ubwcp, true);
  2264. if (ret) {
  2265. ERR("failed to power on");
  2266. goto err;
  2267. }
  2268. ubwcp_hw_set_buf_desc(ubwcp->base, (u64) ubwcp->buffer_desc_dma_handle,
  2269. UBWCP_BUFFER_DESC_OFFSET);
  2270. ret = ubwcp_power(ubwcp, false);
  2271. if (ret) {
  2272. ERR("failed to power off");
  2273. goto err;
  2274. }
  2275. domain = iommu_get_domain_for_dev(ubwcp->dev_desc_cb);
  2276. if (domain)
  2277. iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_desc_cb);
  2278. return ret;
  2279. err:
  2280. dma_free_coherent(ubwcp->dev_desc_cb,
  2281. ubwcp->buffer_desc_size,
  2282. ubwcp->buffer_desc_base,
  2283. ubwcp->buffer_desc_dma_handle);
  2284. ubwcp->buffer_desc_base = NULL;
  2285. ubwcp->buffer_desc_dma_handle = 0;
  2286. ubwcp->dev_desc_cb = NULL;
  2287. return -1;
  2288. }
  2289. /* buffer context bank device remove */
  2290. static int ubwcp_remove_cb_buf(struct platform_device *pdev)
  2291. {
  2292. struct ubwcp_driver *ubwcp;
  2293. FENTRY();
  2294. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2295. if (!ubwcp) {
  2296. ERR("failed to get ubwcp ptr");
  2297. return -EINVAL;
  2298. }
  2299. /* remove buf_cb reference */
  2300. ubwcp->dev_buf_cb = NULL;
  2301. return 0;
  2302. }
  2303. /* descriptor context bank device remove */
  2304. static int ubwcp_remove_cb_desc(struct platform_device *pdev)
  2305. {
  2306. struct ubwcp_driver *ubwcp;
  2307. FENTRY();
  2308. ubwcp = dev_get_drvdata(pdev->dev.parent);
  2309. if (!ubwcp) {
  2310. ERR("failed to get ubwcp ptr");
  2311. return -EINVAL;
  2312. }
  2313. if (!ubwcp->dev_desc_cb) {
  2314. ERR("ubwcp->dev_desc_cb == NULL");
  2315. return -1;
  2316. }
  2317. ubwcp_power(ubwcp, true);
  2318. ubwcp_hw_set_buf_desc(ubwcp->base, 0x0, 0x0);
  2319. ubwcp_power(ubwcp, false);
  2320. dma_free_coherent(ubwcp->dev_desc_cb,
  2321. ubwcp->buffer_desc_size,
  2322. ubwcp->buffer_desc_base,
  2323. ubwcp->buffer_desc_dma_handle);
  2324. ubwcp->buffer_desc_base = NULL;
  2325. ubwcp->buffer_desc_dma_handle = 0;
  2326. return 0;
  2327. }
  2328. /* ubwcp device remove */
  2329. static int qcom_ubwcp_remove(struct platform_device *pdev)
  2330. {
  2331. size_t avail;
  2332. size_t psize;
  2333. struct ubwcp_driver *ubwcp;
  2334. FENTRY();
  2335. /* get pdev->dev->driver_data = ubwcp */
  2336. ubwcp = platform_get_drvdata(pdev);
  2337. if (!ubwcp) {
  2338. ERR("ubwcp == NULL");
  2339. return -1;
  2340. }
  2341. ubwcp_power(ubwcp, true);
  2342. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
  2343. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
  2344. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
  2345. ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
  2346. ubwcp_power(ubwcp, false);
  2347. /* before destroying, make sure pool is empty. otherwise pool_destroy() panics.
  2348. * TBD: remove this check for production code and let it panic
  2349. */
  2350. avail = gen_pool_avail(ubwcp->ula_pool);
  2351. psize = gen_pool_size(ubwcp->ula_pool);
  2352. if (psize != avail) {
  2353. ERR("gen_pool is not empty! avail: %zx size: %zx", avail, psize);
  2354. ERR("skipping pool destroy....cause it will PANIC. Fix this!!!!");
  2355. WARN(1, "Fix this!");
  2356. } else {
  2357. gen_pool_destroy(ubwcp->ula_pool);
  2358. }
  2359. ubwcp_debugfs_deinit(ubwcp);
  2360. ubwcp_cdev_deinit(ubwcp);
  2361. return 0;
  2362. }
  2363. /* top level ubwcp device probe function */
  2364. static int ubwcp_probe(struct platform_device *pdev)
  2365. {
  2366. const char *compatible = "";
  2367. FENTRY();
  2368. if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
  2369. return qcom_ubwcp_probe(pdev);
  2370. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
  2371. return ubwcp_probe_cb_desc(pdev);
  2372. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
  2373. return ubwcp_probe_cb_buf(pdev);
  2374. of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
  2375. ERR("unknown device: %s", compatible);
  2376. WARN_ON(1);
  2377. return -EINVAL;
  2378. }
  2379. /* top level ubwcp device remove function */
  2380. static int ubwcp_remove(struct platform_device *pdev)
  2381. {
  2382. const char *compatible = "";
  2383. FENTRY();
  2384. /* TBD: what if buffers are still allocated? locked? etc.
  2385. * also should turn off power?
  2386. */
  2387. if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
  2388. return qcom_ubwcp_remove(pdev);
  2389. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
  2390. return ubwcp_remove_cb_desc(pdev);
  2391. else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
  2392. return ubwcp_remove_cb_buf(pdev);
  2393. of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
  2394. ERR("unknown device: %s", compatible);
  2395. WARN_ON(1);
  2396. return -EINVAL;
  2397. }
  2398. static const struct of_device_id ubwcp_dt_match[] = {
  2399. {.compatible = "qcom,ubwcp"},
  2400. {.compatible = "qcom,ubwcp-context-bank-desc"},
  2401. {.compatible = "qcom,ubwcp-context-bank-buf"},
  2402. {}
  2403. };
  2404. struct platform_driver ubwcp_platform_driver = {
  2405. .probe = ubwcp_probe,
  2406. .remove = ubwcp_remove,
  2407. .driver = {
  2408. .name = "qcom,ubwcp",
  2409. .of_match_table = ubwcp_dt_match,
  2410. },
  2411. };
  2412. int ubwcp_init(void)
  2413. {
  2414. int ret = 0;
  2415. DBG("+++++++++++");
  2416. ret = platform_driver_register(&ubwcp_platform_driver);
  2417. if (ret)
  2418. ERR("platform_driver_register() failed: %d", ret);
  2419. return ret;
  2420. }
  2421. void ubwcp_exit(void)
  2422. {
  2423. platform_driver_unregister(&ubwcp_platform_driver);
  2424. DBG("-----------");
  2425. }
  2426. module_init(ubwcp_init);
  2427. module_exit(ubwcp_exit);
  2428. MODULE_LICENSE("GPL");