123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540 |
- // SPDX-License-Identifier: GPL-2.0-only
- /*
- * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
- */
- #define pr_fmt(fmt) "%s: %s(): " fmt, KBUILD_MODNAME, __func__
- #include <linux/module.h>
- #include <linux/kernel.h>
- #include <linux/dma-buf.h>
- #include <linux/slab.h>
- #include <linux/cdev.h>
- #include <linux/hashtable.h>
- #include <linux/scatterlist.h>
- #include <linux/types.h>
- #include <linux/platform_device.h>
- #include <linux/of.h>
- #include <linux/of_platform.h>
- #include <linux/of_address.h>
- #include <linux/genalloc.h>
- #include <linux/interrupt.h>
- #include <linux/regulator/consumer.h>
- #include <linux/numa.h>
- #include <linux/memory_hotplug.h>
- #include <asm/page.h>
- #include <linux/delay.h>
- #include <linux/ubwcp_dma_heap.h>
- #include <linux/debugfs.h>
- #include <linux/clk.h>
- #include <linux/iommu.h>
- #include <linux/set_memory.h>
- #include <linux/range.h>
- #include <linux/qcom_scm.h>
- MODULE_IMPORT_NS(DMA_BUF);
- #include "include/kernel/ubwcp.h"
- #include "ubwcp_hw.h"
- #include "include/uapi/ubwcp_ioctl.h"
- #define CREATE_TRACE_POINTS
- #include "ubwcp_trace.h"
- #define UBWCP_NUM_DEVICES 1
- #define UBWCP_DEVICE_NAME "ubwcp"
- #define UBWCP_BUFFER_DESC_OFFSET 64
- #define UBWCP_BUFFER_DESC_COUNT 256
- #define CACHE_ADDR(x) ((x) >> 6)
- #define PAGE_ADDR(x) ((x) >> 12)
- #define UBWCP_ALIGN(_x, _y) ((((_x) + (_y) - 1)/(_y))*(_y))
- #define DBG_BUF_ATTR(fmt, args...) do { if (unlikely(ubwcp_debug_trace_enable)) \
- pr_err(fmt "\n", ##args); \
- } while (0)
- #define DBG(fmt, args...) do { if (unlikely(ubwcp_debug_trace_enable)) \
- pr_err(fmt "\n", ##args); \
- } while (0)
- #define ERR(fmt, args...) pr_err_ratelimited("%d: ~~~ERROR~~~: " fmt "\n", __LINE__, ##args)
- #define META_DATA_PITCH_ALIGN 64
- #define META_DATA_HEIGHT_ALIGN 16
- #define META_DATA_SIZE_ALIGN 4096
- #define PIXEL_DATA_SIZE_ALIGN 4096
- #define UBWCP_SYNC_GRANULE 0x4000000L /* 64 MB */
- /* Max values for attributes */
- #define MAX_ATTR_WIDTH (10*1024)
- #define MAX_ATTR_HEIGHT (10*1024)
- #define MAX_ATTR_STRIDE (64*1024)
- #define MAX_ATTR_PLANAR_PAD 4096
- #define MAX_ATTR_SCANLN_HT_DELTA (32*1024)
- enum ula_remove_mem_status {
- ULA_REMOVE_MEM_SUCCESS = 0,
- ULA_REMOVE_MEM_ABORTED = 1
- };
- struct ubwcp_desc {
- int idx;
- void *ptr;
- };
- struct tile_dimension {
- u16 width;
- u16 height;
- };
- struct ubwcp_plane_info {
- u16 pixel_bytes;
- u16 per_pixel;
- struct tile_dimension tilesize_p; /* pixels */
- struct tile_dimension macrotilesize_p; /* pixels */
- };
- struct ubwcp_image_format_info {
- u16 planes;
- struct ubwcp_plane_info p_info[2];
- };
- enum ubwcp_std_image_format {
- RGBA = 0,
- NV12 = 1,
- NV124R = 2,
- P010 = 3,
- TP10 = 4,
- P016 = 5,
- INFO_FORMAT_LIST_SIZE,
- };
- enum ubwcp_state {
- UBWCP_STATE_READY = 0,
- UBWCP_STATE_INVALID = -1,
- UBWCP_STATE_FAULT = -2,
- };
- struct ubwcp_prefetch_tgt_ctrl {
- atomic_t cpu_count;
- bool enable;
- int result;
- };
- struct ubwcp_driver {
- /* cdev related */
- dev_t devt;
- struct class *dev_class; //sysfs dev class
- struct device *dev_sys; //sysfs dev
- struct cdev cdev; //char dev
- /* debugfs */
- struct dentry *debugfs_root;
- bool read_err_irq_en;
- bool write_err_irq_en;
- bool decode_err_irq_en;
- bool encode_err_irq_en;
- /* ubwcp devices */
- struct device *dev; //ubwcp device
- struct device *dev_desc_cb; //smmu dev for descriptors
- struct device *dev_buf_cb; //smmu dev for ubwcp buffers
- void __iomem *base; //ubwcp base address
- struct regulator *vdd;
- struct clk **clocks;
- int num_clocks;
- /* interrupts */
- int irq_range_ck_rd;
- int irq_range_ck_wr;
- int irq_encode;
- int irq_decode;
- /* ula address pool */
- u64 ula_pool_base;
- u64 ula_pool_size;
- struct gen_pool *ula_pool;
- configure_mmap mmap_config_fptr;
- /* HW version */
- u32 hw_ver_major;
- u32 hw_ver_minor;
- /* keep track of all potential buffers.
- * hash table index'ed using dma_buf ptr.
- * 2**13 = 8192 hash values
- */
- DECLARE_HASHTABLE(buf_table, 13);
- /* buffer descriptor */
- void *buffer_desc_base; /* CPU address */
- dma_addr_t buffer_desc_dma_handle; /* dma address */
- size_t buffer_desc_size;
- struct ubwcp_desc desc_list[UBWCP_BUFFER_DESC_COUNT];
- struct ubwcp_image_format_info format_info[INFO_FORMAT_LIST_SIZE];
- /* driver state */
- enum ubwcp_state state;
- atomic_t num_non_lin_buffers;
- bool mem_online;
- struct mutex desc_lock; /* allocate/free descriptors */
- spinlock_t buf_table_lock; /* add/remove dma_buf into list of managed bufffers */
- struct mutex mem_hotplug_lock; /* memory hotplug lock */
- struct mutex ula_lock; /* allocate/free ula */
- struct mutex ubwcp_flush_lock; /* ubwcp flush */
- struct mutex hw_range_ck_lock; /* range ck */
- struct list_head err_handler_list; /* error handler list */
- spinlock_t err_handler_list_lock; /* err_handler_list lock */
- struct dev_pagemap pgmap;
- /* power state tracking */
- int power_on;
- struct mutex power_ctrl_lock;
- struct ubwcp_prefetch_tgt_ctrl ctrl;
- };
- struct ubwcp_buf {
- struct hlist_node hnode;
- struct ubwcp_driver *ubwcp;
- struct ubwcp_buffer_attrs buf_attr;
- bool perm;
- struct ubwcp_desc *desc;
- bool buf_attr_set;
- enum dma_data_direction dma_dir;
- int lock_count;
- /* dma_buf info */
- struct dma_buf *dma_buf;
- struct dma_buf_attachment *attachment;
- struct sg_table *sgt;
- /* ula info */
- phys_addr_t ula_pa;
- size_t ula_size;
- /* meta metadata */
- struct ubwcp_hw_meta_metadata mmdata;
- struct mutex lock;
- };
- static struct ubwcp_driver *me;
- static u32 ubwcp_debug_trace_enable;
- static void prefetch_tgt_per_cpu(void *info)
- {
- int ret = 0;
- struct ubwcp_prefetch_tgt_ctrl *ctrl;
- ctrl = (struct ubwcp_prefetch_tgt_ctrl *) info;
- ret = qcom_scm_prefetch_tgt_ctrl(ctrl->enable);
- if (ret) {
- //ctrl->result = ret;
- //ERR("scm call failed, ret: %d enable: %d", ret, ctrl->enable);
- DBG("scm call failed, ret: %d missing the matching TZ?", ret);
- }
- atomic_dec(&ctrl->cpu_count);
- }
- /* Enable/disable generation of prefetch target opcode. smc call must be done from each core
- * to update the core specific register. Not thread-safe.
- */
- static int prefetch_tgt(struct ubwcp_driver *ubwcp, bool enable)
- {
- int cpu;
- trace_ubwcp_prefetch_tgt_start(enable);
- DBG("enable: %d", enable);
- ubwcp->ctrl.enable = enable;
- ubwcp->ctrl.result = 0;
- atomic_set(&ubwcp->ctrl.cpu_count, 0);
- cpus_read_lock();
- for_each_cpu(cpu, cpu_online_mask) {
- atomic_inc(&ubwcp->ctrl.cpu_count);
- smp_call_function_single(cpu, prefetch_tgt_per_cpu, (void *) &ubwcp->ctrl, false);
- }
- cpus_read_unlock();
- while (atomic_read(&ubwcp->ctrl.cpu_count))
- ;
- DBG("done");
- trace_ubwcp_prefetch_tgt_end(enable);
- return ubwcp->ctrl.result;
- }
- static struct ubwcp_driver *ubwcp_get_driver(void)
- {
- if (!me)
- WARN(1, "ubwcp: driver ptr requested but driver not initialized");
- return me;
- }
- static void image_format_init(struct ubwcp_driver *ubwcp)
- { /* planes, bytes/p, Tp , MTp */
- ubwcp->format_info[RGBA] = (struct ubwcp_image_format_info)
- {1, {{4, 1, {16, 4}, {64, 16}}}};
- ubwcp->format_info[NV12] = (struct ubwcp_image_format_info)
- {2, {{1, 1, {32, 8}, {128, 32}},
- {2, 1, {16, 8}, { 64, 32}}}};
- ubwcp->format_info[NV124R] = (struct ubwcp_image_format_info)
- {2, {{1, 1, {64, 4}, {256, 16}},
- {2, 1, {32, 4}, {128, 16}}}};
- ubwcp->format_info[P010] = (struct ubwcp_image_format_info)
- {2, {{2, 1, {32, 4}, {128, 16}},
- {4, 1, {16, 4}, { 64, 16}}}};
- ubwcp->format_info[TP10] = (struct ubwcp_image_format_info)
- {2, {{4, 3, {48, 4}, {192, 16}},
- {8, 3, {24, 4}, { 96, 16}}}};
- ubwcp->format_info[P016] = (struct ubwcp_image_format_info)
- {2, {{2, 1, {32, 4}, {128, 16}},
- {4, 1, {16, 4}, { 64, 16}}}};
- }
- static void ubwcp_buf_desc_list_init(struct ubwcp_driver *ubwcp)
- {
- int idx;
- struct ubwcp_desc *desc_list = ubwcp->desc_list;
- for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
- desc_list[idx].idx = -1;
- desc_list[idx].ptr = NULL;
- }
- }
- static int ubwcp_init_clocks(struct ubwcp_driver *ubwcp, struct device *dev)
- {
- const char *cname;
- struct property *prop;
- int i;
- ubwcp->num_clocks =
- of_property_count_strings(dev->of_node, "clock-names");
- if (ubwcp->num_clocks < 1) {
- ubwcp->num_clocks = 0;
- return 0;
- }
- ubwcp->clocks = devm_kzalloc(dev,
- sizeof(*ubwcp->clocks) * ubwcp->num_clocks, GFP_KERNEL);
- if (!ubwcp->clocks)
- return -ENOMEM;
- i = 0;
- of_property_for_each_string(dev->of_node, "clock-names",
- prop, cname) {
- struct clk *c = devm_clk_get(dev, cname);
- if (IS_ERR(c)) {
- ERR("Couldn't get clock: %s\n", cname);
- return PTR_ERR(c);
- }
- ubwcp->clocks[i] = c;
- ++i;
- }
- return 0;
- }
- static int ubwcp_enable_clocks(struct ubwcp_driver *ubwcp)
- {
- int i, ret = 0;
- for (i = 0; i < ubwcp->num_clocks; ++i) {
- ret = clk_prepare_enable(ubwcp->clocks[i]);
- if (ret) {
- ERR("Couldn't enable clock #%d\n", i);
- while (i--)
- clk_disable_unprepare(ubwcp->clocks[i]);
- break;
- }
- }
- return ret;
- }
- static void ubwcp_disable_clocks(struct ubwcp_driver *ubwcp)
- {
- int i;
- for (i = ubwcp->num_clocks; i; --i)
- clk_disable_unprepare(ubwcp->clocks[i - 1]);
- }
- /* UBWCP Power control
- * Due to hw bug, ubwcp block cannot handle prefetch target opcode. Thus we disable the opcode
- * when ubwcp is powered on and enable it back when ubwcp is powered off.
- */
- static int ubwcp_power(struct ubwcp_driver *ubwcp, bool enable)
- {
- int ret = 0;
- mutex_lock(&ubwcp->power_ctrl_lock);
- if (enable) {
- ubwcp->power_on++;
- if (ubwcp->power_on != 1)
- goto done;
- } else {
- ubwcp->power_on--;
- if (ubwcp->power_on != 0)
- goto done;
- }
- if (enable) {
- ret = prefetch_tgt(ubwcp, 0);
- if (ret)
- goto done;
- ret = regulator_enable(ubwcp->vdd);
- if (ret) {
- ERR("regulator call (enable: %d) failed: %d", enable, ret);
- goto done;
- }
- ret = ubwcp_enable_clocks(ubwcp);
- if (ret) {
- ERR("enable clocks failed: %d", ret);
- regulator_disable(ubwcp->vdd);
- goto done;
- }
- } else {
- ret = regulator_disable(ubwcp->vdd);
- if (ret) {
- ERR("regulator call (enable: %d) failed: %d", enable, ret);
- goto done;
- }
- ubwcp_disable_clocks(ubwcp);
- ret = prefetch_tgt(ubwcp, 1);
- }
- done:
- mutex_unlock(&ubwcp->power_ctrl_lock);
- return ret;
- }
- /* get ubwcp_buf corresponding to the given dma_buf */
- static struct ubwcp_buf *dma_buf_to_ubwcp_buf(struct dma_buf *dmabuf)
- {
- struct ubwcp_buf *buf = NULL;
- struct ubwcp_buf *ret_buf = NULL;
- struct ubwcp_driver *ubwcp = ubwcp_get_driver();
- unsigned long flags;
- if (!dmabuf || !ubwcp)
- return NULL;
- spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
- /* look up ubwcp_buf corresponding to this dma_buf */
- hash_for_each_possible(ubwcp->buf_table, buf, hnode, (u64)dmabuf) {
- if (buf->dma_buf == dmabuf) {
- ret_buf = buf;
- break;
- }
- }
- spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
- return ret_buf;
- }
- /* return ubwcp hardware version */
- int ubwcp_get_hw_version(struct ubwcp_ioctl_hw_version *ver)
- {
- struct ubwcp_driver *ubwcp;
- if (!ver) {
- ERR("invalid version ptr");
- return -EINVAL;
- }
- ubwcp = ubwcp_get_driver();
- if (!ubwcp)
- return -1;
- if (ubwcp->state == UBWCP_STATE_INVALID)
- return -EPERM;
- ver->major = ubwcp->hw_ver_major;
- ver->minor = ubwcp->hw_ver_minor;
- return 0;
- }
- EXPORT_SYMBOL(ubwcp_get_hw_version);
- static int ula_add_mem(struct ubwcp_driver *ubwcp)
- {
- int ret = 0;
- int nid;
- void *ptr;
- nid = memory_add_physaddr_to_nid(ubwcp->ula_pool_base);
- DBG("calling memremap_pages()...");
- ubwcp->pgmap.type = MEMORY_DEVICE_GENERIC;
- ubwcp->pgmap.nr_range = 1;
- ubwcp->pgmap.range.start = ubwcp->ula_pool_base;
- ubwcp->pgmap.range.end = ubwcp->ula_pool_base + ubwcp->ula_pool_size - 1;
- trace_ubwcp_memremap_pages_start(ubwcp->ula_pool_size);
- ptr = memremap_pages(&ubwcp->pgmap, nid);
- trace_ubwcp_memremap_pages_end(ubwcp->ula_pool_size);
- if (IS_ERR(ptr)) {
- ret = IS_ERR(ptr);
- ERR("memremap_pages() failed st:0x%lx sz:0x%lx err: %d",
- ubwcp->ula_pool_base,
- ubwcp->ula_pool_size,
- ret);
- } else {
- DBG("memremap_pages() ula_pool_base:0x%llx, size:0x%zx, kernel addr:0x%p",
- ubwcp->ula_pool_base,
- ubwcp->ula_pool_size,
- page_to_virt(pfn_to_page(PFN_DOWN(ubwcp->ula_pool_base))));
- }
- return ret;
- }
- static int ula_map_uncached(u64 base, u64 size)
- {
- int ret;
- trace_ubwcp_set_direct_map_range_uncached_start(size);
- ret = set_direct_map_range_uncached((unsigned long)phys_to_virt(base), size >> PAGE_SHIFT);
- trace_ubwcp_set_direct_map_range_uncached_end(size);
- if (ret)
- ERR("set_direct_map_range_uncached failed st:0x%lx num pages:%lu err: %d",
- base, size >> PAGE_SHIFT, ret);
- return ret;
- }
- static void ula_unmap(struct ubwcp_driver *ubwcp)
- {
- DBG("Calling memunmap_pages() for ULA PA pool");
- trace_ubwcp_memunmap_pages_start(ubwcp->ula_pool_size);
- memunmap_pages(&ubwcp->pgmap);
- trace_ubwcp_memunmap_pages_end(ubwcp->ula_pool_size);
- }
- static void ula_sync_for_cpu(struct device *dev, u64 addr, unsigned long size)
- {
- trace_ubwcp_dma_sync_single_for_cpu_start(size, DMA_BIDIRECTIONAL);
- dma_sync_single_for_cpu(dev, addr, size, DMA_BIDIRECTIONAL);
- trace_ubwcp_dma_sync_single_for_cpu_end(size, DMA_BIDIRECTIONAL);
- }
- /** Remove ula memory in chunks
- * Abort if new buffer addition is detected
- * If remove succeeds or aborted, return success
- * status value indicates if mem was removed or aborted (not removed)
- * Otherwise return failure
- */
- static int ula_remove_mem(struct ubwcp_driver *ubwcp, enum ula_remove_mem_status *status)
- {
- int ret = 0;
- unsigned long sync_remain = ubwcp->ula_pool_size;
- unsigned long sync_offset = 0;
- unsigned long sync_size = 0;
- ret = ula_map_uncached(ubwcp->ula_pool_base, ubwcp->ula_pool_size);
- if (ret)
- return ret;
- trace_ubwcp_offline_sync_start(ubwcp->ula_pool_size);
- while (sync_remain > 0) {
- if (atomic_read(&ubwcp->num_non_lin_buffers) > 0) {
- trace_ubwcp_offline_sync_end(ubwcp->ula_pool_size);
- ula_unmap(ubwcp);
- if (ula_add_mem(ubwcp)) {
- ERR("remove mem: failed to add back during abort");
- return -1;
- }
- *status = ULA_REMOVE_MEM_ABORTED;
- return 0;
- }
- if (UBWCP_SYNC_GRANULE > sync_remain) {
- sync_size = sync_remain;
- sync_remain = 0;
- } else {
- sync_size = UBWCP_SYNC_GRANULE;
- sync_remain -= UBWCP_SYNC_GRANULE;
- }
- ula_sync_for_cpu(ubwcp->dev, ubwcp->ula_pool_base + sync_offset, sync_size);
- sync_offset += sync_size;
- }
- trace_ubwcp_offline_sync_end(ubwcp->ula_pool_size);
- ula_unmap(ubwcp);
- *status = ULA_REMOVE_MEM_SUCCESS;
- return 0;
- }
- static int inc_num_non_lin_buffers(struct ubwcp_driver *ubwcp)
- {
- atomic_inc(&ubwcp->num_non_lin_buffers);
- mutex_lock(&ubwcp->mem_hotplug_lock);
- if (!ubwcp->mem_online) {
- if (atomic_read(&ubwcp->num_non_lin_buffers) == 0) {
- ERR("Bad state: num_non_lin_buffers should not be 0");
- goto err;
- }
- if (ubwcp_power(ubwcp, true))
- goto err;
- if (ula_add_mem(ubwcp))
- goto err_add_memory;
- ubwcp->mem_online = true;
- }
- mutex_unlock(&ubwcp->mem_hotplug_lock);
- return 0;
- err_add_memory:
- ubwcp_power(ubwcp, false);
- err:
- atomic_dec(&ubwcp->num_non_lin_buffers);
- mutex_unlock(&ubwcp->mem_hotplug_lock);
- ubwcp->state = UBWCP_STATE_FAULT;
- ERR("state set to fault");
- return -1;
- }
- static int dec_num_non_lin_buffers(struct ubwcp_driver *ubwcp)
- {
- int ret;
- enum ula_remove_mem_status remove_status;
- atomic_dec(&ubwcp->num_non_lin_buffers);
- mutex_lock(&ubwcp->mem_hotplug_lock);
- if (atomic_read(&ubwcp->num_non_lin_buffers) == 0) {
- DBG("last buffer: ~~~~~~~~~~~");
- if (!ubwcp->mem_online) {
- ERR("Bad state: mem_online should not be false");
- goto err;
- }
- ret = ula_remove_mem(ubwcp, &remove_status);
- if (ret)
- goto err;
- if (remove_status == ULA_REMOVE_MEM_SUCCESS) {
- ubwcp->mem_online = false;
- if (ubwcp_power(ubwcp, false))
- goto err;
- } else if (remove_status == ULA_REMOVE_MEM_ABORTED) {
- DBG("ula memory offline aborted");
- } else {
- ERR("unexpected ula remove status: %d", remove_status);
- goto err;
- }
- }
- mutex_unlock(&ubwcp->mem_hotplug_lock);
- return 0;
- err:
- atomic_inc(&ubwcp->num_non_lin_buffers);
- mutex_unlock(&ubwcp->mem_hotplug_lock);
- ubwcp->state = UBWCP_STATE_FAULT;
- ERR("state set to fault");
- return -1;
- }
- /**
- *
- * Initialize ubwcp buffer for the given dma_buf. This
- * initializes ubwcp internal data structures and possibly hw to
- * use ubwcp for this buffer.
- *
- * @param dmabuf : ptr to the buffer to be configured for ubwcp
- *
- * @return int : 0 on success, otherwise error code
- */
- static int ubwcp_init_buffer(struct dma_buf *dmabuf)
- {
- struct ubwcp_buf *buf;
- struct ubwcp_driver *ubwcp = ubwcp_get_driver();
- unsigned long flags;
- trace_ubwcp_init_buffer_start(dmabuf);
- if (!ubwcp) {
- trace_ubwcp_init_buffer_end(dmabuf);
- return -1;
- }
- if (ubwcp->state != UBWCP_STATE_READY) {
- ERR("driver in invalid state: %d", ubwcp->state);
- trace_ubwcp_init_buffer_end(dmabuf);
- return -EPERM;
- }
- if (!dmabuf) {
- ERR("NULL dmabuf input ptr");
- trace_ubwcp_init_buffer_end(dmabuf);
- return -EINVAL;
- }
- if (dma_buf_to_ubwcp_buf(dmabuf)) {
- ERR("dma_buf already initialized for ubwcp");
- trace_ubwcp_init_buffer_end(dmabuf);
- return -EEXIST;
- }
- buf = kzalloc(sizeof(*buf), GFP_KERNEL);
- if (!buf) {
- ERR("failed to alloc for new ubwcp_buf");
- trace_ubwcp_init_buffer_end(dmabuf);
- return -ENOMEM;
- }
- mutex_init(&buf->lock);
- buf->dma_buf = dmabuf;
- buf->ubwcp = ubwcp;
- buf->buf_attr.image_format = UBWCP_LINEAR;
- spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
- hash_add(ubwcp->buf_table, &buf->hnode, (u64)buf->dma_buf);
- spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
- trace_ubwcp_init_buffer_end(dmabuf);
- return 0;
- }
- static void dump_attributes(struct ubwcp_buffer_attrs *attr)
- {
- DBG_BUF_ATTR("");
- DBG_BUF_ATTR("image_format: %d", attr->image_format);
- DBG_BUF_ATTR("major_ubwc_ver: %d", attr->major_ubwc_ver);
- DBG_BUF_ATTR("minor_ubwc_ver: %d", attr->minor_ubwc_ver);
- DBG_BUF_ATTR("compression_type: %d", attr->compression_type);
- DBG_BUF_ATTR("lossy_params: %llu", attr->lossy_params);
- DBG_BUF_ATTR("width: %d", attr->width);
- DBG_BUF_ATTR("height: %d", attr->height);
- DBG_BUF_ATTR("stride: %d", attr->stride);
- DBG_BUF_ATTR("scanlines: %d", attr->scanlines);
- DBG_BUF_ATTR("planar_padding: %d", attr->planar_padding);
- DBG_BUF_ATTR("subsample: %d", attr->subsample);
- DBG_BUF_ATTR("sub_system_target: %d", attr->sub_system_target);
- DBG_BUF_ATTR("y_offset: %d", attr->y_offset);
- DBG_BUF_ATTR("batch_size: %d", attr->batch_size);
- DBG_BUF_ATTR("");
- }
- static int to_std_format(u16 ioctl_image_format, enum ubwcp_std_image_format *format)
- {
- switch (ioctl_image_format) {
- case UBWCP_RGBA8888:
- *format = RGBA;
- return 0;
- case UBWCP_NV12:
- case UBWCP_NV12_Y:
- case UBWCP_NV12_UV:
- *format = NV12;
- return 0;
- case UBWCP_NV124R:
- case UBWCP_NV124R_Y:
- case UBWCP_NV124R_UV:
- *format = NV124R;
- return 0;
- case UBWCP_TP10:
- case UBWCP_TP10_Y:
- case UBWCP_TP10_UV:
- *format = TP10;
- return 0;
- case UBWCP_P010:
- case UBWCP_P010_Y:
- case UBWCP_P010_UV:
- *format = P010;
- return 0;
- case UBWCP_P016:
- case UBWCP_P016_Y:
- case UBWCP_P016_UV:
- *format = P016;
- return 0;
- default:
- ERR("Failed to convert ioctl image format to std format: %d", ioctl_image_format);
- return -1;
- }
- }
- static int std_to_hw_img_fmt(enum ubwcp_std_image_format format, u16 *hw_fmt)
- {
- switch (format) {
- case RGBA:
- *hw_fmt = HW_BUFFER_FORMAT_RGBA;
- return 0;
- case NV12:
- *hw_fmt = HW_BUFFER_FORMAT_NV12;
- return 0;
- case NV124R:
- *hw_fmt = HW_BUFFER_FORMAT_NV124R;
- return 0;
- case P010:
- *hw_fmt = HW_BUFFER_FORMAT_P010;
- return 0;
- case TP10:
- *hw_fmt = HW_BUFFER_FORMAT_TP10;
- return 0;
- case P016:
- *hw_fmt = HW_BUFFER_FORMAT_P016;
- return 0;
- default:
- ERR("Failed to convert std image format to hw format: %d", format);
- return -1;
- }
- }
- static int get_stride_alignment(enum ubwcp_std_image_format format, u16 *align)
- {
- switch (format) {
- case TP10:
- *align = 64;
- return 0;
- case NV12:
- *align = 128;
- return 0;
- case RGBA:
- case NV124R:
- case P010:
- case P016:
- *align = 256;
- return 0;
- default:
- return -1;
- }
- }
- /* returns stride of compressed image */
- static u32 get_compressed_stride(struct ubwcp_driver *ubwcp,
- enum ubwcp_std_image_format format, u32 width)
- {
- struct ubwcp_plane_info p_info;
- u16 macro_tile_width_p;
- u16 pixel_bytes;
- u16 per_pixel;
- p_info = ubwcp->format_info[format].p_info[0];
- macro_tile_width_p = p_info.macrotilesize_p.width;
- pixel_bytes = p_info.pixel_bytes;
- per_pixel = p_info.per_pixel;
- return UBWCP_ALIGN(width, macro_tile_width_p)*pixel_bytes/per_pixel;
- }
- static void
- ubwcp_pixel_to_bytes(struct ubwcp_driver *ubwcp,
- enum ubwcp_std_image_format format,
- u32 width_p, u32 height_p,
- u32 *width_b, u32 *height_b)
- {
- u16 pixel_bytes;
- u16 per_pixel;
- struct ubwcp_image_format_info f_info;
- struct ubwcp_plane_info p_info;
- f_info = ubwcp->format_info[format];
- p_info = f_info.p_info[0];
- pixel_bytes = p_info.pixel_bytes;
- per_pixel = p_info.per_pixel;
- *width_b = (width_p*pixel_bytes)/per_pixel;
- *height_b = (height_p*pixel_bytes)/per_pixel;
- }
- /* check if linear stride conforms to hw limitations
- * always returns false for linear image
- */
- static bool stride_is_valid(struct ubwcp_driver *ubwcp,
- enum ubwcp_std_image_format format, u32 width, u32 lin_stride)
- {
- u32 compressed_stride;
- u32 width_b;
- u32 height_b;
- ubwcp_pixel_to_bytes(ubwcp, format, width, 0, &width_b, &height_b);
- if ((lin_stride < width_b) || (lin_stride > MAX_ATTR_STRIDE)) {
- ERR("Invalid stride: %u width: %u width_b: %u", lin_stride, width, width_b);
- return false;
- }
- if (format == TP10) {
- if(!IS_ALIGNED(lin_stride, 64)) {
- ERR("stride must be aligned to 64: %d", lin_stride);
- return false;
- }
- } else {
- compressed_stride = get_compressed_stride(ubwcp, format, width);
- if (lin_stride != compressed_stride) {
- ERR("linear stride: %d must be same as compressed stride: %d",
- lin_stride, compressed_stride);
- return false;
- }
- }
- return true;
- }
- static bool ioctl_format_is_valid(u16 ioctl_image_format)
- {
- switch (ioctl_image_format) {
- case UBWCP_LINEAR:
- case UBWCP_RGBA8888:
- case UBWCP_NV12:
- case UBWCP_NV12_Y:
- case UBWCP_NV12_UV:
- case UBWCP_NV124R:
- case UBWCP_NV124R_Y:
- case UBWCP_NV124R_UV:
- case UBWCP_TP10:
- case UBWCP_TP10_Y:
- case UBWCP_TP10_UV:
- case UBWCP_P010:
- case UBWCP_P010_Y:
- case UBWCP_P010_UV:
- case UBWCP_P016:
- case UBWCP_P016_Y:
- case UBWCP_P016_UV:
- return true;
- default:
- return false;
- }
- }
- /* validate buffer attributes */
- static bool ubwcp_buf_attrs_valid(struct ubwcp_driver *ubwcp, struct ubwcp_buffer_attrs *attr)
- {
- enum ubwcp_std_image_format format;
- if (attr->unused1 || attr->unused2 || attr->unused3 || attr->unused4 || attr->unused5 ||
- attr->unused6 || attr->unused7 || attr->unused8 || attr->unused9) {
- ERR("buf attr unused values must be set to 0");
- goto err;
- }
- if (!attr->width || !attr->height || !attr->stride || !attr->scanlines) {
- ERR("width/height/stride/scanlines cannot be 0");
- goto err;
- }
- if (!ioctl_format_is_valid(attr->image_format)) {
- ERR("invalid image format: %d", attr->image_format);
- goto err;
- }
- /* rest of the fields are ignored for linear format */
- if (attr->image_format == UBWCP_LINEAR) {
- goto valid;
- }
- if (to_std_format(attr->image_format, &format))
- goto err;
- if (attr->major_ubwc_ver || attr->minor_ubwc_ver) {
- ERR("major/minor ubwc ver must be 0. major: %d minor: %d",
- attr->major_ubwc_ver, attr->minor_ubwc_ver);
- goto err;
- }
- if (attr->compression_type != UBWCP_COMPRESSION_LOSSLESS) {
- ERR("compression_type is not valid: %d",
- attr->compression_type);
- goto err;
- }
- if (attr->lossy_params != 0) {
- ERR("lossy_params is not valid: %d", attr->lossy_params);
- goto err;
- }
- if (attr->width > MAX_ATTR_WIDTH) {
- ERR("width is invalid (above upper limit): %d", attr->width);
- goto err;
- }
- if (attr->height > MAX_ATTR_HEIGHT) {
- ERR("height is invalid (above upper limit): %d", attr->height);
- goto err;
- }
- if(!stride_is_valid(ubwcp, format, attr->width, attr->stride)) {
- ERR("stride is invalid: %d", attr->stride);
- goto err;
- }
- if ((attr->scanlines < attr->height) ||
- (attr->scanlines > attr->height + MAX_ATTR_SCANLN_HT_DELTA)) {
- ERR("scanlines is not valid - height: %d scanlines: %d",
- attr->height, attr->scanlines);
- goto err;
- }
- if (attr->planar_padding > MAX_ATTR_PLANAR_PAD) {
- ERR("planar_padding is not valid: %d", attr->planar_padding);
- goto err;
- }
- if (attr->subsample != UBWCP_SUBSAMPLE_4_2_0) {
- ERR("subsample is not valid: %d", attr->subsample);
- goto err;
- }
- if (attr->sub_system_target & ~UBWCP_SUBSYSTEM_TARGET_CPU) {
- ERR("sub_system_target other that CPU is not supported: %d",
- attr->sub_system_target);
- goto err;
- }
- if (!(attr->sub_system_target & UBWCP_SUBSYSTEM_TARGET_CPU)) {
- ERR("sub_system_target is not set to CPU: %d",
- attr->sub_system_target);
- goto err;
- }
- if (attr->y_offset != 0) {
- ERR("y_offset is not valid: %d", attr->y_offset);
- goto err;
- }
- if (attr->batch_size != 1) {
- ERR("batch_size is not valid: %d", attr->batch_size);
- goto err;
- }
- valid:
- dump_attributes(attr);
- return true;
- err:
- dump_attributes(attr);
- return false;
- }
- /* calculate and return metadata buffer size for a given plane
- * and buffer attributes
- */
- static int metadata_buf_sz(struct ubwcp_driver *ubwcp,
- enum ubwcp_std_image_format format,
- u32 width, u32 height, u8 plane, size_t *size)
- {
- u64 pitch;
- u64 lines;
- u64 tile_width;
- u32 tile_height;
- struct ubwcp_image_format_info f_info;
- struct ubwcp_plane_info p_info;
- f_info = ubwcp->format_info[format];
- DBG_BUF_ATTR("");
- DBG_BUF_ATTR("");
- DBG_BUF_ATTR("Calculating metadata buffer size: format = %d, plane = %d", format, plane);
- if (plane >= f_info.planes) {
- ERR("Missing plane info: format: %d, plane: %d", format, plane);
- return -1;
- }
- p_info = f_info.p_info[plane];
- /* UV plane */
- if (plane == 1) {
- width = width/2;
- height = height/2;
- }
- tile_width = p_info.tilesize_p.width;
- tile_height = p_info.tilesize_p.height;
- /* pitch: # of tiles in a row
- * lines: # of tile rows
- */
- pitch = UBWCP_ALIGN((width + tile_width - 1)/tile_width, META_DATA_PITCH_ALIGN);
- lines = UBWCP_ALIGN((height + tile_height - 1)/tile_height, META_DATA_HEIGHT_ALIGN);
- DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
- DBG_BUF_ATTR("tile params : %d x %d (pixels)", tile_width, tile_height);
- DBG_BUF_ATTR("pitch : %d (%d)", pitch, width/tile_width);
- DBG_BUF_ATTR("lines : %d (%d)", lines, height);
- DBG_BUF_ATTR("size (p*l*bytes) : %d", pitch*lines*1);
- /* x1 below is only to clarify that we are multiplying by 1 bytes/tile */
- *size = UBWCP_ALIGN(pitch*lines*1, META_DATA_SIZE_ALIGN);
- DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", *size, *size);
- return 0;
- }
- /* calculate and return size of pixel data buffer for a given plane
- * and buffer attributes
- */
- static int pixeldata_buf_sz(struct ubwcp_driver *ubwcp,
- u16 format, u32 width,
- u32 height, u8 plane, size_t *size)
- {
- u64 pitch;
- u64 lines;
- u16 pixel_bytes;
- u16 per_pixel;
- u64 macro_tile_width_p;
- u64 macro_tile_height_p;
- struct ubwcp_image_format_info f_info;
- struct ubwcp_plane_info p_info;
- f_info = ubwcp->format_info[format];
- DBG_BUF_ATTR("");
- DBG_BUF_ATTR("");
- DBG_BUF_ATTR("Calculating Pixeldata buffer size: format = %d, plane = %d", format, plane);
- if (plane >= f_info.planes) {
- ERR("Missing plane info: format: %d, plane: %d", format, plane);
- return -1;
- }
- p_info = f_info.p_info[plane];
- pixel_bytes = p_info.pixel_bytes;
- per_pixel = p_info.per_pixel;
- /* UV plane */
- if (plane == 1) {
- width = width/2;
- height = height/2;
- }
- macro_tile_width_p = p_info.macrotilesize_p.width;
- macro_tile_height_p = p_info.macrotilesize_p.height;
- /* align pixel width and height macro tile width and height */
- pitch = UBWCP_ALIGN(width, macro_tile_width_p);
- lines = UBWCP_ALIGN(height, macro_tile_height_p);
- DBG_BUF_ATTR("image params : %d x %d (pixels)", width, height);
- DBG_BUF_ATTR("macro tile params: %d x %d (pixels)", macro_tile_width_p,
- macro_tile_height_p);
- DBG_BUF_ATTR("bytes_per_pixel : %d/%d", pixel_bytes, per_pixel);
- DBG_BUF_ATTR("pitch : %d", pitch);
- DBG_BUF_ATTR("lines : %d", lines);
- DBG_BUF_ATTR("size (p*l*bytes) : %d", (pitch*lines*pixel_bytes)/per_pixel);
- *size = UBWCP_ALIGN((pitch*lines*pixel_bytes)/per_pixel, PIXEL_DATA_SIZE_ALIGN);
- DBG_BUF_ATTR("size (aligned 4K): %zu (0x%zx)", *size, *size);
- return 0;
- }
- static int get_tile_height(struct ubwcp_driver *ubwcp, enum ubwcp_std_image_format format,
- u8 plane)
- {
- struct ubwcp_image_format_info f_info;
- struct ubwcp_plane_info p_info;
- f_info = ubwcp->format_info[format];
- p_info = f_info.p_info[plane];
- return p_info.tilesize_p.height;
- }
- /*
- * plane: must be 0 or 1 (1st plane == 0, 2nd plane == 1)
- */
- static size_t ubwcp_ula_size(struct ubwcp_driver *ubwcp, u16 format,
- u32 stride_b, u32 scanlines, u8 plane,
- bool add_tile_pad)
- {
- size_t size;
- DBG_BUF_ATTR("%s(format = %d, plane = %d)", __func__, format, plane);
- /* UV plane */
- if (plane == 1)
- scanlines = scanlines/2;
- if (add_tile_pad) {
- int tile_height = get_tile_height(ubwcp, format, plane);
- /* Align plane size to plane tile height */
- scanlines = ((scanlines + tile_height - 1) / tile_height) * tile_height;
- }
- size = stride_b*scanlines;
- DBG_BUF_ATTR("Size of plane-%u: (%u * %u) = %zu (0x%zx)",
- plane, stride_b, scanlines, size, size);
- return size;
- }
- static int missing_plane_from_format(u16 ioctl_image_format)
- {
- int missing_plane;
- switch (ioctl_image_format) {
- case UBWCP_NV12_Y:
- missing_plane = 2;
- break;
- case UBWCP_NV12_UV:
- missing_plane = 1;
- break;
- case UBWCP_NV124R_Y:
- missing_plane = 2;
- break;
- case UBWCP_NV124R_UV:
- missing_plane = 1;
- break;
- case UBWCP_TP10_Y:
- missing_plane = 2;
- break;
- case UBWCP_TP10_UV:
- missing_plane = 1;
- break;
- case UBWCP_P010_Y:
- missing_plane = 2;
- break;
- case UBWCP_P010_UV:
- missing_plane = 1;
- break;
- case UBWCP_P016_Y:
- missing_plane = 2;
- break;
- case UBWCP_P016_UV:
- missing_plane = 1;
- break;
- default:
- missing_plane = 0;
- }
- return missing_plane;
- }
- static int planes_in_format(enum ubwcp_std_image_format format)
- {
- if (format == RGBA)
- return 1;
- else
- return 2;
- }
- static int ubwcp_validate_uv_align(struct ubwcp_driver *ubwcp,
- struct ubwcp_buffer_attrs *attr,
- size_t ula_y_plane_size,
- size_t uv_start_offset)
- {
- int ret = 0;
- size_t ula_y_plane_size_align;
- size_t y_tile_align_bytes;
- int y_tile_height;
- int planes;
- enum ubwcp_std_image_format format;
- ret = to_std_format(attr->image_format, &format);
- if (ret)
- goto err;
- /* Only validate UV align if there is both a Y and UV plane */
- planes = planes_in_format(format);
- if (planes != 2)
- return 0;
- /* Check it is cache line size aligned */
- if ((uv_start_offset % 64) != 0) {
- ret = -EINVAL;
- ERR("uv_start_offset %zu not cache line aligned",
- uv_start_offset);
- goto err;
- }
- /*
- * Check that UV plane does not overlap with any of the Y plane’s tiles
- */
- y_tile_height = get_tile_height(ubwcp, format, 0);
- y_tile_align_bytes = y_tile_height * attr->stride;
- ula_y_plane_size_align = ((ula_y_plane_size + y_tile_align_bytes - 1) /
- y_tile_align_bytes) * y_tile_align_bytes;
- if (uv_start_offset < ula_y_plane_size_align) {
- ret = -EINVAL;
- ERR("uv offset %zu less than y plane align %zu for y plane size %zu",
- uv_start_offset, ula_y_plane_size_align,
- ula_y_plane_size);
- goto err;
- }
- return 0;
- err:
- return ret;
- }
- /* calculate ULA buffer parms */
- static int ubwcp_calc_ula_params(struct ubwcp_driver *ubwcp,
- struct ubwcp_buffer_attrs *attr,
- size_t *ula_size,
- size_t *ula_y_plane_size,
- size_t *uv_start_offset)
- {
- size_t size;
- enum ubwcp_std_image_format format;
- int planes;
- int missing_plane;
- u32 stride;
- u32 scanlines;
- u32 planar_padding;
- int ret;
- ret = to_std_format(attr->image_format, &format);
- if (ret)
- return ret;
- stride = attr->stride;
- scanlines = attr->scanlines;
- planar_padding = attr->planar_padding;
- /* Number of "expected" planes in "the standard defined" image format */
- planes = planes_in_format(format);
- missing_plane = missing_plane_from_format(attr->image_format);
- DBG_BUF_ATTR("ula params -->");
- DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
- DBG_BUF_ATTR("planes_in_format : %d", planes);
- DBG_BUF_ATTR("missing_plane : %d", missing_plane);
- DBG_BUF_ATTR("Planar Padding : %d", planar_padding);
- if (planes == 1) {
- /* uv_start beyond ULA range */
- size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
- *uv_start_offset = size;
- *ula_y_plane_size = size;
- } else {
- if (!missing_plane) {
- /* size for both planes and padding */
- /* Don't pad out Y plane as client would not expect this padding */
- size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, false);
- *ula_y_plane_size = size;
- size += planar_padding;
- *uv_start_offset = size;
- size += ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
- } else {
- if (missing_plane == 2) {
- /* Y-only image, set uv_start beyond ULA range */
- size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 0, true);
- *uv_start_offset = size;
- *ula_y_plane_size = size;
- } else {
- /* first plane data is not there */
- size = ubwcp_ula_size(ubwcp, format, stride, scanlines, 1, true);
- *uv_start_offset = 0; /* uv data is at the beginning */
- *ula_y_plane_size = 0;
- }
- }
- }
- *ula_size = UBWCP_ALIGN(size, 4096);
- DBG_BUF_ATTR("ULA_Size: %zu (0x%x) (before 4K align: %zu)", *ula_size, *ula_size, size);
- return 0;
- }
- /* calculate UBWCP buffer parms */
- static int ubwcp_calc_ubwcp_buf_params(struct ubwcp_driver *ubwcp,
- struct ubwcp_buffer_attrs *attr,
- size_t *md_p0, size_t *pd_p0,
- size_t *md_p1, size_t *pd_p1,
- size_t *stride_tp10_b)
- {
- int planes;
- int missing_plane;
- enum ubwcp_std_image_format format;
- size_t stride_tp10_p;
- int ret;
- ret = to_std_format(attr->image_format, &format);
- if (ret)
- return ret;
- missing_plane = missing_plane_from_format(attr->image_format);
- planes = planes_in_format(format);
- DBG_BUF_ATTR("ubwcp params -->");
- DBG_BUF_ATTR("ioctl_image_format : %d, std_format: %d", attr->image_format, format);
- DBG_BUF_ATTR("planes_in_format : %d", planes);
- DBG_BUF_ATTR("missing_plane : %d", missing_plane);
- *md_p0 = 0;
- *pd_p0 = 0;
- *md_p1 = 0;
- *pd_p1 = 0;
- *stride_tp10_b = 0;
- if (missing_plane != 1) {
- if (metadata_buf_sz(ubwcp, format, attr->width, attr->height, 0, md_p0))
- return -1;
- if (pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 0, pd_p0))
- return -1;
- }
- if ((planes == 2) && (missing_plane != 2)){
- if (metadata_buf_sz(ubwcp, format, attr->width, attr->height, 1, md_p1))
- return -1;
- if (pixeldata_buf_sz(ubwcp, format, attr->width, attr->height, 1, pd_p1))
- return -1;
- }
- if (format == TP10) {
- stride_tp10_p = UBWCP_ALIGN(attr->width, 192);
- *stride_tp10_b = (stride_tp10_p/3) + stride_tp10_p;
- }
- return 0;
- }
- /* reserve ULA address space of the given size */
- static phys_addr_t ubwcp_ula_alloc(struct ubwcp_driver *ubwcp, size_t size)
- {
- phys_addr_t pa;
- mutex_lock(&ubwcp->ula_lock);
- pa = gen_pool_alloc(ubwcp->ula_pool, size);
- mutex_unlock(&ubwcp->ula_lock);
- return pa;
- }
- /* free ULA address space of the given address and size */
- static void ubwcp_ula_free(struct ubwcp_driver *ubwcp, phys_addr_t pa, size_t size)
- {
- mutex_lock(&ubwcp->ula_lock);
- if (!gen_pool_has_addr(ubwcp->ula_pool, pa, size)) {
- ERR("Attempt to free mem not from gen_pool: pa: %p, size: %zx", pa, size);
- goto err;
- }
- DBG("addr: %p, size: %zx", pa, size);
- gen_pool_free(ubwcp->ula_pool, pa, size);
- mutex_unlock(&ubwcp->ula_lock);
- return;
- err:
- mutex_unlock(&ubwcp->ula_lock);
- }
- /* free up or expand current_pa and return the new pa */
- static phys_addr_t ubwcp_ula_realloc(struct ubwcp_driver *ubwcp,
- phys_addr_t pa,
- size_t size,
- size_t new_size)
- {
- if (size == new_size)
- return pa;
- if (pa)
- ubwcp_ula_free(ubwcp, pa, size);
- return ubwcp_ula_alloc(ubwcp, new_size);
- }
- /* unmap dma buf */
- static void ubwcp_dma_unmap(struct ubwcp_buf *buf)
- {
- if (buf->dma_buf && buf->attachment) {
- DBG("Calling dma_buf_unmap_attachment()");
- dma_buf_unmap_attachment(buf->attachment, buf->sgt, DMA_BIDIRECTIONAL);
- buf->sgt = NULL;
- dma_buf_detach(buf->dma_buf, buf->attachment);
- buf->attachment = NULL;
- }
- }
- static bool verify_dma_buf_size(struct ubwcp_buf *buf, size_t min_size)
- {
- size_t dma_len;
- dma_len = sg_dma_len(buf->sgt->sgl);
- if (dma_len < min_size) {
- ERR("dma len: %zu is less than min ubwcp buffer size: %zu", dma_len, min_size);
- return false;
- } else
- return true;
- }
- /* dma map ubwcp buffer */
- static int ubwcp_dma_map(struct ubwcp_buf *buf,
- struct device *dev,
- dma_addr_t *iova)
- {
- int ret = 0;
- struct dma_buf *dma_buf = buf->dma_buf;
- struct dma_buf_attachment *attachment;
- struct sg_table *sgt;
- /* Map buffer to SMMU and get IOVA */
- attachment = dma_buf_attach(dma_buf, dev);
- if (IS_ERR(attachment)) {
- ret = PTR_ERR(attachment);
- ERR("dma_buf_attach() failed: %d", ret);
- goto err;
- }
- dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
- dma_set_seg_boundary(dev, (unsigned long)DMA_BIT_MASK(64));
- sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
- if (IS_ERR_OR_NULL(sgt)) {
- ret = PTR_ERR(sgt);
- ERR("dma_buf_map_attachment() failed: %d", ret);
- goto err_detach;
- }
- if (sgt->nents != 1) {
- ERR("nents = %d", sgt->nents);
- goto err_unmap;
- }
- *iova = sg_dma_address(sgt->sgl);
- buf->attachment = attachment;
- buf->sgt = sgt;
- return ret;
- err_unmap:
- dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
- err_detach:
- dma_buf_detach(dma_buf, attachment);
- err:
- if (!ret)
- ret = -1;
- return ret;
- }
- static void reset_buf_attrs(struct ubwcp_buf *buf)
- {
- struct ubwcp_hw_meta_metadata *mmdata;
- struct ubwcp_driver *ubwcp;
- ubwcp = buf->ubwcp;
- mmdata = &buf->mmdata;
- ubwcp_dma_unmap(buf);
- /* reset ula params */
- if (buf->ula_size) {
- ubwcp_ula_free(ubwcp, buf->ula_pa, buf->ula_size);
- buf->ula_size = 0;
- buf->ula_pa = 0;
- }
- /* reset ubwcp params */
- memset(mmdata, 0, sizeof(*mmdata));
- buf->buf_attr_set = false;
- buf->buf_attr.image_format = UBWCP_LINEAR;
- }
- static void print_mmdata_desc(struct ubwcp_hw_meta_metadata *mmdata)
- {
- DBG_BUF_ATTR("");
- DBG_BUF_ATTR("--------MM_DATA DESC ---------");
- DBG_BUF_ATTR("uv_start_addr : 0x%08llx (cache addr) (actual: 0x%llx)",
- mmdata->uv_start_addr, mmdata->uv_start_addr << 6);
- DBG_BUF_ATTR("format : 0x%08x", mmdata->format);
- DBG_BUF_ATTR("stride : 0x%08x (cache addr) (actual: 0x%x)",
- mmdata->stride, mmdata->stride << 6);
- DBG_BUF_ATTR("stride_ubwcp : 0x%08x (cache addr) (actual: 0x%zx)",
- mmdata->stride_ubwcp, mmdata->stride_ubwcp << 6);
- DBG_BUF_ATTR("metadata_base_y : 0x%08x (page addr) (actual: 0x%llx)",
- mmdata->metadata_base_y, mmdata->metadata_base_y << 12);
- DBG_BUF_ATTR("metadata_base_uv: 0x%08x (page addr) (actual: 0x%zx)",
- mmdata->metadata_base_uv, mmdata->metadata_base_uv << 12);
- DBG_BUF_ATTR("buffer_y_offset : 0x%08x (page addr) (actual: 0x%zx)",
- mmdata->buffer_y_offset, mmdata->buffer_y_offset << 12);
- DBG_BUF_ATTR("buffer_uv_offset: 0x%08x (page addr) (actual: 0x%zx)",
- mmdata->buffer_uv_offset, mmdata->buffer_uv_offset << 12);
- DBG_BUF_ATTR("width_height : 0x%08x (width: 0x%x height: 0x%x)",
- mmdata->width_height, mmdata->width_height >> 16, mmdata->width_height & 0xFFFF);
- DBG_BUF_ATTR("");
- }
- /* set buffer attributes:
- * Failure:
- * This call may fail for multiple reasons and it will leave the buffer in an undefined state.
- * In some situations it may leave the buffer in linear mapped state, and in other situations it
- * may leave the buffer in previously set attributes state.
- */
- int ubwcp_set_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
- {
- int ret = 0;
- size_t ula_size = 0;
- size_t uv_start_offset = 0;
- size_t ula_y_plane_size = 0;
- phys_addr_t ula_pa = 0x0;
- struct ubwcp_buf *buf;
- struct ubwcp_driver *ubwcp;
- size_t metadata_p0;
- size_t pixeldata_p0;
- size_t metadata_p1;
- size_t pixeldata_p1;
- size_t iova_min_size;
- size_t stride_tp10_b;
- dma_addr_t iova_base;
- struct ubwcp_hw_meta_metadata *mmdata;
- u64 uv_start;
- u32 stride_b;
- u32 width_b;
- u32 height_b;
- enum ubwcp_std_image_format std_image_format;
- bool is_non_lin_buf;
- u16 hw_img_format;
- trace_ubwcp_set_buf_attrs_start(dmabuf);
- if (!dmabuf) {
- ERR("NULL dmabuf input ptr");
- ret = -EINVAL;
- goto err_validation;
- }
- if (!attr) {
- ERR("NULL attr ptr");
- ret = -EINVAL;
- goto err_validation;
- }
- buf = dma_buf_to_ubwcp_buf(dmabuf);
- if (!buf) {
- ERR("No corresponding ubwcp_buf for the passed in dma_buf");
- ret = -EINVAL;
- goto err_validation;
- }
- ubwcp = buf->ubwcp;
- if (ubwcp->state != UBWCP_STATE_READY) {
- ret = EPERM;
- goto err_validation;
- }
- if (!ubwcp_buf_attrs_valid(ubwcp, attr)) {
- ERR("Invalid buf attrs");
- ret = -EINVAL;
- goto err_validation;
- }
- mutex_lock(&buf->lock);
- if (buf->lock_count) {
- ERR("Cannot set attr when buffer is locked");
- ret = -EBUSY;
- goto unlock;
- }
- mmdata = &buf->mmdata;
- is_non_lin_buf = (buf->buf_attr.image_format != UBWCP_LINEAR);
- /* note: this also checks if buf is mmap'ed */
- ret = ubwcp->mmap_config_fptr(buf->dma_buf, true, 0, 0);
- if (ret) {
- ERR("dma_buf_mmap_config(0,0) failed: %d", ret);
- goto unlock;
- }
- if (attr->image_format == UBWCP_LINEAR) {
- DBG_BUF_ATTR("Linear format requested");
- if (buf->buf_attr_set)
- reset_buf_attrs(buf);
- if (is_non_lin_buf) {
- /*
- * Changing buffer from ubwc to linear so decrement
- * number of ubwc buffers
- */
- ret = dec_num_non_lin_buffers(ubwcp);
- }
- mutex_unlock(&buf->lock);
- trace_ubwcp_set_buf_attrs_end(dmabuf);
- return ret;
- }
- if (to_std_format(attr->image_format, &std_image_format)) {
- ERR("Unable to map ioctl image format to std image format");
- goto unlock;
- }
- if (std_to_hw_img_fmt(std_image_format, &hw_img_format)) {
- ERR("Unable to map std image format to hw image format");
- goto unlock;
- }
- /* Calculate uncompressed-buffer size. */
- ret = ubwcp_calc_ula_params(ubwcp, attr, &ula_size, &ula_y_plane_size, &uv_start_offset);
- if (ret) {
- ERR("ubwcp_calc_ula_params() failed: %d", ret);
- goto unlock;
- }
- ret = ubwcp_validate_uv_align(ubwcp, attr, ula_y_plane_size, uv_start_offset);
- if (ret) {
- ERR("ubwcp_validate_uv_align() failed: %d", ret);
- goto unlock;
- }
- ret = ubwcp_calc_ubwcp_buf_params(ubwcp, attr, &metadata_p0, &pixeldata_p0, &metadata_p1,
- &pixeldata_p1, &stride_tp10_b);
- if (ret) {
- ERR("ubwcp_calc_buf_params() failed: %d", ret);
- goto unlock;
- }
- iova_min_size = metadata_p0 + pixeldata_p0 + metadata_p1 + pixeldata_p1;
- DBG_BUF_ATTR("");
- DBG_BUF_ATTR("");
- DBG_BUF_ATTR("------Summary ULA Calculated Params ------");
- DBG_BUF_ATTR("ULA Size : %8zu (0x%8zx)", ula_size, ula_size);
- DBG_BUF_ATTR("UV Start Offset : %8zu (0x%8zx)", uv_start_offset, uv_start_offset);
- DBG_BUF_ATTR("------Summary UBCP Calculated Params ------");
- DBG_BUF_ATTR("metadata_p0 : %8d (0x%8zx)", metadata_p0, metadata_p0);
- DBG_BUF_ATTR("pixeldata_p0 : %8d (0x%8zx)", pixeldata_p0, pixeldata_p0);
- DBG_BUF_ATTR("metadata_p1 : %8d (0x%8zx)", metadata_p1, metadata_p1);
- DBG_BUF_ATTR("pixeldata_p1 : %8d (0x%8zx)", pixeldata_p1, pixeldata_p1);
- DBG_BUF_ATTR("stride_tp10 : %8d (0x%8zx)", stride_tp10_b, stride_tp10_b);
- DBG_BUF_ATTR("iova_min_size : %8d (0x%8zx)", iova_min_size, iova_min_size);
- DBG_BUF_ATTR("");
- if (!ula_size) {
- ERR("Invalid ula_size (0)");
- goto unlock;
- }
- /* assign ULA PA with uncompressed-size range */
- ula_pa = ubwcp_ula_realloc(ubwcp, buf->ula_pa, buf->ula_size, ula_size);
- if (!ula_pa) {
- ERR("ubwcp_ula_alloc/realloc() failed. running out of ULA PA space?");
- goto err;
- }
- buf->ula_size = ula_size;
- buf->ula_pa = ula_pa;
- DBG_BUF_ATTR("Allocated ULA_PA: 0x%p of size: 0x%zx", ula_pa, ula_size);
- DBG_BUF_ATTR("");
- /* dma map only the first time attribute is set */
- if (!buf->buf_attr_set) {
- /* linear -> ubwcp. map ubwcp buffer */
- ret = ubwcp_dma_map(buf, ubwcp->dev_buf_cb, &iova_base);
- if (ret) {
- ERR("ubwcp_dma_map() failed: %d", ret);
- goto err;
- }
- DBG_BUF_ATTR("dma_buf IOVA range: 0x%llx + min_size (0x%zx): 0x%llx",
- iova_base, iova_min_size, iova_base + iova_min_size);
- }
- if(!verify_dma_buf_size(buf, iova_min_size))
- goto err;
- uv_start = ula_pa + uv_start_offset;
- if (!IS_ALIGNED(uv_start, 64)) {
- ERR("ERROR: uv_start is NOT aligned to cache line");
- goto err;
- }
- /* Convert height and width to bytes for writing to mmdata */
- if (std_image_format != TP10) {
- ubwcp_pixel_to_bytes(ubwcp, std_image_format, attr->width,
- attr->height, &width_b, &height_b);
- } else {
- /* for tp10 image compression, we need to program p010 width/height */
- ubwcp_pixel_to_bytes(ubwcp, P010, attr->width,
- attr->height, &width_b, &height_b);
- }
- stride_b = attr->stride;
- /* create the mmdata descriptor */
- memset(mmdata, 0, sizeof(*mmdata));
- mmdata->uv_start_addr = CACHE_ADDR(uv_start);
- mmdata->format = hw_img_format;
- if (std_image_format != TP10) {
- mmdata->stride = CACHE_ADDR(stride_b); /* uncompressed stride */
- } else {
- mmdata->stride = CACHE_ADDR(stride_tp10_b); /* compressed stride */
- mmdata->stride_ubwcp = CACHE_ADDR(stride_b); /* uncompressed stride */
- }
- mmdata->metadata_base_y = PAGE_ADDR(iova_base);
- mmdata->metadata_base_uv = PAGE_ADDR(iova_base + metadata_p0 + pixeldata_p0);
- mmdata->buffer_y_offset = PAGE_ADDR(metadata_p0);
- mmdata->buffer_uv_offset = PAGE_ADDR(metadata_p1);
- /* NOTE: For version 1.1, both width & height needs to be in bytes.
- * For other versions, width in bytes & height in pixels.
- */
- if ((ubwcp->hw_ver_major == 1) && (ubwcp->hw_ver_minor == 1))
- mmdata->width_height = width_b << 16 | height_b;
- else
- mmdata->width_height = width_b << 16 | attr->height;
- print_mmdata_desc(mmdata);
- if (!is_non_lin_buf) {
- /*
- * Changing buffer from linear to ubwc so increment
- * number of ubwc buffers
- */
- ret = inc_num_non_lin_buffers(ubwcp);
- }
- if (ret) {
- ERR("inc_num_non_lin_buffers failed: %d", ret);
- goto err;
- }
- /* inform ULA-PA to dma-heap */
- DBG_BUF_ATTR("Calling mmap_config(): ULA_PA: 0x%p size: 0x%zx", ula_pa, ula_size);
- ret = ubwcp->mmap_config_fptr(buf->dma_buf, false, buf->ula_pa, buf->ula_size);
- if (ret) {
- ERR("dma_buf_mmap_config() failed: %d", ret);
- if (!is_non_lin_buf)
- dec_num_non_lin_buffers(ubwcp);
- goto err;
- }
- buf->buf_attr = *attr;
- buf->buf_attr_set = true;
- mutex_unlock(&buf->lock);
- trace_ubwcp_set_buf_attrs_end(dmabuf);
- return 0;
- err:
- reset_buf_attrs(buf);
- if (is_non_lin_buf) {
- /*
- * Changing buffer from ubwc to linear so decrement
- * number of ubwc buffers
- */
- dec_num_non_lin_buffers(ubwcp);
- }
- unlock:
- mutex_unlock(&buf->lock);
- err_validation:
- if (!ret)
- ret = -1;
- trace_ubwcp_set_buf_attrs_end(dmabuf);
- return ret;
- }
- EXPORT_SYMBOL(ubwcp_set_buf_attrs);
- /* Free up the buffer descriptor */
- static void ubwcp_buf_desc_free(struct ubwcp_driver *ubwcp, struct ubwcp_desc *desc)
- {
- int idx = desc->idx;
- struct ubwcp_desc *desc_list = ubwcp->desc_list;
- mutex_lock(&ubwcp->desc_lock);
- desc_list[idx].idx = -1;
- desc_list[idx].ptr = NULL;
- DBG("freed descriptor_id: %d", idx);
- mutex_unlock(&ubwcp->desc_lock);
- }
- /* Allocate next available buffer descriptor. */
- static struct ubwcp_desc *ubwcp_buf_desc_allocate(struct ubwcp_driver *ubwcp)
- {
- int idx;
- struct ubwcp_desc *desc_list = ubwcp->desc_list;
- mutex_lock(&ubwcp->desc_lock);
- for (idx = 0; idx < UBWCP_BUFFER_DESC_COUNT; idx++) {
- if (desc_list[idx].idx == -1) {
- desc_list[idx].idx = idx;
- desc_list[idx].ptr = ubwcp->buffer_desc_base +
- idx*UBWCP_BUFFER_DESC_OFFSET;
- DBG("allocated descriptor_id: %d", idx);
- mutex_unlock(&ubwcp->desc_lock);
- return &desc_list[idx];
- }
- }
- mutex_unlock(&ubwcp->desc_lock);
- return NULL;
- }
- static int ubwcp_flush(struct ubwcp_driver *ubwcp)
- {
- int ret = 0;
- mutex_lock(&ubwcp->ubwcp_flush_lock);
- trace_ubwcp_hw_flush_start(0);
- ret = ubwcp_hw_flush(ubwcp->base);
- trace_ubwcp_hw_flush_end(0);
- if (ret)
- ERR("ubwcp_hw_flush() failed, ret = %d", ret);
- mutex_unlock(&ubwcp->ubwcp_flush_lock);
- return ret;
- }
- static int range_check_disable(struct ubwcp_driver *ubwcp, int idx)
- {
- int ret;
- mutex_lock(&ubwcp->ubwcp_flush_lock);
- mutex_lock(&ubwcp->hw_range_ck_lock);
- trace_ubwcp_hw_flush_start(0);
- ret = ubwcp_hw_disable_range_check_with_flush(ubwcp->base, idx);
- trace_ubwcp_hw_flush_end(0);
- if (ret)
- ERR("disable_range_check_with_flush() failed: %d", ret);
- mutex_unlock(&ubwcp->hw_range_ck_lock);
- mutex_unlock(&ubwcp->ubwcp_flush_lock);
- return ret;
- }
- static void range_check_enable(struct ubwcp_driver *ubwcp, int idx)
- {
- mutex_lock(&ubwcp->hw_range_ck_lock);
- ubwcp_hw_enable_range_check(ubwcp->base, idx);
- mutex_unlock(&ubwcp->hw_range_ck_lock);
- }
- /**
- * Lock buffer for CPU access. This prepares ubwcp hw to allow
- * CPU access to the compressed buffer. It will perform
- * necessary address translation configuration and cache maintenance ops
- * so that CPU can safely access ubwcp buffer, if this call is
- * successful.
- * Allocate descriptor if not already,
- * perform CMO and then enable range check
- *
- * @param dmabuf : ptr to the dma buf
- * @param direction : direction of access
- *
- * @return int : 0 on success, otherwise error code
- */
- static int ubwcp_lock(struct dma_buf *dmabuf, enum dma_data_direction dir)
- {
- int ret = 0;
- struct ubwcp_buf *buf;
- struct ubwcp_driver *ubwcp;
- trace_ubwcp_lock_start(dmabuf);
- if (!dmabuf) {
- ERR("NULL dmabuf input ptr");
- trace_ubwcp_lock_end(dmabuf);
- return -EINVAL;
- }
- if (!valid_dma_direction(dir)) {
- ERR("invalid direction: %d", dir);
- trace_ubwcp_lock_end(dmabuf);
- return -EINVAL;
- }
- buf = dma_buf_to_ubwcp_buf(dmabuf);
- if (!buf) {
- ERR("ubwcp_buf ptr not found");
- trace_ubwcp_lock_end(dmabuf);
- return -1;
- }
- ubwcp = buf->ubwcp;
- if (ubwcp->state != UBWCP_STATE_READY) {
- ERR("driver in invalid state: %d", ubwcp->state);
- trace_ubwcp_lock_end(dmabuf);
- return -EPERM;
- }
- mutex_lock(&buf->lock);
- if (!buf->buf_attr_set) {
- ERR("lock() called on buffer, but attr not set");
- goto err;
- }
- if (buf->buf_attr.image_format == UBWCP_LINEAR) {
- ERR("lock() called on linear buffer");
- goto err;
- }
- if (!buf->lock_count) {
- DBG("first lock on buffer");
- /* buf->desc could already be allocated because of perm range xlation */
- if (!buf->desc) {
- /* allocate a buffer descriptor */
- buf->desc = ubwcp_buf_desc_allocate(buf->ubwcp);
- if (!buf->desc) {
- ERR("ubwcp_allocate_buf_desc() failed");
- goto err;
- }
- memcpy(buf->desc->ptr, &buf->mmdata, sizeof(buf->mmdata));
- /* Flushing of updated mmdata:
- * mmdata is iocoherent and ubwcp will get it from CPU cache -
- * *as long as* it has not cached that itself during previous
- * access to the same descriptor.
- *
- * During unlock of previous use of this descriptor,
- * we do hw flush, which will get rid of this mmdata from
- * ubwcp cache.
- *
- * In addition, we also do a hw flush after enable_range_ck().
- * That will also get rid of any speculative fetch of mmdata
- * by the ubwcp hw. At this time, the assumption is that ubwcp
- * will cache mmdata only for active descriptor. But if ubwcp
- * is speculatively fetching mmdata for all descriptors
- * (irrespetive of enabled or not), the flush during lock
- * will be necessary to make sure ubwcp sees updated mmdata
- * that we just updated
- */
- /* program ULA range for this buffer */
- DBG("setting range check: descriptor_id: %d, addr: %p, size: %zx",
- buf->desc->idx, buf->ula_pa, buf->ula_size);
- ubwcp_hw_set_range_check(ubwcp->base, buf->desc->idx, buf->ula_pa,
- buf->ula_size);
- }
- /* enable range check */
- DBG("enabling range check, descriptor_id: %d", buf->desc->idx);
- range_check_enable(ubwcp, buf->desc->idx);
- /* Flush/invalidate UBWCP caches */
- /* Why: cpu could have done a speculative fetch before
- * enable_range_ck() and ubwcp in process of returning "default" data
- * we don't want that stashing of default data pending.
- * we force completion of that and then we also cpu invalidate which
- * will get rid of that line.
- */
- ret = ubwcp_flush(ubwcp);
- if (ret) {
- ubwcp->state = UBWCP_STATE_FAULT;
- ERR("state set to fault");
- goto err_flush_failed;
- }
- /* Only apply CMOs if there are potential CPU reads */
- if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) {
- trace_ubwcp_dma_sync_single_for_cpu_start(buf->ula_size, dir);
- dma_sync_single_for_cpu(ubwcp->dev, buf->ula_pa, buf->ula_size, dir);
- trace_ubwcp_dma_sync_single_for_cpu_end(buf->ula_size, dir);
- }
- buf->dma_dir = dir;
- } else {
- DBG("buf already locked");
- /* For write locks, always upgrade direction to bi_directional.
- * A previous read lock will now become write lock.
- * This will ensure a flush when the last unlock comes in.
- */
- if (buf->dma_dir == DMA_TO_DEVICE &&
- buf->dma_dir != dir) {
- /*
- * Locking for read would require doing a cache invalidation which
- * we don't want to do while a client may be writing to the buffer
- * as that could drop valid lines from the cache.
- */
- ret = -EINVAL;
- ERR("no support for locking a write only buffer for read");
- goto err;
- } else if (buf->dma_dir != dir) {
- buf->dma_dir = DMA_BIDIRECTIONAL;
- }
- }
- buf->lock_count++;
- DBG("new lock_count: %d", buf->lock_count);
- mutex_unlock(&buf->lock);
- trace_ubwcp_lock_end(dmabuf);
- return ret;
- err_flush_failed:
- range_check_disable(ubwcp, buf->desc->idx);
- ubwcp_buf_desc_free(ubwcp, buf->desc);
- buf->desc = NULL;
- err:
- mutex_unlock(&buf->lock);
- if (!ret)
- ret = -1;
- trace_ubwcp_lock_end(dmabuf);
- return ret;
- }
- /* This can be called as a result of external unlock() call or
- * internally if free() is called without unlock().
- */
- static int unlock_internal(struct ubwcp_buf *buf, enum dma_data_direction dir, bool free_buffer)
- {
- int ret = 0;
- struct ubwcp_driver *ubwcp;
- DBG("current lock_count: %d", buf->lock_count);
- if (free_buffer) {
- buf->lock_count = 0;
- DBG("Forced lock_count: %d", buf->lock_count);
- } else {
- /* for write unlocks, remember the direction so we flush on last unlock */
- if (buf->dma_dir != dir)
- buf->dma_dir = DMA_BIDIRECTIONAL;
- buf->lock_count--;
- DBG("new lock_count: %d", buf->lock_count);
- if (buf->lock_count) {
- DBG("more than 1 lock on buffer. waiting until last unlock");
- return 0;
- }
- }
- ubwcp = buf->ubwcp;
- /* Only apply CMOs if there were potential CPU writes */
- if (buf->dma_dir == DMA_TO_DEVICE || buf->dma_dir == DMA_BIDIRECTIONAL) {
- /* Flush/invalidate ULA PA from CPU caches */
- trace_ubwcp_dma_sync_single_for_device_start(buf->ula_size, buf->dma_dir);
- dma_sync_single_for_device(ubwcp->dev, buf->ula_pa, buf->ula_size, buf->dma_dir);
- trace_ubwcp_dma_sync_single_for_device_end(buf->ula_size, buf->dma_dir);
- }
- /* disable range check */
- DBG("disabling range check");
- ret = range_check_disable(ubwcp, buf->desc->idx);
- if (ret) {
- ubwcp->state = UBWCP_STATE_FAULT;
- ERR("state set to fault");
- }
- /* release descriptor if perm range xlation is not set */
- if (!buf->perm) {
- ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
- buf->desc = NULL;
- }
- return ret;
- }
- /**
- * Unlock buffer from CPU access. This prepares ubwcp hw to
- * safely allow for device access to the compressed buffer including any
- * necessary cache maintenance ops. It may also free up certain ubwcp
- * resources that could result in error when accessed by CPU in
- * unlocked state.
- *
- * @param dmabuf : ptr to the dma buf
- * @param direction : direction of access
- *
- * @return int : 0 on success, otherwise error code
- */
- static int ubwcp_unlock(struct dma_buf *dmabuf, enum dma_data_direction dir)
- {
- struct ubwcp_buf *buf;
- int ret;
- trace_ubwcp_unlock_start(dmabuf);
- if (!dmabuf) {
- ERR("NULL dmabuf input ptr");
- trace_ubwcp_unlock_end(dmabuf);
- return -EINVAL;
- }
- if (!valid_dma_direction(dir)) {
- ERR("invalid direction: %d", dir);
- trace_ubwcp_unlock_end(dmabuf);
- return -EINVAL;
- }
- buf = dma_buf_to_ubwcp_buf(dmabuf);
- if (!buf) {
- ERR("ubwcp_buf not found");
- trace_ubwcp_unlock_end(dmabuf);
- return -1;
- }
- if (buf->ubwcp->state != UBWCP_STATE_READY) {
- ERR("driver in invalid state: %d", buf->ubwcp->state);
- trace_ubwcp_unlock_end(dmabuf);
- return -EPERM;
- }
- mutex_lock(&buf->lock);
- if (!buf->lock_count) {
- ERR("unlock() called on buffer which not in locked state");
- trace_ubwcp_unlock_end(dmabuf);
- mutex_unlock(&buf->lock);
- return -1;
- }
- ret = unlock_internal(buf, dir, false);
- mutex_unlock(&buf->lock);
- trace_ubwcp_unlock_end(dmabuf);
- return ret;
- }
- /* Return buffer attributes for the given buffer */
- int ubwcp_get_buf_attrs(struct dma_buf *dmabuf, struct ubwcp_buffer_attrs *attr)
- {
- int ret = 0;
- struct ubwcp_buf *buf;
- if (!dmabuf) {
- ERR("NULL dmabuf input ptr");
- return -EINVAL;
- }
- if (!attr) {
- ERR("NULL attr ptr");
- return -EINVAL;
- }
- buf = dma_buf_to_ubwcp_buf(dmabuf);
- if (!buf) {
- ERR("ubwcp_buf ptr not found");
- return -1;
- }
- if (buf->ubwcp->state != UBWCP_STATE_READY) {
- ERR("driver in invalid state: %d", buf->ubwcp->state);
- return -EPERM;
- }
- mutex_lock(&buf->lock);
- if (!buf->buf_attr_set) {
- ERR("buffer attributes not set");
- mutex_unlock(&buf->lock);
- return -1;
- }
- *attr = buf->buf_attr;
- mutex_unlock(&buf->lock);
- return ret;
- }
- EXPORT_SYMBOL(ubwcp_get_buf_attrs);
- /* Set permanent range translation.
- * enable: Descriptor will be reserved for this buffer until disabled,
- * making lock/unlock quicker.
- * disable: Descriptor will not be reserved for this buffer. Instead,
- * descriptor will be allocated and released for each lock/unlock.
- * If currently allocated but not being used, descriptor will be
- * released.
- */
- int ubwcp_set_perm_range_translation(struct dma_buf *dmabuf, bool enable)
- {
- int ret = 0;
- struct ubwcp_buf *buf;
- if (!dmabuf) {
- ERR("NULL dmabuf input ptr");
- return -EINVAL;
- }
- buf = dma_buf_to_ubwcp_buf(dmabuf);
- if (!buf) {
- ERR("ubwcp_buf not found");
- return -1;
- }
- if (buf->ubwcp->state != UBWCP_STATE_READY) {
- ERR("driver in invalid state: %d", buf->ubwcp->state);
- return -EPERM;
- }
- /* not implemented */
- if (1) {
- ERR("API not implemented yet");
- return -1;
- }
- /* TBD: make sure we acquire buf lock while setting this so there is
- * no race condition with attr_set/lock/unlock
- */
- buf->perm = enable;
- /* if "disable" and we have allocated a desc and it is not being
- * used currently, release it
- */
- if (!enable && buf->desc && !buf->lock_count) {
- ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
- buf->desc = NULL;
- /* Flush/invalidate UBWCP caches */
- //TBD: need to do anything?
- }
- return ret;
- }
- EXPORT_SYMBOL(ubwcp_set_perm_range_translation);
- /**
- * Free up ubwcp resources for this buffer.
- *
- * @param dmabuf : ptr to the dma buf
- *
- * @return int : 0 on success, otherwise error code
- */
- static int ubwcp_free_buffer(struct dma_buf *dmabuf)
- {
- int ret = 0;
- struct ubwcp_buf *buf;
- struct ubwcp_driver *ubwcp;
- unsigned long flags;
- bool is_non_lin_buf;
- trace_ubwcp_free_buffer_start(dmabuf);
- if (!dmabuf) {
- ERR("NULL dmabuf input ptr");
- trace_ubwcp_free_buffer_end(dmabuf);
- return -EINVAL;
- }
- buf = dma_buf_to_ubwcp_buf(dmabuf);
- if (!buf) {
- ERR("ubwcp_buf ptr not found");
- trace_ubwcp_free_buffer_end(dmabuf);
- return -1;
- }
- ubwcp = buf->ubwcp;
- if (ubwcp->state != UBWCP_STATE_READY) {
- ERR("driver in invalid state: %d", ubwcp->state);
- trace_ubwcp_free_buffer_end(dmabuf);
- return -EPERM;
- }
- mutex_lock(&buf->lock);
- is_non_lin_buf = (buf->buf_attr.image_format != UBWCP_LINEAR);
- if (buf->lock_count) {
- DBG("free before unlock (lock_count: %d). unlock()'ing first", buf->lock_count);
- ret = unlock_internal(buf, buf->dma_dir, true);
- if (ret)
- ERR("unlock_internal(): failed : %d, but continuing free()", ret);
- }
- /* if we are still holding a desc, release it. this can happen only if perm == true */
- if (buf->desc) {
- if (!buf->perm) {
- ubwcp->state = UBWCP_STATE_FAULT;
- ERR("state set to fault");
- }
- ubwcp_buf_desc_free(buf->ubwcp, buf->desc);
- buf->desc = NULL;
- }
- if (buf->buf_attr_set)
- reset_buf_attrs(buf);
- spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
- hash_del(&buf->hnode);
- spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
- mutex_unlock(&buf->lock);
- kfree(buf);
- if (is_non_lin_buf)
- dec_num_non_lin_buffers(ubwcp);
- trace_ubwcp_free_buffer_end(dmabuf);
- return ret;
- }
- /* file open: TBD: increment ref count? */
- static int ubwcp_open(struct inode *i, struct file *f)
- {
- return 0;
- }
- /* file open: TBD: decrement ref count? */
- static int ubwcp_close(struct inode *i, struct file *f)
- {
- return 0;
- }
- static int ioctl_set_buf_attr(struct ubwcp_driver *ubwcp, unsigned long ioctl_param)
- {
- int ret;
- struct dma_buf *dmabuf;
- struct ubwcp_ioctl_buffer_attrs buf_attr_ioctl;
- if (copy_from_user(&buf_attr_ioctl, (const void __user *) ioctl_param,
- sizeof(buf_attr_ioctl))) {
- ERR("copy_from_user() failed");
- return -EFAULT;
- }
- DBG("IOCTL: SET_BUF_ATTR: fd = %d", buf_attr_ioctl.fd);
- dmabuf = dma_buf_get(buf_attr_ioctl.fd);
- if (IS_ERR(dmabuf)) {
- ERR("dmabuf ptr not found for dma_buf_fd = %d", buf_attr_ioctl.fd);
- return PTR_ERR(dmabuf);
- }
- ret = ubwcp_set_buf_attrs(dmabuf, &buf_attr_ioctl.attr);
- dma_buf_put(dmabuf);
- return ret;
- }
- static int ioctl_get_hw_ver(struct ubwcp_driver *ubwcp, unsigned long ioctl_param)
- {
- struct ubwcp_ioctl_hw_version hw_ver;
- DBG("IOCTL: GET_HW_VER");
- if (ubwcp_get_hw_version(&hw_ver))
- return -EINVAL;
- if (copy_to_user((void __user *)ioctl_param, &hw_ver, sizeof(hw_ver))) {
- ERR("copy_to_user() failed");
- return -EFAULT;
- }
- return 0;
- }
- static int ioctl_get_stride_align(struct ubwcp_driver *ubwcp, unsigned long ioctl_param)
- {
- struct ubwcp_ioctl_stride_align stride_align_ioctl;
- enum ubwcp_std_image_format format;
- DBG("IOCTL: GET_STRIDE_ALIGN");
- if (copy_from_user(&stride_align_ioctl, (const void __user *) ioctl_param,
- sizeof(stride_align_ioctl))) {
- ERR("copy_from_user() failed");
- return -EFAULT;
- }
- if (stride_align_ioctl.unused != 0) {
- ERR("unused values must be set to 0");
- return -EINVAL;
- }
- if (!ioctl_format_is_valid(stride_align_ioctl.image_format)) {
- ERR("invalid image format: %d", stride_align_ioctl.image_format);
- return -EINVAL;
- }
- if (stride_align_ioctl.image_format == UBWCP_LINEAR) {
- ERR("not supported for LINEAR format");
- return -EINVAL;
- }
- if (to_std_format(stride_align_ioctl.image_format, &format)) {
- ERR("Unable to map ioctl image format to std image format");
- return -EINVAL;
- }
- if (get_stride_alignment(format, &stride_align_ioctl.stride_align)) {
- ERR("failed for format: %d", format);
- return -EFAULT;
- }
- if (copy_to_user((void __user *)ioctl_param, &stride_align_ioctl,
- sizeof(stride_align_ioctl))) {
- ERR("copy_to_user() failed");
- return -EFAULT;
- }
- return 0;
- }
- static int ioctl_validate_stride(struct ubwcp_driver *ubwcp, unsigned long ioctl_param)
- {
- struct ubwcp_ioctl_validate_stride validate_stride_ioctl;
- enum ubwcp_std_image_format format;
- DBG("IOCTL: VALIDATE_STRIDE");
- if (copy_from_user(&validate_stride_ioctl, (const void __user *) ioctl_param,
- sizeof(validate_stride_ioctl))) {
- ERR("copy_from_user() failed");
- return -EFAULT;
- }
- if (validate_stride_ioctl.unused1 || validate_stride_ioctl.unused2) {
- ERR("unused values must be set to 0");
- return -EINVAL;
- }
- if (!ioctl_format_is_valid(validate_stride_ioctl.image_format)) {
- ERR("not supported for LINEAR format");
- return -EINVAL;
- }
- if (validate_stride_ioctl.image_format == UBWCP_LINEAR) {
- ERR("not supported for LINEAR format");
- return -EINVAL;
- }
- if (to_std_format(validate_stride_ioctl.image_format, &format)) {
- ERR("Unable to map ioctl image format to std image format");
- return -EINVAL;
- }
- validate_stride_ioctl.valid = stride_is_valid(ubwcp, format, validate_stride_ioctl.width,
- validate_stride_ioctl.stride);
- if (copy_to_user((void __user *)ioctl_param, &validate_stride_ioctl,
- sizeof(validate_stride_ioctl))) {
- ERR("copy_to_user() failed");
- return -EFAULT;
- }
- return 0;
- }
- /* handle IOCTLs */
- static long ubwcp_ioctl(struct file *file, unsigned int ioctl_num, unsigned long ioctl_param)
- {
- struct ubwcp_driver *ubwcp;
- ubwcp = ubwcp_get_driver();
- if (!ubwcp)
- return -EINVAL;
- if (ubwcp->state != UBWCP_STATE_READY) {
- ERR("driver in invalid state: %d", ubwcp->state);
- return -EPERM;
- }
- switch (ioctl_num) {
- case UBWCP_IOCTL_SET_BUF_ATTR:
- return ioctl_set_buf_attr(ubwcp, ioctl_param);
- case UBWCP_IOCTL_GET_HW_VER:
- return ioctl_get_hw_ver(ubwcp, ioctl_param);
- case UBWCP_IOCTL_GET_STRIDE_ALIGN:
- return ioctl_get_stride_align(ubwcp, ioctl_param);
- case UBWCP_IOCTL_VALIDATE_STRIDE:
- return ioctl_validate_stride(ubwcp, ioctl_param);
- default:
- ERR("Invalid ioctl_num = %d", ioctl_num);
- return -EINVAL;
- }
- return 0;
- }
- static const struct file_operations ubwcp_fops = {
- .owner = THIS_MODULE,
- .open = ubwcp_open,
- .release = ubwcp_close,
- .unlocked_ioctl = ubwcp_ioctl,
- };
- static int read_err_r_op(void *data, u64 *value)
- {
- struct ubwcp_driver *ubwcp = data;
- *value = ubwcp->read_err_irq_en;
- return 0;
- }
- static int read_err_w_op(void *data, u64 value)
- {
- struct ubwcp_driver *ubwcp = data;
- if (ubwcp->state != UBWCP_STATE_READY)
- return -EPERM;
- if (ubwcp_power(ubwcp, true))
- goto err;
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, value);
- ubwcp->read_err_irq_en = value;
- if (ubwcp_power(ubwcp, false))
- goto err;
- return 0;
- err:
- ubwcp->state = UBWCP_STATE_FAULT;
- ERR("state set to fault");
- return -1;
- }
- static int write_err_r_op(void *data, u64 *value)
- {
- struct ubwcp_driver *ubwcp = data;
- if (ubwcp->state != UBWCP_STATE_READY)
- return -EPERM;
- *value = ubwcp->write_err_irq_en;
- return 0;
- }
- static int write_err_w_op(void *data, u64 value)
- {
- struct ubwcp_driver *ubwcp = data;
- if (ubwcp->state != UBWCP_STATE_READY)
- return -EPERM;
- if (ubwcp_power(ubwcp, true))
- goto err;
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, value);
- ubwcp->write_err_irq_en = value;
- if (ubwcp_power(ubwcp, false))
- goto err;
- return 0;
- err:
- ubwcp->state = UBWCP_STATE_FAULT;
- ERR("state set to fault");
- return -1;
- }
- static int decode_err_r_op(void *data, u64 *value)
- {
- struct ubwcp_driver *ubwcp = data;
- if (ubwcp->state != UBWCP_STATE_READY)
- return -EPERM;
- *value = ubwcp->decode_err_irq_en;
- return 0;
- }
- static int decode_err_w_op(void *data, u64 value)
- {
- struct ubwcp_driver *ubwcp = data;
- if (ubwcp->state != UBWCP_STATE_READY)
- return -EPERM;
- if (ubwcp_power(ubwcp, true))
- goto err;
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, value);
- ubwcp->decode_err_irq_en = value;
- if (ubwcp_power(ubwcp, false))
- goto err;
- return 0;
- err:
- ubwcp->state = UBWCP_STATE_FAULT;
- ERR("state set to fault");
- return -1;
- }
- static int encode_err_r_op(void *data, u64 *value)
- {
- struct ubwcp_driver *ubwcp = data;
- if (ubwcp->state != UBWCP_STATE_READY)
- return -EPERM;
- *value = ubwcp->encode_err_irq_en;
- return 0;
- }
- static int encode_err_w_op(void *data, u64 value)
- {
- struct ubwcp_driver *ubwcp = data;
- if (ubwcp->state != UBWCP_STATE_READY)
- return -EPERM;
- if (ubwcp_power(ubwcp, true))
- goto err;
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, value);
- ubwcp->encode_err_irq_en = value;
- if (ubwcp_power(ubwcp, false))
- goto err;
- return 0;
- err:
- ubwcp->state = UBWCP_STATE_FAULT;
- ERR("state set to fault");
- return -1;
- }
- static int reg_rw_trace_w_op(void *data, u64 value)
- {
- struct ubwcp_driver *ubwcp = data;
- if (ubwcp->state != UBWCP_STATE_READY)
- return -EPERM;
- ubwcp_hw_trace_set(value);
- return 0;
- }
- static int reg_rw_trace_r_op(void *data, u64 *value)
- {
- struct ubwcp_driver *ubwcp = data;
- bool trace_status;
- if (ubwcp->state != UBWCP_STATE_READY)
- return -EPERM;
- ubwcp_hw_trace_get(&trace_status);
- *value = trace_status;
- return 0;
- }
- DEFINE_DEBUGFS_ATTRIBUTE(read_err_fops, read_err_r_op, read_err_w_op, "%d\n");
- DEFINE_DEBUGFS_ATTRIBUTE(decode_err_fops, decode_err_r_op, decode_err_w_op, "%d\n");
- DEFINE_DEBUGFS_ATTRIBUTE(write_err_fops, write_err_r_op, write_err_w_op, "%d\n");
- DEFINE_DEBUGFS_ATTRIBUTE(encode_err_fops, encode_err_r_op, encode_err_w_op, "%d\n");
- DEFINE_DEBUGFS_ATTRIBUTE(reg_rw_trace_fops, reg_rw_trace_r_op, reg_rw_trace_w_op, "%d\n");
- static void ubwcp_debugfs_init(struct ubwcp_driver *ubwcp)
- {
- struct dentry *debugfs_root;
- struct dentry *dfile;
- debugfs_root = debugfs_create_dir("ubwcp", NULL);
- if (IS_ERR_OR_NULL(debugfs_root)) {
- ERR("Failed to create debugfs for ubwcp\n");
- return;
- }
- debugfs_create_u32("debug_trace_enable", 0644, debugfs_root, &ubwcp_debug_trace_enable);
- dfile = debugfs_create_file("reg_rw_trace_en", 0644, debugfs_root, ubwcp, ®_rw_trace_fops);
- if (IS_ERR_OR_NULL(dfile)) {
- ERR("failed to create reg_rw_trace_en debugfs file");
- goto err;
- }
- dfile = debugfs_create_file("read_err_irq_en", 0644, debugfs_root, ubwcp, &read_err_fops);
- if (IS_ERR_OR_NULL(dfile)) {
- ERR("failed to create read_err_irq debugfs file");
- goto err;
- }
- dfile = debugfs_create_file("write_err_irq_en", 0644, debugfs_root, ubwcp, &write_err_fops);
- if (IS_ERR_OR_NULL(dfile)) {
- ERR("failed to create write_err_irq debugfs file");
- goto err;
- }
- dfile = debugfs_create_file("decode_err_irq_en", 0644, debugfs_root, ubwcp,
- &decode_err_fops);
- if (IS_ERR_OR_NULL(dfile)) {
- ERR("failed to create decode_err_irq debugfs file");
- goto err;
- }
- dfile = debugfs_create_file("encode_err_irq_en", 0644, debugfs_root, ubwcp,
- &encode_err_fops);
- if (IS_ERR_OR_NULL(dfile)) {
- ERR("failed to create encode_err_irq debugfs file");
- goto err;
- }
- ubwcp->debugfs_root = debugfs_root;
- return;
- err:
- debugfs_remove_recursive(ubwcp->debugfs_root);
- ubwcp->debugfs_root = NULL;
- }
- static void ubwcp_debugfs_deinit(struct ubwcp_driver *ubwcp)
- {
- debugfs_remove_recursive(ubwcp->debugfs_root);
- }
- /* ubwcp char device initialization */
- static int ubwcp_cdev_init(struct ubwcp_driver *ubwcp)
- {
- int ret;
- dev_t devt;
- struct class *dev_class;
- struct device *dev_sys;
- /* allocate major device number (/proc/devices -> major_num ubwcp) */
- ret = alloc_chrdev_region(&devt, 0, UBWCP_NUM_DEVICES, UBWCP_DEVICE_NAME);
- if (ret) {
- ERR("alloc_chrdev_region() failed: %d", ret);
- return ret;
- }
- /* create device class (/sys/class/ubwcp_class) */
- dev_class = class_create(THIS_MODULE, "ubwcp_class");
- if (IS_ERR(dev_class)) {
- ret = PTR_ERR(dev_class);
- ERR("class_create() failed, ret: %d", ret);
- goto err;
- }
- /* Create device and register with sysfs
- * (/sys/class/ubwcp_class/ubwcp/... -> dev/power/subsystem/uevent)
- */
- dev_sys = device_create(dev_class, NULL, devt, NULL,
- UBWCP_DEVICE_NAME);
- if (IS_ERR(dev_sys)) {
- ret = PTR_ERR(dev_sys);
- ERR("device_create() failed, ret: %d", ret);
- goto err_device_create;
- }
- /* register file operations and get cdev */
- cdev_init(&ubwcp->cdev, &ubwcp_fops);
- /* associate cdev and device major/minor with file system
- * can do file ops on /dev/ubwcp after this
- */
- ret = cdev_add(&ubwcp->cdev, devt, 1);
- if (ret) {
- ERR("cdev_add() failed, ret: %d", ret);
- goto err_cdev_add;
- }
- ubwcp->devt = devt;
- ubwcp->dev_class = dev_class;
- ubwcp->dev_sys = dev_sys;
- return 0;
- err_cdev_add:
- device_destroy(dev_class, devt);
- err_device_create:
- class_destroy(dev_class);
- err:
- unregister_chrdev_region(devt, UBWCP_NUM_DEVICES);
- return ret;
- }
- static void ubwcp_cdev_deinit(struct ubwcp_driver *ubwcp)
- {
- device_destroy(ubwcp->dev_class, ubwcp->devt);
- class_destroy(ubwcp->dev_class);
- cdev_del(&ubwcp->cdev);
- unregister_chrdev_region(ubwcp->devt, UBWCP_NUM_DEVICES);
- }
- struct handler_node {
- struct list_head list;
- u32 client_id;
- ubwcp_error_handler_t handler;
- void *data;
- };
- int ubwcp_register_error_handler(u32 client_id, ubwcp_error_handler_t handler,
- void *data)
- {
- struct handler_node *node;
- unsigned long flags;
- struct ubwcp_driver *ubwcp = ubwcp_get_driver();
- if (!ubwcp)
- return -EINVAL;
- if (client_id != -1)
- return -EINVAL;
- if (!handler)
- return -EINVAL;
- if (ubwcp->state != UBWCP_STATE_READY)
- return -EPERM;
- node = kzalloc(sizeof(*node), GFP_KERNEL);
- if (!node)
- return -ENOMEM;
- node->client_id = client_id;
- node->handler = handler;
- node->data = data;
- spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
- list_add_tail(&node->list, &ubwcp->err_handler_list);
- spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
- return 0;
- }
- EXPORT_SYMBOL(ubwcp_register_error_handler);
- static void ubwcp_notify_error_handlers(struct ubwcp_err_info *err)
- {
- struct handler_node *node;
- unsigned long flags;
- struct ubwcp_driver *ubwcp = ubwcp_get_driver();
- if (!ubwcp)
- return;
- spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
- list_for_each_entry(node, &ubwcp->err_handler_list, list)
- node->handler(err, node->data);
- spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
- }
- int ubwcp_unregister_error_handler(u32 client_id)
- {
- int ret = -EINVAL;
- struct handler_node *node;
- unsigned long flags;
- struct ubwcp_driver *ubwcp = ubwcp_get_driver();
- if (!ubwcp)
- return -EINVAL;
- if (ubwcp->state != UBWCP_STATE_INVALID)
- return -EPERM;
- spin_lock_irqsave(&ubwcp->err_handler_list_lock, flags);
- list_for_each_entry(node, &ubwcp->err_handler_list, list)
- if (node->client_id == client_id) {
- list_del(&node->list);
- kfree(node);
- ret = 0;
- break;
- }
- spin_unlock_irqrestore(&ubwcp->err_handler_list_lock, flags);
- return ret;
- }
- EXPORT_SYMBOL(ubwcp_unregister_error_handler);
- /* get ubwcp_buf corresponding to the ULA PA*/
- static struct dma_buf *get_dma_buf_from_ulapa(phys_addr_t addr)
- {
- struct ubwcp_buf *buf = NULL;
- struct dma_buf *ret_buf = NULL;
- struct ubwcp_driver *ubwcp = ubwcp_get_driver();
- unsigned long flags;
- u32 i;
- if (!ubwcp)
- return NULL;
- spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
- hash_for_each(ubwcp->buf_table, i, buf, hnode) {
- if (buf->ula_pa <= addr && addr < buf->ula_pa + buf->ula_size) {
- ret_buf = buf->dma_buf;
- break;
- }
- }
- spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
- return ret_buf;
- }
- /* get ubwcp_buf corresponding to the IOVA*/
- static struct dma_buf *get_dma_buf_from_iova(unsigned long addr)
- {
- struct ubwcp_buf *buf = NULL;
- struct dma_buf *ret_buf = NULL;
- struct ubwcp_driver *ubwcp = ubwcp_get_driver();
- unsigned long flags;
- u32 i;
- if (!ubwcp)
- return NULL;
- spin_lock_irqsave(&ubwcp->buf_table_lock, flags);
- hash_for_each(ubwcp->buf_table, i, buf, hnode) {
- unsigned long iova_base;
- unsigned int iova_size;
- if (!buf->sgt)
- continue;
- iova_base = sg_dma_address(buf->sgt->sgl);
- iova_size = sg_dma_len(buf->sgt->sgl);
- if (iova_base <= addr && addr < iova_base + iova_size) {
- ret_buf = buf->dma_buf;
- break;
- }
- }
- spin_unlock_irqrestore(&ubwcp->buf_table_lock, flags);
- return ret_buf;
- }
- int ubwcp_iommu_fault_handler(struct iommu_domain *domain, struct device *dev,
- unsigned long iova, int flags, void *data)
- {
- int ret = 0;
- struct ubwcp_err_info err;
- struct ubwcp_driver *ubwcp = ubwcp_get_driver();
- struct device *cb_dev = (struct device *)data;
- if (!ubwcp) {
- ret = -EINVAL;
- goto err;
- }
- err.err_code = UBWCP_SMMU_FAULT;
- if (cb_dev == ubwcp->dev_desc_cb)
- err.smmu_err.iommu_dev_id = UBWCP_DESC_CB_ID;
- else if (cb_dev == ubwcp->dev_buf_cb)
- err.smmu_err.iommu_dev_id = UBWCP_BUF_CB_ID;
- else
- err.smmu_err.iommu_dev_id = UBWCP_UNKNOWN_CB_ID;
- err.smmu_err.dmabuf = get_dma_buf_from_iova(iova);
- err.smmu_err.iova = iova;
- err.smmu_err.iommu_fault_flags = flags;
- ERR("ubwcp_err: err code: %d (smmu), iommu_dev_id: %d, iova: 0x%llx, flags: 0x%x",
- err.err_code, err.smmu_err.iommu_dev_id, err.smmu_err.iova,
- err.smmu_err.iommu_fault_flags);
- ubwcp_notify_error_handlers(&err);
- err:
- return ret;
- }
- static irqreturn_t ubwcp_irq_handler(int irq, void *ptr)
- {
- struct ubwcp_driver *ubwcp;
- void __iomem *base;
- phys_addr_t addr;
- struct ubwcp_err_info err;
- ubwcp = (struct ubwcp_driver *) ptr;
- base = ubwcp->base;
- if (irq == ubwcp->irq_range_ck_rd) {
- addr = ubwcp_hw_interrupt_src_address(base, 0) << 6;
- err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
- err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
- err.translation_err.ula_pa = addr;
- err.translation_err.read = true;
- ERR("err_code: %d (range read), dmabuf: 0x%llx, read: %d, addr: 0x%llx",
- err.err_code, err.translation_err.dmabuf, err.translation_err.read, addr);
- ubwcp_notify_error_handlers(&err);
- ubwcp_hw_interrupt_clear(ubwcp->base, 0);
- } else if (irq == ubwcp->irq_range_ck_wr) {
- addr = ubwcp_hw_interrupt_src_address(base, 1) << 6;
- err.err_code = UBWCP_RANGE_TRANSLATION_ERROR;
- err.translation_err.dmabuf = get_dma_buf_from_ulapa(addr);
- err.translation_err.ula_pa = addr;
- err.translation_err.read = false;
- ERR("err_code: %d (range write), dmabuf: 0x%llx, read: %d, addr: 0x%llx",
- err.err_code, err.translation_err.dmabuf, err.translation_err.read, addr);
- ubwcp_notify_error_handlers(&err);
- ubwcp_hw_interrupt_clear(ubwcp->base, 1);
- } else if (irq == ubwcp->irq_encode) {
- addr = ubwcp_hw_interrupt_src_address(base, 3) << 6;
- err.err_code = UBWCP_ENCODE_ERROR;
- err.enc_err.dmabuf = get_dma_buf_from_ulapa(addr);
- err.enc_err.ula_pa = addr;
- ERR("err_code: %d (encode), dmabuf: 0x%llx, addr: 0x%llx",
- err.err_code, err.enc_err.dmabuf, addr);
- ubwcp_notify_error_handlers(&err);
- ubwcp_hw_interrupt_clear(ubwcp->base, 3);
- } else if (irq == ubwcp->irq_decode) {
- addr = ubwcp_hw_interrupt_src_address(base, 2) << 6;
- err.err_code = UBWCP_DECODE_ERROR;
- err.dec_err.dmabuf = get_dma_buf_from_ulapa(addr);
- err.dec_err.ula_pa = addr;
- ERR("err_code: %d (decode), dmabuf: 0x%llx, addr: 0x%llx",
- err.err_code, err.enc_err.dmabuf, addr);
- ubwcp_notify_error_handlers(&err);
- ubwcp_hw_interrupt_clear(ubwcp->base, 2);
- } else {
- ERR("unknown irq: %d", irq);
- return IRQ_NONE;
- }
- return IRQ_HANDLED;
- }
- static int ubwcp_interrupt_register(struct platform_device *pdev, struct ubwcp_driver *ubwcp)
- {
- int ret = 0;
- struct device *dev = &pdev->dev;
- ubwcp->irq_range_ck_rd = platform_get_irq(pdev, 0);
- if (ubwcp->irq_range_ck_rd < 0)
- return ubwcp->irq_range_ck_rd;
- ubwcp->irq_range_ck_wr = platform_get_irq(pdev, 1);
- if (ubwcp->irq_range_ck_wr < 0)
- return ubwcp->irq_range_ck_wr;
- ubwcp->irq_encode = platform_get_irq(pdev, 2);
- if (ubwcp->irq_encode < 0)
- return ubwcp->irq_encode;
- ubwcp->irq_decode = platform_get_irq(pdev, 3);
- if (ubwcp->irq_decode < 0)
- return ubwcp->irq_decode;
- DBG("got irqs: %d %d %d %d", ubwcp->irq_range_ck_rd,
- ubwcp->irq_range_ck_wr,
- ubwcp->irq_encode,
- ubwcp->irq_decode);
- ret = devm_request_irq(dev, ubwcp->irq_range_ck_rd, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
- if (ret) {
- ERR("request_irq() failed. irq: %d ret: %d",
- ubwcp->irq_range_ck_rd, ret);
- return ret;
- }
- ret = devm_request_irq(dev, ubwcp->irq_range_ck_wr, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
- if (ret) {
- ERR("request_irq() failed. irq: %d ret: %d",
- ubwcp->irq_range_ck_wr, ret);
- return ret;
- }
- ret = devm_request_irq(dev, ubwcp->irq_encode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
- if (ret) {
- ERR("request_irq() failed. irq: %d ret: %d",
- ubwcp->irq_encode, ret);
- return ret;
- }
- ret = devm_request_irq(dev, ubwcp->irq_decode, ubwcp_irq_handler, 0, "ubwcp", ubwcp);
- if (ret) {
- ERR("request_irq() failed. irq: %d ret: %d",
- ubwcp->irq_decode, ret);
- return ret;
- }
- return ret;
- }
- /* ubwcp device probe */
- static int qcom_ubwcp_probe(struct platform_device *pdev)
- {
- int ret = 0;
- struct ubwcp_driver *ubwcp;
- struct device *ubwcp_dev = &pdev->dev;
- ubwcp = devm_kzalloc(ubwcp_dev, sizeof(*ubwcp), GFP_KERNEL);
- if (!ubwcp) {
- ERR("devm_kzalloc() failed");
- return -ENOMEM;
- }
- ubwcp->dev = &pdev->dev;
- ret = dma_set_mask_and_coherent(ubwcp->dev, DMA_BIT_MASK(64));
- ubwcp->base = devm_platform_ioremap_resource(pdev, 0);
- if (IS_ERR(ubwcp->base)) {
- ERR("devm ioremap() failed: %d", PTR_ERR(ubwcp->base));
- return PTR_ERR(ubwcp->base);
- }
- DBG("ubwcp->base: %p", ubwcp->base);
- ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 0, &ubwcp->ula_pool_base);
- if (ret) {
- ERR("failed reading ula_range (base): %d", ret);
- return ret;
- }
- DBG("ubwcp: ula_range: base = 0x%lx", ubwcp->ula_pool_base);
- ret = of_property_read_u64_index(ubwcp_dev->of_node, "ula_range", 1, &ubwcp->ula_pool_size);
- if (ret) {
- ERR("failed reading ula_range (size): %d", ret);
- return ret;
- }
- DBG("ubwcp: ula_range: size = 0x%lx", ubwcp->ula_pool_size);
- INIT_LIST_HEAD(&ubwcp->err_handler_list);
- /* driver initial state */
- ubwcp->state = UBWCP_STATE_INVALID;
- atomic_set(&ubwcp->num_non_lin_buffers, 0);
- ubwcp->mem_online = false;
- mutex_init(&ubwcp->desc_lock);
- spin_lock_init(&ubwcp->buf_table_lock);
- mutex_init(&ubwcp->mem_hotplug_lock);
- mutex_init(&ubwcp->ula_lock);
- mutex_init(&ubwcp->ubwcp_flush_lock);
- mutex_init(&ubwcp->hw_range_ck_lock);
- mutex_init(&ubwcp->power_ctrl_lock);
- spin_lock_init(&ubwcp->err_handler_list_lock);
- /* Regulator */
- ubwcp->vdd = devm_regulator_get(ubwcp_dev, "vdd");
- if (IS_ERR_OR_NULL(ubwcp->vdd)) {
- ret = PTR_ERR(ubwcp->vdd);
- ERR("devm_regulator_get() failed: %d", ret);
- return ret;
- }
- ret = ubwcp_init_clocks(ubwcp, ubwcp_dev);
- if (ret) {
- ERR("failed to initialize ubwcp clocks err: %d", ret);
- return ret;
- }
- if (ubwcp_power(ubwcp, true))
- return -1;
- if (ubwcp_cdev_init(ubwcp))
- return -1;
- /* disable all interrupts (reset value has some interrupts enabled by default) */
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
- if (ubwcp_interrupt_register(pdev, ubwcp))
- return -1;
- ubwcp_debugfs_init(ubwcp);
- /* create ULA pool */
- ubwcp->ula_pool = gen_pool_create(PAGE_SHIFT, -1);
- if (!ubwcp->ula_pool) {
- ERR("failed gen_pool_create()");
- ret = -1;
- goto err_pool_create;
- }
- ret = gen_pool_add(ubwcp->ula_pool, ubwcp->ula_pool_base, ubwcp->ula_pool_size, -1);
- if (ret) {
- ERR("failed gen_pool_add(): %d", ret);
- ret = -1;
- goto err_pool_add;
- }
- /* register the default config mmap function. */
- ubwcp->mmap_config_fptr = msm_ubwcp_dma_buf_configure_mmap;
- hash_init(ubwcp->buf_table);
- ubwcp_buf_desc_list_init(ubwcp);
- image_format_init(ubwcp);
- /* one time hw init */
- ubwcp_hw_one_time_init(ubwcp->base);
- ubwcp_hw_version(ubwcp->base, &ubwcp->hw_ver_major, &ubwcp->hw_ver_minor);
- pr_err("ubwcp: hw version: major %d, minor %d\n", ubwcp->hw_ver_major, ubwcp->hw_ver_minor);
- if (ubwcp->hw_ver_major == 0) {
- ERR("Failed to read HW version");
- ret = -1;
- goto err_pool_add;
- }
- /* set pdev->dev->driver_data = ubwcp */
- platform_set_drvdata(pdev, ubwcp);
- /* enable interrupts */
- if (ubwcp->read_err_irq_en)
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, true);
- if (ubwcp->write_err_irq_en)
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, true);
- if (ubwcp->decode_err_irq_en)
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, true);
- if (ubwcp->encode_err_irq_en)
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, true);
- /* Turn OFF until buffers are allocated */
- if (ubwcp_power(ubwcp, false)) {
- ret = -1;
- goto err_power_off;
- }
- ret = msm_ubwcp_set_ops(ubwcp_init_buffer, ubwcp_free_buffer, ubwcp_lock, ubwcp_unlock);
- if (ret) {
- ERR("msm_ubwcp_set_ops() failed: %d", ret);
- goto err_power_off;
- } else {
- DBG("msm_ubwcp_set_ops(): success"); }
- me = ubwcp;
- return ret;
- err_power_off:
- if (!ubwcp_power(ubwcp, true)) {
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
- ubwcp_power(ubwcp, false);
- }
- err_pool_add:
- gen_pool_destroy(ubwcp->ula_pool);
- err_pool_create:
- ubwcp_debugfs_deinit(ubwcp);
- ubwcp_cdev_deinit(ubwcp);
- return ret;
- }
- /* buffer context bank device probe */
- static int ubwcp_probe_cb_buf(struct platform_device *pdev)
- {
- struct ubwcp_driver *ubwcp;
- struct iommu_domain *domain = NULL;
- ubwcp = dev_get_drvdata(pdev->dev.parent);
- if (!ubwcp) {
- ERR("failed to get ubwcp ptr");
- return -EINVAL;
- }
- ubwcp->dev_buf_cb = &pdev->dev;
- domain = iommu_get_domain_for_dev(ubwcp->dev_buf_cb);
- if (domain)
- iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_buf_cb);
- if (ubwcp->dev_desc_cb)
- ubwcp->state = UBWCP_STATE_READY;
- return 0;
- }
- /* descriptor context bank device probe */
- static int ubwcp_probe_cb_desc(struct platform_device *pdev)
- {
- int ret = 0;
- struct ubwcp_driver *ubwcp;
- struct iommu_domain *domain = NULL;
- ubwcp = dev_get_drvdata(pdev->dev.parent);
- if (!ubwcp) {
- ERR("failed to get ubwcp ptr");
- return -EINVAL;
- }
- ubwcp->buffer_desc_size = UBWCP_BUFFER_DESC_OFFSET *
- UBWCP_BUFFER_DESC_COUNT;
- ubwcp->dev_desc_cb = &pdev->dev;
- dma_set_max_seg_size(ubwcp->dev_desc_cb, DMA_BIT_MASK(32));
- dma_set_seg_boundary(ubwcp->dev_desc_cb, (unsigned long)DMA_BIT_MASK(64));
- /* Allocate buffer descriptors. UBWCP is iocoherent device.
- * Thus we don't need to flush after updates to buffer descriptors.
- */
- ubwcp->buffer_desc_base = dma_alloc_coherent(ubwcp->dev_desc_cb,
- ubwcp->buffer_desc_size,
- &ubwcp->buffer_desc_dma_handle,
- GFP_KERNEL);
- if (!ubwcp->buffer_desc_base) {
- ERR("failed to allocate desc buffer");
- return -ENOMEM;
- }
- DBG("desc_base = %p size = %zu", ubwcp->buffer_desc_base,
- ubwcp->buffer_desc_size);
- ret = ubwcp_power(ubwcp, true);
- if (ret) {
- ERR("failed to power on");
- goto err;
- }
- ubwcp_hw_set_buf_desc(ubwcp->base, (u64) ubwcp->buffer_desc_dma_handle,
- UBWCP_BUFFER_DESC_OFFSET);
- ret = ubwcp_power(ubwcp, false);
- if (ret) {
- ERR("failed to power off");
- goto err;
- }
- domain = iommu_get_domain_for_dev(ubwcp->dev_desc_cb);
- if (domain)
- iommu_set_fault_handler(domain, ubwcp_iommu_fault_handler, ubwcp->dev_desc_cb);
- if (ubwcp->dev_buf_cb)
- ubwcp->state = UBWCP_STATE_READY;
- return ret;
- err:
- dma_free_coherent(ubwcp->dev_desc_cb,
- ubwcp->buffer_desc_size,
- ubwcp->buffer_desc_base,
- ubwcp->buffer_desc_dma_handle);
- ubwcp->buffer_desc_base = NULL;
- ubwcp->buffer_desc_dma_handle = 0;
- ubwcp->dev_desc_cb = NULL;
- return -1;
- }
- /* buffer context bank device remove */
- static int ubwcp_remove_cb_buf(struct platform_device *pdev)
- {
- struct ubwcp_driver *ubwcp;
- ubwcp = dev_get_drvdata(pdev->dev.parent);
- if (!ubwcp) {
- ERR("failed to get ubwcp ptr");
- return -EINVAL;
- }
- ubwcp->state = UBWCP_STATE_INVALID;
- ubwcp->dev_buf_cb = NULL;
- return 0;
- }
- /* descriptor context bank device remove */
- static int ubwcp_remove_cb_desc(struct platform_device *pdev)
- {
- struct ubwcp_driver *ubwcp;
- ubwcp = dev_get_drvdata(pdev->dev.parent);
- if (!ubwcp) {
- ERR("failed to get ubwcp ptr");
- return -EINVAL;
- }
- if (!ubwcp->dev_desc_cb) {
- ERR("ubwcp->dev_desc_cb == NULL");
- return -1;
- }
- if (!ubwcp_power(ubwcp, true)) {
- ubwcp_hw_set_buf_desc(ubwcp->base, 0x0, 0x0);
- ubwcp_power(ubwcp, false);
- }
- ubwcp->state = UBWCP_STATE_INVALID;
- dma_free_coherent(ubwcp->dev_desc_cb,
- ubwcp->buffer_desc_size,
- ubwcp->buffer_desc_base,
- ubwcp->buffer_desc_dma_handle);
- ubwcp->buffer_desc_base = NULL;
- ubwcp->buffer_desc_dma_handle = 0;
- return 0;
- }
- /* ubwcp device remove */
- static int qcom_ubwcp_remove(struct platform_device *pdev)
- {
- size_t avail;
- size_t psize;
- struct ubwcp_driver *ubwcp;
- /* get pdev->dev->driver_data = ubwcp */
- ubwcp = platform_get_drvdata(pdev);
- if (!ubwcp) {
- ERR("ubwcp == NULL");
- return -1;
- }
- if (!ubwcp_power(ubwcp, true)) {
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_READ_ERROR, false);
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_WRITE_ERROR, false);
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_ENCODE_ERROR, false);
- ubwcp_hw_interrupt_enable(ubwcp->base, INTERRUPT_DECODE_ERROR, false);
- ubwcp_power(ubwcp, false);
- }
- ubwcp->state = UBWCP_STATE_INVALID;
- /* before destroying, make sure pool is empty. otherwise pool_destroy() panics. */
- avail = gen_pool_avail(ubwcp->ula_pool);
- psize = gen_pool_size(ubwcp->ula_pool);
- if (psize != avail) {
- ERR("gen_pool is not empty! avail: %zx size: %zx", avail, psize);
- ERR("skipping pool destroy....cause it will PANIC. Fix this!!!!");
- } else {
- gen_pool_destroy(ubwcp->ula_pool);
- }
- ubwcp_debugfs_deinit(ubwcp);
- ubwcp_cdev_deinit(ubwcp);
- return 0;
- }
- /* top level ubwcp device probe function */
- static int ubwcp_probe(struct platform_device *pdev)
- {
- const char *compatible = "";
- trace_ubwcp_probe(pdev);
- if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
- return qcom_ubwcp_probe(pdev);
- else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
- return ubwcp_probe_cb_desc(pdev);
- else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
- return ubwcp_probe_cb_buf(pdev);
- of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
- ERR("unknown device: %s", compatible);
- return -EINVAL;
- }
- /* top level ubwcp device remove function */
- static int ubwcp_remove(struct platform_device *pdev)
- {
- const char *compatible = "";
- trace_ubwcp_remove(pdev);
- /* TBD: what if buffers are still allocated? locked? etc.
- * also should turn off power?
- */
- if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp"))
- return qcom_ubwcp_remove(pdev);
- else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-desc"))
- return ubwcp_remove_cb_desc(pdev);
- else if (of_device_is_compatible(pdev->dev.of_node, "qcom,ubwcp-context-bank-buf"))
- return ubwcp_remove_cb_buf(pdev);
- of_property_read_string(pdev->dev.of_node, "compatible", &compatible);
- ERR("unknown device: %s", compatible);
- return -EINVAL;
- }
- static const struct of_device_id ubwcp_dt_match[] = {
- {.compatible = "qcom,ubwcp"},
- {.compatible = "qcom,ubwcp-context-bank-desc"},
- {.compatible = "qcom,ubwcp-context-bank-buf"},
- {}
- };
- struct platform_driver ubwcp_platform_driver = {
- .probe = ubwcp_probe,
- .remove = ubwcp_remove,
- .driver = {
- .name = "qcom,ubwcp",
- .of_match_table = ubwcp_dt_match,
- },
- };
- int ubwcp_init(void)
- {
- int ret = 0;
- DBG("+++++++++++");
- ret = platform_driver_register(&ubwcp_platform_driver);
- if (ret)
- ERR("platform_driver_register() failed: %d", ret);
- return ret;
- }
- void ubwcp_exit(void)
- {
- platform_driver_unregister(&ubwcp_platform_driver);
- DBG("-----------");
- }
- module_init(ubwcp_init);
- module_exit(ubwcp_exit);
- MODULE_LICENSE("GPL");
|