drm_vblank.c 68 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178
  1. /*
  2. * drm_irq.c IRQ and vblank support
  3. *
  4. * \author Rickard E. (Rik) Faith <[email protected]>
  5. * \author Gareth Hughes <[email protected]>
  6. *
  7. * Permission is hereby granted, free of charge, to any person obtaining a
  8. * copy of this software and associated documentation files (the "Software"),
  9. * to deal in the Software without restriction, including without limitation
  10. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11. * and/or sell copies of the Software, and to permit persons to whom the
  12. * Software is furnished to do so, subject to the following conditions:
  13. *
  14. * The above copyright notice and this permission notice (including the next
  15. * paragraph) shall be included in all copies or substantial portions of the
  16. * Software.
  17. *
  18. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  21. * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  24. * OTHER DEALINGS IN THE SOFTWARE.
  25. */
  26. #include <linux/export.h>
  27. #include <linux/kthread.h>
  28. #include <linux/moduleparam.h>
  29. #include <drm/drm_crtc.h>
  30. #include <drm/drm_drv.h>
  31. #include <drm/drm_framebuffer.h>
  32. #include <drm/drm_managed.h>
  33. #include <drm/drm_modeset_helper_vtables.h>
  34. #include <drm/drm_print.h>
  35. #include <drm/drm_vblank.h>
  36. #include "drm_internal.h"
  37. #include "drm_trace.h"
  38. /**
  39. * DOC: vblank handling
  40. *
  41. * From the computer's perspective, every time the monitor displays
  42. * a new frame the scanout engine has "scanned out" the display image
  43. * from top to bottom, one row of pixels at a time. The current row
  44. * of pixels is referred to as the current scanline.
  45. *
  46. * In addition to the display's visible area, there's usually a couple of
  47. * extra scanlines which aren't actually displayed on the screen.
  48. * These extra scanlines don't contain image data and are occasionally used
  49. * for features like audio and infoframes. The region made up of these
  50. * scanlines is referred to as the vertical blanking region, or vblank for
  51. * short.
  52. *
  53. * For historical reference, the vertical blanking period was designed to
  54. * give the electron gun (on CRTs) enough time to move back to the top of
  55. * the screen to start scanning out the next frame. Similar for horizontal
  56. * blanking periods. They were designed to give the electron gun enough
  57. * time to move back to the other side of the screen to start scanning the
  58. * next scanline.
  59. *
  60. * ::
  61. *
  62. *
  63. * physical → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
  64. * top of | |
  65. * display | |
  66. * | New frame |
  67. * | |
  68. * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|
  69. * |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline,
  70. * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the
  71. * | | frame as it
  72. * | | travels down
  73. * | | ("scan out")
  74. * | Old frame |
  75. * | |
  76. * | |
  77. * | |
  78. * | | physical
  79. * | | bottom of
  80. * vertical |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display
  81. * blanking ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
  82. * region → ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
  83. * ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
  84. * start of → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
  85. * new frame
  86. *
  87. * "Physical top of display" is the reference point for the high-precision/
  88. * corrected timestamp.
  89. *
  90. * On a lot of display hardware, programming needs to take effect during the
  91. * vertical blanking period so that settings like gamma, the image buffer
  92. * buffer to be scanned out, etc. can safely be changed without showing
  93. * any visual artifacts on the screen. In some unforgiving hardware, some of
  94. * this programming has to both start and end in the same vblank. To help
  95. * with the timing of the hardware programming, an interrupt is usually
  96. * available to notify the driver when it can start the updating of registers.
  97. * The interrupt is in this context named the vblank interrupt.
  98. *
  99. * The vblank interrupt may be fired at different points depending on the
  100. * hardware. Some hardware implementations will fire the interrupt when the
  101. * new frame start, other implementations will fire the interrupt at different
  102. * points in time.
  103. *
  104. * Vertical blanking plays a major role in graphics rendering. To achieve
  105. * tear-free display, users must synchronize page flips and/or rendering to
  106. * vertical blanking. The DRM API offers ioctls to perform page flips
  107. * synchronized to vertical blanking and wait for vertical blanking.
  108. *
  109. * The DRM core handles most of the vertical blanking management logic, which
  110. * involves filtering out spurious interrupts, keeping race-free blanking
  111. * counters, coping with counter wrap-around and resets and keeping use counts.
  112. * It relies on the driver to generate vertical blanking interrupts and
  113. * optionally provide a hardware vertical blanking counter.
  114. *
  115. * Drivers must initialize the vertical blanking handling core with a call to
  116. * drm_vblank_init(). Minimally, a driver needs to implement
  117. * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
  118. * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
  119. * support.
  120. *
  121. * Vertical blanking interrupts can be enabled by the DRM core or by drivers
  122. * themselves (for instance to handle page flipping operations). The DRM core
  123. * maintains a vertical blanking use count to ensure that the interrupts are not
  124. * disabled while a user still needs them. To increment the use count, drivers
  125. * call drm_crtc_vblank_get() and release the vblank reference again with
  126. * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
  127. * guaranteed to be enabled.
  128. *
  129. * On many hardware disabling the vblank interrupt cannot be done in a race-free
  130. * manner, see &drm_driver.vblank_disable_immediate and
  131. * &drm_driver.max_vblank_count. In that case the vblank core only disables the
  132. * vblanks after a timer has expired, which can be configured through the
  133. * ``vblankoffdelay`` module parameter.
  134. *
  135. * Drivers for hardware without support for vertical-blanking interrupts
  136. * must not call drm_vblank_init(). For such drivers, atomic helpers will
  137. * automatically generate fake vblank events as part of the display update.
  138. * This functionality also can be controlled by the driver by enabling and
  139. * disabling struct drm_crtc_state.no_vblank.
  140. */
  141. /* Retry timestamp calculation up to 3 times to satisfy
  142. * drm_timestamp_precision before giving up.
  143. */
  144. #define DRM_TIMESTAMP_MAXRETRIES 3
  145. /* Threshold in nanoseconds for detection of redundant
  146. * vblank irq in drm_handle_vblank(). 1 msec should be ok.
  147. */
  148. #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
  149. static bool
  150. drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
  151. ktime_t *tvblank, bool in_vblank_irq);
  152. static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
  153. static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */
  154. module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
  155. module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
  156. MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
  157. MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
  158. static void store_vblank(struct drm_device *dev, unsigned int pipe,
  159. u32 vblank_count_inc,
  160. ktime_t t_vblank, u32 last)
  161. {
  162. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  163. assert_spin_locked(&dev->vblank_time_lock);
  164. vblank->last = last;
  165. write_seqlock(&vblank->seqlock);
  166. vblank->time = t_vblank;
  167. atomic64_add(vblank_count_inc, &vblank->count);
  168. write_sequnlock(&vblank->seqlock);
  169. }
  170. static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
  171. {
  172. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  173. return vblank->max_vblank_count ?: dev->max_vblank_count;
  174. }
  175. /*
  176. * "No hw counter" fallback implementation of .get_vblank_counter() hook,
  177. * if there is no usable hardware frame counter available.
  178. */
  179. static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
  180. {
  181. drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);
  182. return 0;
  183. }
  184. static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
  185. {
  186. if (drm_core_check_feature(dev, DRIVER_MODESET)) {
  187. struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
  188. if (drm_WARN_ON(dev, !crtc))
  189. return 0;
  190. if (crtc->funcs->get_vblank_counter)
  191. return crtc->funcs->get_vblank_counter(crtc);
  192. }
  193. #ifdef CONFIG_DRM_LEGACY
  194. else if (dev->driver->get_vblank_counter) {
  195. return dev->driver->get_vblank_counter(dev, pipe);
  196. }
  197. #endif
  198. return drm_vblank_no_hw_counter(dev, pipe);
  199. }
  200. /*
  201. * Reset the stored timestamp for the current vblank count to correspond
  202. * to the last vblank occurred.
  203. *
  204. * Only to be called from drm_crtc_vblank_on().
  205. *
  206. * Note: caller must hold &drm_device.vbl_lock since this reads & writes
  207. * device vblank fields.
  208. */
  209. static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
  210. {
  211. u32 cur_vblank;
  212. bool rc;
  213. ktime_t t_vblank;
  214. int count = DRM_TIMESTAMP_MAXRETRIES;
  215. spin_lock(&dev->vblank_time_lock);
  216. /*
  217. * sample the current counter to avoid random jumps
  218. * when drm_vblank_enable() applies the diff
  219. */
  220. do {
  221. cur_vblank = __get_vblank_counter(dev, pipe);
  222. rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
  223. } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
  224. /*
  225. * Only reinitialize corresponding vblank timestamp if high-precision query
  226. * available and didn't fail. Otherwise reinitialize delayed at next vblank
  227. * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
  228. */
  229. if (!rc)
  230. t_vblank = 0;
  231. /*
  232. * +1 to make sure user will never see the same
  233. * vblank counter value before and after a modeset
  234. */
  235. store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
  236. spin_unlock(&dev->vblank_time_lock);
  237. }
  238. /*
  239. * Call back into the driver to update the appropriate vblank counter
  240. * (specified by @pipe). Deal with wraparound, if it occurred, and
  241. * update the last read value so we can deal with wraparound on the next
  242. * call if necessary.
  243. *
  244. * Only necessary when going from off->on, to account for frames we
  245. * didn't get an interrupt for.
  246. *
  247. * Note: caller must hold &drm_device.vbl_lock since this reads & writes
  248. * device vblank fields.
  249. */
  250. static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
  251. bool in_vblank_irq)
  252. {
  253. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  254. u32 cur_vblank, diff;
  255. bool rc;
  256. ktime_t t_vblank;
  257. int count = DRM_TIMESTAMP_MAXRETRIES;
  258. int framedur_ns = vblank->framedur_ns;
  259. u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
  260. /*
  261. * Interrupts were disabled prior to this call, so deal with counter
  262. * wrap if needed.
  263. * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events
  264. * here if the register is small or we had vblank interrupts off for
  265. * a long time.
  266. *
  267. * We repeat the hardware vblank counter & timestamp query until
  268. * we get consistent results. This to prevent races between gpu
  269. * updating its hardware counter while we are retrieving the
  270. * corresponding vblank timestamp.
  271. */
  272. do {
  273. cur_vblank = __get_vblank_counter(dev, pipe);
  274. rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
  275. } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
  276. if (max_vblank_count) {
  277. /* trust the hw counter when it's around */
  278. diff = (cur_vblank - vblank->last) & max_vblank_count;
  279. } else if (rc && framedur_ns) {
  280. u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
  281. /*
  282. * Figure out how many vblanks we've missed based
  283. * on the difference in the timestamps and the
  284. * frame/field duration.
  285. */
  286. drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks."
  287. " diff_ns = %lld, framedur_ns = %d)\n",
  288. pipe, (long long)diff_ns, framedur_ns);
  289. diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
  290. if (diff == 0 && in_vblank_irq)
  291. drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n",
  292. pipe);
  293. } else {
  294. /* some kind of default for drivers w/o accurate vbl timestamping */
  295. diff = in_vblank_irq ? 1 : 0;
  296. }
  297. /*
  298. * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
  299. * interval? If so then vblank irqs keep running and it will likely
  300. * happen that the hardware vblank counter is not trustworthy as it
  301. * might reset at some point in that interval and vblank timestamps
  302. * are not trustworthy either in that interval. Iow. this can result
  303. * in a bogus diff >> 1 which must be avoided as it would cause
  304. * random large forward jumps of the software vblank counter.
  305. */
  306. if (diff > 1 && (vblank->inmodeset & 0x2)) {
  307. drm_dbg_vbl(dev,
  308. "clamping vblank bump to 1 on crtc %u: diffr=%u"
  309. " due to pre-modeset.\n", pipe, diff);
  310. diff = 1;
  311. }
  312. drm_dbg_vbl(dev, "updating vblank count on crtc %u:"
  313. " current=%llu, diff=%u, hw=%u hw_last=%u\n",
  314. pipe, (unsigned long long)atomic64_read(&vblank->count),
  315. diff, cur_vblank, vblank->last);
  316. if (diff == 0) {
  317. drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last);
  318. return;
  319. }
  320. /*
  321. * Only reinitialize corresponding vblank timestamp if high-precision query
  322. * available and didn't fail, or we were called from the vblank interrupt.
  323. * Otherwise reinitialize delayed at next vblank interrupt and assign 0
  324. * for now, to mark the vblanktimestamp as invalid.
  325. */
  326. if (!rc && !in_vblank_irq)
  327. t_vblank = 0;
  328. store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
  329. }
  330. u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
  331. {
  332. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  333. u64 count;
  334. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
  335. return 0;
  336. count = atomic64_read(&vblank->count);
  337. /*
  338. * This read barrier corresponds to the implicit write barrier of the
  339. * write seqlock in store_vblank(). Note that this is the only place
  340. * where we need an explicit barrier, since all other access goes
  341. * through drm_vblank_count_and_time(), which already has the required
  342. * read barrier curtesy of the read seqlock.
  343. */
  344. smp_rmb();
  345. return count;
  346. }
  347. /**
  348. * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
  349. * @crtc: which counter to retrieve
  350. *
  351. * This function is similar to drm_crtc_vblank_count() but this function
  352. * interpolates to handle a race with vblank interrupts using the high precision
  353. * timestamping support.
  354. *
  355. * This is mostly useful for hardware that can obtain the scanout position, but
  356. * doesn't have a hardware frame counter.
  357. */
  358. u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
  359. {
  360. struct drm_device *dev = crtc->dev;
  361. unsigned int pipe = drm_crtc_index(crtc);
  362. u64 vblank;
  363. unsigned long flags;
  364. drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) &&
  365. !crtc->funcs->get_vblank_timestamp,
  366. "This function requires support for accurate vblank timestamps.");
  367. spin_lock_irqsave(&dev->vblank_time_lock, flags);
  368. drm_update_vblank_count(dev, pipe, false);
  369. vblank = drm_vblank_count(dev, pipe);
  370. spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
  371. return vblank;
  372. }
  373. EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
  374. static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
  375. {
  376. if (drm_core_check_feature(dev, DRIVER_MODESET)) {
  377. struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
  378. if (drm_WARN_ON(dev, !crtc))
  379. return;
  380. if (crtc->funcs->disable_vblank)
  381. crtc->funcs->disable_vblank(crtc);
  382. }
  383. #ifdef CONFIG_DRM_LEGACY
  384. else {
  385. dev->driver->disable_vblank(dev, pipe);
  386. }
  387. #endif
  388. }
  389. /*
  390. * Disable vblank irq's on crtc, make sure that last vblank count
  391. * of hardware and corresponding consistent software vblank counter
  392. * are preserved, even if there are any spurious vblank irq's after
  393. * disable.
  394. */
  395. void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
  396. {
  397. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  398. unsigned long irqflags;
  399. assert_spin_locked(&dev->vbl_lock);
  400. /* Prevent vblank irq processing while disabling vblank irqs,
  401. * so no updates of timestamps or count can happen after we've
  402. * disabled. Needed to prevent races in case of delayed irq's.
  403. */
  404. spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
  405. /*
  406. * Update vblank count and disable vblank interrupts only if the
  407. * interrupts were enabled. This avoids calling the ->disable_vblank()
  408. * operation in atomic context with the hardware potentially runtime
  409. * suspended.
  410. */
  411. if (!vblank->enabled)
  412. goto out;
  413. /*
  414. * Update the count and timestamp to maintain the
  415. * appearance that the counter has been ticking all along until
  416. * this time. This makes the count account for the entire time
  417. * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
  418. */
  419. drm_update_vblank_count(dev, pipe, false);
  420. __disable_vblank(dev, pipe);
  421. vblank->enabled = false;
  422. out:
  423. spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
  424. }
  425. static void vblank_disable_fn(struct timer_list *t)
  426. {
  427. struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
  428. struct drm_device *dev = vblank->dev;
  429. unsigned int pipe = vblank->pipe;
  430. unsigned long irqflags;
  431. spin_lock_irqsave(&dev->vbl_lock, irqflags);
  432. if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
  433. drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);
  434. drm_vblank_disable_and_save(dev, pipe);
  435. }
  436. spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
  437. }
  438. static void drm_vblank_init_release(struct drm_device *dev, void *ptr)
  439. {
  440. struct drm_vblank_crtc *vblank = ptr;
  441. drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&
  442. drm_core_check_feature(dev, DRIVER_MODESET));
  443. drm_vblank_destroy_worker(vblank);
  444. del_timer_sync(&vblank->disable_timer);
  445. }
  446. /**
  447. * drm_vblank_init - initialize vblank support
  448. * @dev: DRM device
  449. * @num_crtcs: number of CRTCs supported by @dev
  450. *
  451. * This function initializes vblank support for @num_crtcs display pipelines.
  452. * Cleanup is handled automatically through a cleanup function added with
  453. * drmm_add_action_or_reset().
  454. *
  455. * Returns:
  456. * Zero on success or a negative error code on failure.
  457. */
  458. int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
  459. {
  460. int ret;
  461. unsigned int i;
  462. spin_lock_init(&dev->vbl_lock);
  463. spin_lock_init(&dev->vblank_time_lock);
  464. dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
  465. if (!dev->vblank)
  466. return -ENOMEM;
  467. dev->num_crtcs = num_crtcs;
  468. for (i = 0; i < num_crtcs; i++) {
  469. struct drm_vblank_crtc *vblank = &dev->vblank[i];
  470. vblank->dev = dev;
  471. vblank->pipe = i;
  472. init_waitqueue_head(&vblank->queue);
  473. timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
  474. seqlock_init(&vblank->seqlock);
  475. ret = drmm_add_action_or_reset(dev, drm_vblank_init_release,
  476. vblank);
  477. if (ret)
  478. return ret;
  479. ret = drm_vblank_worker_init(vblank);
  480. if (ret)
  481. return ret;
  482. }
  483. return 0;
  484. }
  485. EXPORT_SYMBOL(drm_vblank_init);
  486. /**
  487. * drm_dev_has_vblank - test if vblanking has been initialized for
  488. * a device
  489. * @dev: the device
  490. *
  491. * Drivers may call this function to test if vblank support is
  492. * initialized for a device. For most hardware this means that vblanking
  493. * can also be enabled.
  494. *
  495. * Atomic helpers use this function to initialize
  496. * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset().
  497. *
  498. * Returns:
  499. * True if vblanking has been initialized for the given device, false
  500. * otherwise.
  501. */
  502. bool drm_dev_has_vblank(const struct drm_device *dev)
  503. {
  504. return dev->num_crtcs != 0;
  505. }
  506. EXPORT_SYMBOL(drm_dev_has_vblank);
  507. /**
  508. * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
  509. * @crtc: which CRTC's vblank waitqueue to retrieve
  510. *
  511. * This function returns a pointer to the vblank waitqueue for the CRTC.
  512. * Drivers can use this to implement vblank waits using wait_event() and related
  513. * functions.
  514. */
  515. wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
  516. {
  517. return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
  518. }
  519. EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
  520. /**
  521. * drm_calc_timestamping_constants - calculate vblank timestamp constants
  522. * @crtc: drm_crtc whose timestamp constants should be updated.
  523. * @mode: display mode containing the scanout timings
  524. *
  525. * Calculate and store various constants which are later needed by vblank and
  526. * swap-completion timestamping, e.g, by
  527. * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from
  528. * CRTC's true scanout timing, so they take things like panel scaling or
  529. * other adjustments into account.
  530. */
  531. void drm_calc_timestamping_constants(struct drm_crtc *crtc,
  532. const struct drm_display_mode *mode)
  533. {
  534. struct drm_device *dev = crtc->dev;
  535. unsigned int pipe = drm_crtc_index(crtc);
  536. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  537. int linedur_ns = 0, framedur_ns = 0;
  538. int dotclock = mode->crtc_clock;
  539. if (!drm_dev_has_vblank(dev))
  540. return;
  541. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
  542. return;
  543. /* Valid dotclock? */
  544. if (dotclock > 0) {
  545. int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
  546. /*
  547. * Convert scanline length in pixels and video
  548. * dot clock to line duration and frame duration
  549. * in nanoseconds:
  550. */
  551. linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
  552. framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
  553. /*
  554. * Fields of interlaced scanout modes are only half a frame duration.
  555. */
  556. if (mode->flags & DRM_MODE_FLAG_INTERLACE)
  557. framedur_ns /= 2;
  558. } else {
  559. drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n",
  560. crtc->base.id);
  561. }
  562. vblank->linedur_ns = linedur_ns;
  563. vblank->framedur_ns = framedur_ns;
  564. drm_mode_copy(&vblank->hwmode, mode);
  565. drm_dbg_core(dev,
  566. "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
  567. crtc->base.id, mode->crtc_htotal,
  568. mode->crtc_vtotal, mode->crtc_vdisplay);
  569. drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n",
  570. crtc->base.id, dotclock, framedur_ns, linedur_ns);
  571. }
  572. EXPORT_SYMBOL(drm_calc_timestamping_constants);
  573. /**
  574. * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank
  575. * timestamp helper
  576. * @crtc: CRTC whose vblank timestamp to retrieve
  577. * @max_error: Desired maximum allowable error in timestamps (nanosecs)
  578. * On return contains true maximum error of timestamp
  579. * @vblank_time: Pointer to time which should receive the timestamp
  580. * @in_vblank_irq:
  581. * True when called from drm_crtc_handle_vblank(). Some drivers
  582. * need to apply some workarounds for gpu-specific vblank irq quirks
  583. * if flag is set.
  584. * @get_scanout_position:
  585. * Callback function to retrieve the scanout position. See
  586. * @struct drm_crtc_helper_funcs.get_scanout_position.
  587. *
  588. * Implements calculation of exact vblank timestamps from given drm_display_mode
  589. * timings and current video scanout position of a CRTC.
  590. *
  591. * The current implementation only handles standard video modes. For double scan
  592. * and interlaced modes the driver is supposed to adjust the hardware mode
  593. * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
  594. * match the scanout position reported.
  595. *
  596. * Note that atomic drivers must call drm_calc_timestamping_constants() before
  597. * enabling a CRTC. The atomic helpers already take care of that in
  598. * drm_atomic_helper_calc_timestamping_constants().
  599. *
  600. * Returns:
  601. *
  602. * Returns true on success, and false on failure, i.e. when no accurate
  603. * timestamp could be acquired.
  604. */
  605. bool
  606. drm_crtc_vblank_helper_get_vblank_timestamp_internal(
  607. struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time,
  608. bool in_vblank_irq,
  609. drm_vblank_get_scanout_position_func get_scanout_position)
  610. {
  611. struct drm_device *dev = crtc->dev;
  612. unsigned int pipe = crtc->index;
  613. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  614. struct timespec64 ts_etime, ts_vblank_time;
  615. ktime_t stime, etime;
  616. bool vbl_status;
  617. const struct drm_display_mode *mode;
  618. int vpos, hpos, i;
  619. int delta_ns, duration_ns;
  620. if (pipe >= dev->num_crtcs) {
  621. drm_err(dev, "Invalid crtc %u\n", pipe);
  622. return false;
  623. }
  624. /* Scanout position query not supported? Should not happen. */
  625. if (!get_scanout_position) {
  626. drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n");
  627. return false;
  628. }
  629. if (drm_drv_uses_atomic_modeset(dev))
  630. mode = &vblank->hwmode;
  631. else
  632. mode = &crtc->hwmode;
  633. /* If mode timing undefined, just return as no-op:
  634. * Happens during initial modesetting of a crtc.
  635. */
  636. if (mode->crtc_clock == 0) {
  637. drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n",
  638. pipe);
  639. drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev));
  640. return false;
  641. }
  642. /* Get current scanout position with system timestamp.
  643. * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
  644. * if single query takes longer than max_error nanoseconds.
  645. *
  646. * This guarantees a tight bound on maximum error if
  647. * code gets preempted or delayed for some reason.
  648. */
  649. for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
  650. /*
  651. * Get vertical and horizontal scanout position vpos, hpos,
  652. * and bounding timestamps stime, etime, pre/post query.
  653. */
  654. vbl_status = get_scanout_position(crtc, in_vblank_irq,
  655. &vpos, &hpos,
  656. &stime, &etime,
  657. mode);
  658. /* Return as no-op if scanout query unsupported or failed. */
  659. if (!vbl_status) {
  660. drm_dbg_core(dev,
  661. "crtc %u : scanoutpos query failed.\n",
  662. pipe);
  663. return false;
  664. }
  665. /* Compute uncertainty in timestamp of scanout position query. */
  666. duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
  667. /* Accept result with < max_error nsecs timing uncertainty. */
  668. if (duration_ns <= *max_error)
  669. break;
  670. }
  671. /* Noisy system timing? */
  672. if (i == DRM_TIMESTAMP_MAXRETRIES) {
  673. drm_dbg_core(dev,
  674. "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
  675. pipe, duration_ns / 1000, *max_error / 1000, i);
  676. }
  677. /* Return upper bound of timestamp precision error. */
  678. *max_error = duration_ns;
  679. /* Convert scanout position into elapsed time at raw_time query
  680. * since start of scanout at first display scanline. delta_ns
  681. * can be negative if start of scanout hasn't happened yet.
  682. */
  683. delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
  684. mode->crtc_clock);
  685. /* Subtract time delta from raw timestamp to get final
  686. * vblank_time timestamp for end of vblank.
  687. */
  688. *vblank_time = ktime_sub_ns(etime, delta_ns);
  689. if (!drm_debug_enabled(DRM_UT_VBL))
  690. return true;
  691. ts_etime = ktime_to_timespec64(etime);
  692. ts_vblank_time = ktime_to_timespec64(*vblank_time);
  693. drm_dbg_vbl(dev,
  694. "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
  695. pipe, hpos, vpos,
  696. (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
  697. (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
  698. duration_ns / 1000, i);
  699. return true;
  700. }
  701. EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal);
  702. /**
  703. * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp
  704. * helper
  705. * @crtc: CRTC whose vblank timestamp to retrieve
  706. * @max_error: Desired maximum allowable error in timestamps (nanosecs)
  707. * On return contains true maximum error of timestamp
  708. * @vblank_time: Pointer to time which should receive the timestamp
  709. * @in_vblank_irq:
  710. * True when called from drm_crtc_handle_vblank(). Some drivers
  711. * need to apply some workarounds for gpu-specific vblank irq quirks
  712. * if flag is set.
  713. *
  714. * Implements calculation of exact vblank timestamps from given drm_display_mode
  715. * timings and current video scanout position of a CRTC. This can be directly
  716. * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms
  717. * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented.
  718. *
  719. * The current implementation only handles standard video modes. For double scan
  720. * and interlaced modes the driver is supposed to adjust the hardware mode
  721. * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
  722. * match the scanout position reported.
  723. *
  724. * Note that atomic drivers must call drm_calc_timestamping_constants() before
  725. * enabling a CRTC. The atomic helpers already take care of that in
  726. * drm_atomic_helper_calc_timestamping_constants().
  727. *
  728. * Returns:
  729. *
  730. * Returns true on success, and false on failure, i.e. when no accurate
  731. * timestamp could be acquired.
  732. */
  733. bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,
  734. int *max_error,
  735. ktime_t *vblank_time,
  736. bool in_vblank_irq)
  737. {
  738. return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
  739. crtc, max_error, vblank_time, in_vblank_irq,
  740. crtc->helper_private->get_scanout_position);
  741. }
  742. EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
  743. /**
  744. * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
  745. * vblank interval
  746. * @dev: DRM device
  747. * @pipe: index of CRTC whose vblank timestamp to retrieve
  748. * @tvblank: Pointer to target time which should receive the timestamp
  749. * @in_vblank_irq:
  750. * True when called from drm_crtc_handle_vblank(). Some drivers
  751. * need to apply some workarounds for gpu-specific vblank irq quirks
  752. * if flag is set.
  753. *
  754. * Fetches the system timestamp corresponding to the time of the most recent
  755. * vblank interval on specified CRTC. May call into kms-driver to
  756. * compute the timestamp with a high-precision GPU specific method.
  757. *
  758. * Returns zero if timestamp originates from uncorrected do_gettimeofday()
  759. * call, i.e., it isn't very precisely locked to the true vblank.
  760. *
  761. * Returns:
  762. * True if timestamp is considered to be very precise, false otherwise.
  763. */
  764. static bool
  765. drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
  766. ktime_t *tvblank, bool in_vblank_irq)
  767. {
  768. struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
  769. bool ret = false;
  770. /* Define requested maximum error on timestamps (nanoseconds). */
  771. int max_error = (int) drm_timestamp_precision * 1000;
  772. /* Query driver if possible and precision timestamping enabled. */
  773. if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {
  774. struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
  775. ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,
  776. tvblank, in_vblank_irq);
  777. }
  778. /* GPU high precision timestamp query unsupported or failed.
  779. * Return current monotonic/gettimeofday timestamp as best estimate.
  780. */
  781. if (!ret)
  782. *tvblank = ktime_get();
  783. return ret;
  784. }
  785. /**
  786. * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
  787. * @crtc: which counter to retrieve
  788. *
  789. * Fetches the "cooked" vblank count value that represents the number of
  790. * vblank events since the system was booted, including lost events due to
  791. * modesetting activity. Note that this timer isn't correct against a racing
  792. * vblank interrupt (since it only reports the software vblank counter), see
  793. * drm_crtc_accurate_vblank_count() for such use-cases.
  794. *
  795. * Note that for a given vblank counter value drm_crtc_handle_vblank()
  796. * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
  797. * provide a barrier: Any writes done before calling
  798. * drm_crtc_handle_vblank() will be visible to callers of the later
  799. * functions, if the vblank count is the same or a later one.
  800. *
  801. * See also &drm_vblank_crtc.count.
  802. *
  803. * Returns:
  804. * The software vblank counter.
  805. */
  806. u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
  807. {
  808. return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
  809. }
  810. EXPORT_SYMBOL(drm_crtc_vblank_count);
  811. /**
  812. * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
  813. * system timestamp corresponding to that vblank counter value.
  814. * @dev: DRM device
  815. * @pipe: index of CRTC whose counter to retrieve
  816. * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
  817. *
  818. * Fetches the "cooked" vblank count value that represents the number of
  819. * vblank events since the system was booted, including lost events due to
  820. * modesetting activity. Returns corresponding system timestamp of the time
  821. * of the vblank interval that corresponds to the current vblank counter value.
  822. *
  823. * This is the legacy version of drm_crtc_vblank_count_and_time().
  824. */
  825. static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
  826. ktime_t *vblanktime)
  827. {
  828. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  829. u64 vblank_count;
  830. unsigned int seq;
  831. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {
  832. *vblanktime = 0;
  833. return 0;
  834. }
  835. do {
  836. seq = read_seqbegin(&vblank->seqlock);
  837. vblank_count = atomic64_read(&vblank->count);
  838. *vblanktime = vblank->time;
  839. } while (read_seqretry(&vblank->seqlock, seq));
  840. return vblank_count;
  841. }
  842. /**
  843. * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
  844. * and the system timestamp corresponding to that vblank counter value
  845. * @crtc: which counter to retrieve
  846. * @vblanktime: Pointer to time to receive the vblank timestamp.
  847. *
  848. * Fetches the "cooked" vblank count value that represents the number of
  849. * vblank events since the system was booted, including lost events due to
  850. * modesetting activity. Returns corresponding system timestamp of the time
  851. * of the vblank interval that corresponds to the current vblank counter value.
  852. *
  853. * Note that for a given vblank counter value drm_crtc_handle_vblank()
  854. * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
  855. * provide a barrier: Any writes done before calling
  856. * drm_crtc_handle_vblank() will be visible to callers of the later
  857. * functions, if the vblank count is the same or a later one.
  858. *
  859. * See also &drm_vblank_crtc.count.
  860. */
  861. u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
  862. ktime_t *vblanktime)
  863. {
  864. return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
  865. vblanktime);
  866. }
  867. EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
  868. static void send_vblank_event(struct drm_device *dev,
  869. struct drm_pending_vblank_event *e,
  870. u64 seq, ktime_t now)
  871. {
  872. struct timespec64 tv;
  873. switch (e->event.base.type) {
  874. case DRM_EVENT_VBLANK:
  875. case DRM_EVENT_FLIP_COMPLETE:
  876. tv = ktime_to_timespec64(now);
  877. e->event.vbl.sequence = seq;
  878. /*
  879. * e->event is a user space structure, with hardcoded unsigned
  880. * 32-bit seconds/microseconds. This is safe as we always use
  881. * monotonic timestamps since linux-4.15
  882. */
  883. e->event.vbl.tv_sec = tv.tv_sec;
  884. e->event.vbl.tv_usec = tv.tv_nsec / 1000;
  885. break;
  886. case DRM_EVENT_CRTC_SEQUENCE:
  887. if (seq)
  888. e->event.seq.sequence = seq;
  889. e->event.seq.time_ns = ktime_to_ns(now);
  890. break;
  891. }
  892. trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
  893. /*
  894. * Use the same timestamp for any associated fence signal to avoid
  895. * mismatch in timestamps for vsync & fence events triggered by the
  896. * same HW event. Frameworks like SurfaceFlinger in Android expects the
  897. * retire-fence timestamp to match exactly with HW vsync as it uses it
  898. * for its software vsync modeling.
  899. */
  900. drm_send_event_timestamp_locked(dev, &e->base, now);
  901. }
  902. /**
  903. * drm_crtc_arm_vblank_event - arm vblank event after pageflip
  904. * @crtc: the source CRTC of the vblank event
  905. * @e: the event to send
  906. *
  907. * A lot of drivers need to generate vblank events for the very next vblank
  908. * interrupt. For example when the page flip interrupt happens when the page
  909. * flip gets armed, but not when it actually executes within the next vblank
  910. * period. This helper function implements exactly the required vblank arming
  911. * behaviour.
  912. *
  913. * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
  914. * atomic commit must ensure that the next vblank happens at exactly the same
  915. * time as the atomic commit is committed to the hardware. This function itself
  916. * does **not** protect against the next vblank interrupt racing with either this
  917. * function call or the atomic commit operation. A possible sequence could be:
  918. *
  919. * 1. Driver commits new hardware state into vblank-synchronized registers.
  920. * 2. A vblank happens, committing the hardware state. Also the corresponding
  921. * vblank interrupt is fired off and fully processed by the interrupt
  922. * handler.
  923. * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
  924. * 4. The event is only send out for the next vblank, which is wrong.
  925. *
  926. * An equivalent race can happen when the driver calls
  927. * drm_crtc_arm_vblank_event() before writing out the new hardware state.
  928. *
  929. * The only way to make this work safely is to prevent the vblank from firing
  930. * (and the hardware from committing anything else) until the entire atomic
  931. * commit sequence has run to completion. If the hardware does not have such a
  932. * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
  933. * Instead drivers need to manually send out the event from their interrupt
  934. * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
  935. * possible race with the hardware committing the atomic update.
  936. *
  937. * Caller must hold a vblank reference for the event @e acquired by a
  938. * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
  939. */
  940. void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
  941. struct drm_pending_vblank_event *e)
  942. {
  943. struct drm_device *dev = crtc->dev;
  944. unsigned int pipe = drm_crtc_index(crtc);
  945. assert_spin_locked(&dev->event_lock);
  946. e->pipe = pipe;
  947. e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
  948. list_add_tail(&e->base.link, &dev->vblank_event_list);
  949. }
  950. EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
  951. /**
  952. * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
  953. * @crtc: the source CRTC of the vblank event
  954. * @e: the event to send
  955. *
  956. * Updates sequence # and timestamp on event for the most recently processed
  957. * vblank, and sends it to userspace. Caller must hold event lock.
  958. *
  959. * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
  960. * situation, especially to send out events for atomic commit operations.
  961. */
  962. void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
  963. struct drm_pending_vblank_event *e)
  964. {
  965. struct drm_device *dev = crtc->dev;
  966. u64 seq;
  967. unsigned int pipe = drm_crtc_index(crtc);
  968. ktime_t now;
  969. if (drm_dev_has_vblank(dev)) {
  970. seq = drm_vblank_count_and_time(dev, pipe, &now);
  971. } else {
  972. seq = 0;
  973. now = ktime_get();
  974. }
  975. e->pipe = pipe;
  976. send_vblank_event(dev, e, seq, now);
  977. }
  978. EXPORT_SYMBOL(drm_crtc_send_vblank_event);
  979. static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
  980. {
  981. if (drm_core_check_feature(dev, DRIVER_MODESET)) {
  982. struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
  983. if (drm_WARN_ON(dev, !crtc))
  984. return 0;
  985. if (crtc->funcs->enable_vblank)
  986. return crtc->funcs->enable_vblank(crtc);
  987. }
  988. #ifdef CONFIG_DRM_LEGACY
  989. else if (dev->driver->enable_vblank) {
  990. return dev->driver->enable_vblank(dev, pipe);
  991. }
  992. #endif
  993. return -EINVAL;
  994. }
  995. static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
  996. {
  997. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  998. int ret = 0;
  999. assert_spin_locked(&dev->vbl_lock);
  1000. spin_lock(&dev->vblank_time_lock);
  1001. if (!vblank->enabled) {
  1002. /*
  1003. * Enable vblank irqs under vblank_time_lock protection.
  1004. * All vblank count & timestamp updates are held off
  1005. * until we are done reinitializing master counter and
  1006. * timestamps. Filtercode in drm_handle_vblank() will
  1007. * prevent double-accounting of same vblank interval.
  1008. */
  1009. ret = __enable_vblank(dev, pipe);
  1010. drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n",
  1011. pipe, ret);
  1012. if (ret) {
  1013. atomic_dec(&vblank->refcount);
  1014. } else {
  1015. drm_update_vblank_count(dev, pipe, 0);
  1016. /* drm_update_vblank_count() includes a wmb so we just
  1017. * need to ensure that the compiler emits the write
  1018. * to mark the vblank as enabled after the call
  1019. * to drm_update_vblank_count().
  1020. */
  1021. WRITE_ONCE(vblank->enabled, true);
  1022. }
  1023. }
  1024. spin_unlock(&dev->vblank_time_lock);
  1025. return ret;
  1026. }
  1027. int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
  1028. {
  1029. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1030. unsigned long irqflags;
  1031. int ret = 0;
  1032. if (!drm_dev_has_vblank(dev))
  1033. return -EINVAL;
  1034. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
  1035. return -EINVAL;
  1036. spin_lock_irqsave(&dev->vbl_lock, irqflags);
  1037. /* Going from 0->1 means we have to enable interrupts again */
  1038. if (atomic_add_return(1, &vblank->refcount) == 1) {
  1039. ret = drm_vblank_enable(dev, pipe);
  1040. } else {
  1041. if (!vblank->enabled) {
  1042. atomic_dec(&vblank->refcount);
  1043. ret = -EINVAL;
  1044. }
  1045. }
  1046. spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
  1047. return ret;
  1048. }
  1049. /**
  1050. * drm_crtc_vblank_get - get a reference count on vblank events
  1051. * @crtc: which CRTC to own
  1052. *
  1053. * Acquire a reference count on vblank events to avoid having them disabled
  1054. * while in use.
  1055. *
  1056. * Returns:
  1057. * Zero on success or a negative error code on failure.
  1058. */
  1059. int drm_crtc_vblank_get(struct drm_crtc *crtc)
  1060. {
  1061. return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
  1062. }
  1063. EXPORT_SYMBOL(drm_crtc_vblank_get);
  1064. void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
  1065. {
  1066. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1067. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
  1068. return;
  1069. if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0))
  1070. return;
  1071. /* Last user schedules interrupt disable */
  1072. if (atomic_dec_and_test(&vblank->refcount)) {
  1073. if (drm_vblank_offdelay == 0)
  1074. return;
  1075. else if (drm_vblank_offdelay < 0)
  1076. vblank_disable_fn(&vblank->disable_timer);
  1077. else if (!dev->vblank_disable_immediate)
  1078. mod_timer(&vblank->disable_timer,
  1079. jiffies + ((drm_vblank_offdelay * HZ)/1000));
  1080. }
  1081. }
  1082. /**
  1083. * drm_crtc_vblank_put - give up ownership of vblank events
  1084. * @crtc: which counter to give up
  1085. *
  1086. * Release ownership of a given vblank counter, turning off interrupts
  1087. * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
  1088. */
  1089. void drm_crtc_vblank_put(struct drm_crtc *crtc)
  1090. {
  1091. drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
  1092. }
  1093. EXPORT_SYMBOL(drm_crtc_vblank_put);
  1094. /**
  1095. * drm_wait_one_vblank - wait for one vblank
  1096. * @dev: DRM device
  1097. * @pipe: CRTC index
  1098. *
  1099. * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
  1100. * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
  1101. * due to lack of driver support or because the crtc is off.
  1102. *
  1103. * This is the legacy version of drm_crtc_wait_one_vblank().
  1104. */
  1105. void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
  1106. {
  1107. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1108. int ret;
  1109. u64 last;
  1110. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
  1111. return;
  1112. ret = drm_vblank_get(dev, pipe);
  1113. if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",
  1114. pipe, ret))
  1115. return;
  1116. last = drm_vblank_count(dev, pipe);
  1117. ret = wait_event_timeout(vblank->queue,
  1118. last != drm_vblank_count(dev, pipe),
  1119. msecs_to_jiffies(100));
  1120. drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);
  1121. drm_vblank_put(dev, pipe);
  1122. }
  1123. EXPORT_SYMBOL(drm_wait_one_vblank);
  1124. /**
  1125. * drm_crtc_wait_one_vblank - wait for one vblank
  1126. * @crtc: DRM crtc
  1127. *
  1128. * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
  1129. * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
  1130. * due to lack of driver support or because the crtc is off.
  1131. */
  1132. void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
  1133. {
  1134. drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
  1135. }
  1136. EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
  1137. /**
  1138. * drm_crtc_vblank_off - disable vblank events on a CRTC
  1139. * @crtc: CRTC in question
  1140. *
  1141. * Drivers can use this function to shut down the vblank interrupt handling when
  1142. * disabling a crtc. This function ensures that the latest vblank frame count is
  1143. * stored so that drm_vblank_on can restore it again.
  1144. *
  1145. * Drivers must use this function when the hardware vblank counter can get
  1146. * reset, e.g. when suspending or disabling the @crtc in general.
  1147. */
  1148. void drm_crtc_vblank_off(struct drm_crtc *crtc)
  1149. {
  1150. struct drm_device *dev = crtc->dev;
  1151. unsigned int pipe = drm_crtc_index(crtc);
  1152. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1153. struct drm_pending_vblank_event *e, *t;
  1154. ktime_t now;
  1155. u64 seq;
  1156. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
  1157. return;
  1158. /*
  1159. * Grab event_lock early to prevent vblank work from being scheduled
  1160. * while we're in the middle of shutting down vblank interrupts
  1161. */
  1162. spin_lock_irq(&dev->event_lock);
  1163. spin_lock(&dev->vbl_lock);
  1164. drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
  1165. pipe, vblank->enabled, vblank->inmodeset);
  1166. /* Avoid redundant vblank disables without previous
  1167. * drm_crtc_vblank_on(). */
  1168. if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
  1169. drm_vblank_disable_and_save(dev, pipe);
  1170. wake_up(&vblank->queue);
  1171. /*
  1172. * Prevent subsequent drm_vblank_get() from re-enabling
  1173. * the vblank interrupt by bumping the refcount.
  1174. */
  1175. if (!vblank->inmodeset) {
  1176. atomic_inc(&vblank->refcount);
  1177. vblank->inmodeset = 1;
  1178. }
  1179. spin_unlock(&dev->vbl_lock);
  1180. /* Send any queued vblank events, lest the natives grow disquiet */
  1181. seq = drm_vblank_count_and_time(dev, pipe, &now);
  1182. list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
  1183. if (e->pipe != pipe)
  1184. continue;
  1185. drm_dbg_core(dev, "Sending premature vblank event on disable: "
  1186. "wanted %llu, current %llu\n",
  1187. e->sequence, seq);
  1188. list_del(&e->base.link);
  1189. drm_vblank_put(dev, pipe);
  1190. send_vblank_event(dev, e, seq, now);
  1191. }
  1192. /* Cancel any leftover pending vblank work */
  1193. drm_vblank_cancel_pending_works(vblank);
  1194. spin_unlock_irq(&dev->event_lock);
  1195. /* Will be reset by the modeset helpers when re-enabling the crtc by
  1196. * calling drm_calc_timestamping_constants(). */
  1197. vblank->hwmode.crtc_clock = 0;
  1198. /* Wait for any vblank work that's still executing to finish */
  1199. drm_vblank_flush_worker(vblank);
  1200. }
  1201. EXPORT_SYMBOL(drm_crtc_vblank_off);
  1202. /**
  1203. * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
  1204. * @crtc: CRTC in question
  1205. *
  1206. * Drivers can use this function to reset the vblank state to off at load time.
  1207. * Drivers should use this together with the drm_crtc_vblank_off() and
  1208. * drm_crtc_vblank_on() functions. The difference compared to
  1209. * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
  1210. * and hence doesn't need to call any driver hooks.
  1211. *
  1212. * This is useful for recovering driver state e.g. on driver load, or on resume.
  1213. */
  1214. void drm_crtc_vblank_reset(struct drm_crtc *crtc)
  1215. {
  1216. struct drm_device *dev = crtc->dev;
  1217. unsigned int pipe = drm_crtc_index(crtc);
  1218. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1219. spin_lock_irq(&dev->vbl_lock);
  1220. /*
  1221. * Prevent subsequent drm_vblank_get() from enabling the vblank
  1222. * interrupt by bumping the refcount.
  1223. */
  1224. if (!vblank->inmodeset) {
  1225. atomic_inc(&vblank->refcount);
  1226. vblank->inmodeset = 1;
  1227. }
  1228. spin_unlock_irq(&dev->vbl_lock);
  1229. drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list));
  1230. drm_WARN_ON(dev, !list_empty(&vblank->pending_work));
  1231. }
  1232. EXPORT_SYMBOL(drm_crtc_vblank_reset);
  1233. /**
  1234. * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
  1235. * @crtc: CRTC in question
  1236. * @max_vblank_count: max hardware vblank counter value
  1237. *
  1238. * Update the maximum hardware vblank counter value for @crtc
  1239. * at runtime. Useful for hardware where the operation of the
  1240. * hardware vblank counter depends on the currently active
  1241. * display configuration.
  1242. *
  1243. * For example, if the hardware vblank counter does not work
  1244. * when a specific connector is active the maximum can be set
  1245. * to zero. And when that specific connector isn't active the
  1246. * maximum can again be set to the appropriate non-zero value.
  1247. *
  1248. * If used, must be called before drm_vblank_on().
  1249. */
  1250. void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
  1251. u32 max_vblank_count)
  1252. {
  1253. struct drm_device *dev = crtc->dev;
  1254. unsigned int pipe = drm_crtc_index(crtc);
  1255. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1256. drm_WARN_ON(dev, dev->max_vblank_count);
  1257. drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));
  1258. vblank->max_vblank_count = max_vblank_count;
  1259. }
  1260. EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
  1261. /**
  1262. * drm_crtc_vblank_on - enable vblank events on a CRTC
  1263. * @crtc: CRTC in question
  1264. *
  1265. * This functions restores the vblank interrupt state captured with
  1266. * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
  1267. * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
  1268. * unbalanced and so can also be unconditionally called in driver load code to
  1269. * reflect the current hardware state of the crtc.
  1270. */
  1271. void drm_crtc_vblank_on(struct drm_crtc *crtc)
  1272. {
  1273. struct drm_device *dev = crtc->dev;
  1274. unsigned int pipe = drm_crtc_index(crtc);
  1275. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1276. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
  1277. return;
  1278. spin_lock_irq(&dev->vbl_lock);
  1279. drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
  1280. pipe, vblank->enabled, vblank->inmodeset);
  1281. /* Drop our private "prevent drm_vblank_get" refcount */
  1282. if (vblank->inmodeset) {
  1283. atomic_dec(&vblank->refcount);
  1284. vblank->inmodeset = 0;
  1285. }
  1286. drm_reset_vblank_timestamp(dev, pipe);
  1287. /*
  1288. * re-enable interrupts if there are users left, or the
  1289. * user wishes vblank interrupts to be enabled all the time.
  1290. */
  1291. if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
  1292. drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));
  1293. spin_unlock_irq(&dev->vbl_lock);
  1294. }
  1295. EXPORT_SYMBOL(drm_crtc_vblank_on);
  1296. static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
  1297. {
  1298. ktime_t t_vblank;
  1299. struct drm_vblank_crtc *vblank;
  1300. int framedur_ns;
  1301. u64 diff_ns;
  1302. u32 cur_vblank, diff = 1;
  1303. int count = DRM_TIMESTAMP_MAXRETRIES;
  1304. u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
  1305. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
  1306. return;
  1307. assert_spin_locked(&dev->vbl_lock);
  1308. assert_spin_locked(&dev->vblank_time_lock);
  1309. vblank = &dev->vblank[pipe];
  1310. drm_WARN_ONCE(dev,
  1311. drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,
  1312. "Cannot compute missed vblanks without frame duration\n");
  1313. framedur_ns = vblank->framedur_ns;
  1314. do {
  1315. cur_vblank = __get_vblank_counter(dev, pipe);
  1316. drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
  1317. } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
  1318. diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
  1319. if (framedur_ns)
  1320. diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
  1321. drm_dbg_vbl(dev,
  1322. "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
  1323. diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
  1324. vblank->last = (cur_vblank - diff) & max_vblank_count;
  1325. }
  1326. /**
  1327. * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
  1328. * @crtc: CRTC in question
  1329. *
  1330. * Power manamement features can cause frame counter resets between vblank
  1331. * disable and enable. Drivers can use this function in their
  1332. * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
  1333. * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
  1334. * vblank counter.
  1335. *
  1336. * Note that drivers must have race-free high-precision timestamping support,
  1337. * i.e. &drm_crtc_funcs.get_vblank_timestamp must be hooked up and
  1338. * &drm_driver.vblank_disable_immediate must be set to indicate the
  1339. * time-stamping functions are race-free against vblank hardware counter
  1340. * increments.
  1341. */
  1342. void drm_crtc_vblank_restore(struct drm_crtc *crtc)
  1343. {
  1344. WARN_ON_ONCE(!crtc->funcs->get_vblank_timestamp);
  1345. WARN_ON_ONCE(!crtc->dev->vblank_disable_immediate);
  1346. drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
  1347. }
  1348. EXPORT_SYMBOL(drm_crtc_vblank_restore);
  1349. static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
  1350. unsigned int pipe)
  1351. {
  1352. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1353. /* vblank is not initialized (IRQ not installed ?), or has been freed */
  1354. if (!drm_dev_has_vblank(dev))
  1355. return;
  1356. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
  1357. return;
  1358. /*
  1359. * To avoid all the problems that might happen if interrupts
  1360. * were enabled/disabled around or between these calls, we just
  1361. * have the kernel take a reference on the CRTC (just once though
  1362. * to avoid corrupting the count if multiple, mismatch calls occur),
  1363. * so that interrupts remain enabled in the interim.
  1364. */
  1365. if (!vblank->inmodeset) {
  1366. vblank->inmodeset = 0x1;
  1367. if (drm_vblank_get(dev, pipe) == 0)
  1368. vblank->inmodeset |= 0x2;
  1369. }
  1370. }
  1371. static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
  1372. unsigned int pipe)
  1373. {
  1374. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1375. /* vblank is not initialized (IRQ not installed ?), or has been freed */
  1376. if (!drm_dev_has_vblank(dev))
  1377. return;
  1378. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
  1379. return;
  1380. if (vblank->inmodeset) {
  1381. spin_lock_irq(&dev->vbl_lock);
  1382. drm_reset_vblank_timestamp(dev, pipe);
  1383. spin_unlock_irq(&dev->vbl_lock);
  1384. if (vblank->inmodeset & 0x2)
  1385. drm_vblank_put(dev, pipe);
  1386. vblank->inmodeset = 0;
  1387. }
  1388. }
  1389. int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
  1390. struct drm_file *file_priv)
  1391. {
  1392. struct drm_modeset_ctl *modeset = data;
  1393. unsigned int pipe;
  1394. /* If drm_vblank_init() hasn't been called yet, just no-op */
  1395. if (!drm_dev_has_vblank(dev))
  1396. return 0;
  1397. /* KMS drivers handle this internally */
  1398. if (!drm_core_check_feature(dev, DRIVER_LEGACY))
  1399. return 0;
  1400. pipe = modeset->crtc;
  1401. if (pipe >= dev->num_crtcs)
  1402. return -EINVAL;
  1403. switch (modeset->cmd) {
  1404. case _DRM_PRE_MODESET:
  1405. drm_legacy_vblank_pre_modeset(dev, pipe);
  1406. break;
  1407. case _DRM_POST_MODESET:
  1408. drm_legacy_vblank_post_modeset(dev, pipe);
  1409. break;
  1410. default:
  1411. return -EINVAL;
  1412. }
  1413. return 0;
  1414. }
  1415. static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
  1416. u64 req_seq,
  1417. union drm_wait_vblank *vblwait,
  1418. struct drm_file *file_priv)
  1419. {
  1420. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1421. struct drm_pending_vblank_event *e;
  1422. ktime_t now;
  1423. u64 seq;
  1424. int ret;
  1425. e = kzalloc(sizeof(*e), GFP_KERNEL);
  1426. if (e == NULL) {
  1427. ret = -ENOMEM;
  1428. goto err_put;
  1429. }
  1430. e->pipe = pipe;
  1431. e->event.base.type = DRM_EVENT_VBLANK;
  1432. e->event.base.length = sizeof(e->event.vbl);
  1433. e->event.vbl.user_data = vblwait->request.signal;
  1434. e->event.vbl.crtc_id = 0;
  1435. if (drm_core_check_feature(dev, DRIVER_MODESET)) {
  1436. struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
  1437. if (crtc)
  1438. e->event.vbl.crtc_id = crtc->base.id;
  1439. }
  1440. spin_lock_irq(&dev->event_lock);
  1441. /*
  1442. * drm_crtc_vblank_off() might have been called after we called
  1443. * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
  1444. * vblank disable, so no need for further locking. The reference from
  1445. * drm_vblank_get() protects against vblank disable from another source.
  1446. */
  1447. if (!READ_ONCE(vblank->enabled)) {
  1448. ret = -EINVAL;
  1449. goto err_unlock;
  1450. }
  1451. ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
  1452. &e->event.base);
  1453. if (ret)
  1454. goto err_unlock;
  1455. seq = drm_vblank_count_and_time(dev, pipe, &now);
  1456. drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n",
  1457. req_seq, seq, pipe);
  1458. trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
  1459. e->sequence = req_seq;
  1460. if (drm_vblank_passed(seq, req_seq)) {
  1461. drm_vblank_put(dev, pipe);
  1462. send_vblank_event(dev, e, seq, now);
  1463. vblwait->reply.sequence = seq;
  1464. } else {
  1465. /* drm_handle_vblank_events will call drm_vblank_put */
  1466. list_add_tail(&e->base.link, &dev->vblank_event_list);
  1467. vblwait->reply.sequence = req_seq;
  1468. }
  1469. spin_unlock_irq(&dev->event_lock);
  1470. return 0;
  1471. err_unlock:
  1472. spin_unlock_irq(&dev->event_lock);
  1473. kfree(e);
  1474. err_put:
  1475. drm_vblank_put(dev, pipe);
  1476. return ret;
  1477. }
  1478. static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
  1479. {
  1480. if (vblwait->request.sequence)
  1481. return false;
  1482. return _DRM_VBLANK_RELATIVE ==
  1483. (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
  1484. _DRM_VBLANK_EVENT |
  1485. _DRM_VBLANK_NEXTONMISS));
  1486. }
  1487. /*
  1488. * Widen a 32-bit param to 64-bits.
  1489. *
  1490. * \param narrow 32-bit value (missing upper 32 bits)
  1491. * \param near 64-bit value that should be 'close' to near
  1492. *
  1493. * This function returns a 64-bit value using the lower 32-bits from
  1494. * 'narrow' and constructing the upper 32-bits so that the result is
  1495. * as close as possible to 'near'.
  1496. */
  1497. static u64 widen_32_to_64(u32 narrow, u64 near)
  1498. {
  1499. return near + (s32) (narrow - near);
  1500. }
  1501. static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
  1502. struct drm_wait_vblank_reply *reply)
  1503. {
  1504. ktime_t now;
  1505. struct timespec64 ts;
  1506. /*
  1507. * drm_wait_vblank_reply is a UAPI structure that uses 'long'
  1508. * to store the seconds. This is safe as we always use monotonic
  1509. * timestamps since linux-4.15.
  1510. */
  1511. reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
  1512. ts = ktime_to_timespec64(now);
  1513. reply->tval_sec = (u32)ts.tv_sec;
  1514. reply->tval_usec = ts.tv_nsec / 1000;
  1515. }
  1516. static bool drm_wait_vblank_supported(struct drm_device *dev)
  1517. {
  1518. #if IS_ENABLED(CONFIG_DRM_LEGACY)
  1519. if (unlikely(drm_core_check_feature(dev, DRIVER_LEGACY)))
  1520. return dev->irq_enabled;
  1521. #endif
  1522. return drm_dev_has_vblank(dev);
  1523. }
  1524. int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
  1525. struct drm_file *file_priv)
  1526. {
  1527. struct drm_crtc *crtc;
  1528. struct drm_vblank_crtc *vblank;
  1529. union drm_wait_vblank *vblwait = data;
  1530. int ret;
  1531. u64 req_seq, seq;
  1532. unsigned int pipe_index;
  1533. unsigned int flags, pipe, high_pipe;
  1534. if (!drm_wait_vblank_supported(dev))
  1535. return -EOPNOTSUPP;
  1536. if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
  1537. return -EINVAL;
  1538. if (vblwait->request.type &
  1539. ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
  1540. _DRM_VBLANK_HIGH_CRTC_MASK)) {
  1541. drm_dbg_core(dev,
  1542. "Unsupported type value 0x%x, supported mask 0x%x\n",
  1543. vblwait->request.type,
  1544. (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
  1545. _DRM_VBLANK_HIGH_CRTC_MASK));
  1546. return -EINVAL;
  1547. }
  1548. flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
  1549. high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
  1550. if (high_pipe)
  1551. pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
  1552. else
  1553. pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
  1554. /* Convert lease-relative crtc index into global crtc index */
  1555. if (drm_core_check_feature(dev, DRIVER_MODESET)) {
  1556. pipe = 0;
  1557. drm_for_each_crtc(crtc, dev) {
  1558. if (drm_lease_held(file_priv, crtc->base.id)) {
  1559. if (pipe_index == 0)
  1560. break;
  1561. pipe_index--;
  1562. }
  1563. pipe++;
  1564. }
  1565. } else {
  1566. pipe = pipe_index;
  1567. }
  1568. if (pipe >= dev->num_crtcs)
  1569. return -EINVAL;
  1570. vblank = &dev->vblank[pipe];
  1571. /* If the counter is currently enabled and accurate, short-circuit
  1572. * queries to return the cached timestamp of the last vblank.
  1573. */
  1574. if (dev->vblank_disable_immediate &&
  1575. drm_wait_vblank_is_query(vblwait) &&
  1576. READ_ONCE(vblank->enabled)) {
  1577. drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
  1578. return 0;
  1579. }
  1580. ret = drm_vblank_get(dev, pipe);
  1581. if (ret) {
  1582. drm_dbg_core(dev,
  1583. "crtc %d failed to acquire vblank counter, %d\n",
  1584. pipe, ret);
  1585. return ret;
  1586. }
  1587. seq = drm_vblank_count(dev, pipe);
  1588. switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
  1589. case _DRM_VBLANK_RELATIVE:
  1590. req_seq = seq + vblwait->request.sequence;
  1591. vblwait->request.sequence = req_seq;
  1592. vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
  1593. break;
  1594. case _DRM_VBLANK_ABSOLUTE:
  1595. req_seq = widen_32_to_64(vblwait->request.sequence, seq);
  1596. break;
  1597. default:
  1598. ret = -EINVAL;
  1599. goto done;
  1600. }
  1601. if ((flags & _DRM_VBLANK_NEXTONMISS) &&
  1602. drm_vblank_passed(seq, req_seq)) {
  1603. req_seq = seq + 1;
  1604. vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
  1605. vblwait->request.sequence = req_seq;
  1606. }
  1607. if (flags & _DRM_VBLANK_EVENT) {
  1608. /* must hold on to the vblank ref until the event fires
  1609. * drm_vblank_put will be called asynchronously
  1610. */
  1611. return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
  1612. }
  1613. if (req_seq != seq) {
  1614. int wait;
  1615. drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n",
  1616. req_seq, pipe);
  1617. wait = wait_event_interruptible_timeout(vblank->queue,
  1618. drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
  1619. !READ_ONCE(vblank->enabled),
  1620. msecs_to_jiffies(3000));
  1621. switch (wait) {
  1622. case 0:
  1623. /* timeout */
  1624. ret = -EBUSY;
  1625. break;
  1626. case -ERESTARTSYS:
  1627. /* interrupted by signal */
  1628. ret = -EINTR;
  1629. break;
  1630. default:
  1631. ret = 0;
  1632. break;
  1633. }
  1634. }
  1635. if (ret != -EINTR) {
  1636. drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
  1637. drm_dbg_core(dev, "crtc %d returning %u to client\n",
  1638. pipe, vblwait->reply.sequence);
  1639. } else {
  1640. drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n",
  1641. pipe);
  1642. }
  1643. done:
  1644. drm_vblank_put(dev, pipe);
  1645. return ret;
  1646. }
  1647. static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
  1648. {
  1649. struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
  1650. bool high_prec = false;
  1651. struct drm_pending_vblank_event *e, *t;
  1652. ktime_t now;
  1653. u64 seq;
  1654. assert_spin_locked(&dev->event_lock);
  1655. seq = drm_vblank_count_and_time(dev, pipe, &now);
  1656. list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
  1657. if (e->pipe != pipe)
  1658. continue;
  1659. if (!drm_vblank_passed(seq, e->sequence))
  1660. continue;
  1661. drm_dbg_core(dev, "vblank event on %llu, current %llu\n",
  1662. e->sequence, seq);
  1663. list_del(&e->base.link);
  1664. drm_vblank_put(dev, pipe);
  1665. send_vblank_event(dev, e, seq, now);
  1666. }
  1667. if (crtc && crtc->funcs->get_vblank_timestamp)
  1668. high_prec = true;
  1669. trace_drm_vblank_event(pipe, seq, now, high_prec);
  1670. }
  1671. /**
  1672. * drm_handle_vblank - handle a vblank event
  1673. * @dev: DRM device
  1674. * @pipe: index of CRTC where this event occurred
  1675. *
  1676. * Drivers should call this routine in their vblank interrupt handlers to
  1677. * update the vblank counter and send any signals that may be pending.
  1678. *
  1679. * This is the legacy version of drm_crtc_handle_vblank().
  1680. */
  1681. bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
  1682. {
  1683. struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  1684. unsigned long irqflags;
  1685. bool disable_irq;
  1686. if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev)))
  1687. return false;
  1688. if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
  1689. return false;
  1690. spin_lock_irqsave(&dev->event_lock, irqflags);
  1691. /* Need timestamp lock to prevent concurrent execution with
  1692. * vblank enable/disable, as this would cause inconsistent
  1693. * or corrupted timestamps and vblank counts.
  1694. */
  1695. spin_lock(&dev->vblank_time_lock);
  1696. /* Vblank irq handling disabled. Nothing to do. */
  1697. if (!vblank->enabled) {
  1698. spin_unlock(&dev->vblank_time_lock);
  1699. spin_unlock_irqrestore(&dev->event_lock, irqflags);
  1700. return false;
  1701. }
  1702. drm_update_vblank_count(dev, pipe, true);
  1703. spin_unlock(&dev->vblank_time_lock);
  1704. wake_up(&vblank->queue);
  1705. /* With instant-off, we defer disabling the interrupt until after
  1706. * we finish processing the following vblank after all events have
  1707. * been signaled. The disable has to be last (after
  1708. * drm_handle_vblank_events) so that the timestamp is always accurate.
  1709. */
  1710. disable_irq = (dev->vblank_disable_immediate &&
  1711. drm_vblank_offdelay > 0 &&
  1712. !atomic_read(&vblank->refcount));
  1713. drm_handle_vblank_events(dev, pipe);
  1714. drm_handle_vblank_works(vblank);
  1715. spin_unlock_irqrestore(&dev->event_lock, irqflags);
  1716. if (disable_irq)
  1717. vblank_disable_fn(&vblank->disable_timer);
  1718. return true;
  1719. }
  1720. EXPORT_SYMBOL(drm_handle_vblank);
  1721. /**
  1722. * drm_crtc_handle_vblank - handle a vblank event
  1723. * @crtc: where this event occurred
  1724. *
  1725. * Drivers should call this routine in their vblank interrupt handlers to
  1726. * update the vblank counter and send any signals that may be pending.
  1727. *
  1728. * This is the native KMS version of drm_handle_vblank().
  1729. *
  1730. * Note that for a given vblank counter value drm_crtc_handle_vblank()
  1731. * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
  1732. * provide a barrier: Any writes done before calling
  1733. * drm_crtc_handle_vblank() will be visible to callers of the later
  1734. * functions, if the vblank count is the same or a later one.
  1735. *
  1736. * See also &drm_vblank_crtc.count.
  1737. *
  1738. * Returns:
  1739. * True if the event was successfully handled, false on failure.
  1740. */
  1741. bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
  1742. {
  1743. return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
  1744. }
  1745. EXPORT_SYMBOL(drm_crtc_handle_vblank);
  1746. /*
  1747. * Get crtc VBLANK count.
  1748. *
  1749. * \param dev DRM device
  1750. * \param data user argument, pointing to a drm_crtc_get_sequence structure.
  1751. * \param file_priv drm file private for the user's open file descriptor
  1752. */
  1753. int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
  1754. struct drm_file *file_priv)
  1755. {
  1756. struct drm_crtc *crtc;
  1757. struct drm_vblank_crtc *vblank;
  1758. int pipe;
  1759. struct drm_crtc_get_sequence *get_seq = data;
  1760. ktime_t now;
  1761. bool vblank_enabled;
  1762. int ret;
  1763. if (!drm_core_check_feature(dev, DRIVER_MODESET))
  1764. return -EOPNOTSUPP;
  1765. if (!drm_dev_has_vblank(dev))
  1766. return -EOPNOTSUPP;
  1767. crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
  1768. if (!crtc)
  1769. return -ENOENT;
  1770. pipe = drm_crtc_index(crtc);
  1771. vblank = &dev->vblank[pipe];
  1772. vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
  1773. if (!vblank_enabled) {
  1774. ret = drm_crtc_vblank_get(crtc);
  1775. if (ret) {
  1776. drm_dbg_core(dev,
  1777. "crtc %d failed to acquire vblank counter, %d\n",
  1778. pipe, ret);
  1779. return ret;
  1780. }
  1781. }
  1782. drm_modeset_lock(&crtc->mutex, NULL);
  1783. if (crtc->state)
  1784. get_seq->active = crtc->state->enable;
  1785. else
  1786. get_seq->active = crtc->enabled;
  1787. drm_modeset_unlock(&crtc->mutex);
  1788. get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
  1789. get_seq->sequence_ns = ktime_to_ns(now);
  1790. if (!vblank_enabled)
  1791. drm_crtc_vblank_put(crtc);
  1792. return 0;
  1793. }
  1794. /*
  1795. * Queue a event for VBLANK sequence
  1796. *
  1797. * \param dev DRM device
  1798. * \param data user argument, pointing to a drm_crtc_queue_sequence structure.
  1799. * \param file_priv drm file private for the user's open file descriptor
  1800. */
  1801. int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
  1802. struct drm_file *file_priv)
  1803. {
  1804. struct drm_crtc *crtc;
  1805. struct drm_vblank_crtc *vblank;
  1806. int pipe;
  1807. struct drm_crtc_queue_sequence *queue_seq = data;
  1808. ktime_t now;
  1809. struct drm_pending_vblank_event *e;
  1810. u32 flags;
  1811. u64 seq;
  1812. u64 req_seq;
  1813. int ret;
  1814. if (!drm_core_check_feature(dev, DRIVER_MODESET))
  1815. return -EOPNOTSUPP;
  1816. if (!drm_dev_has_vblank(dev))
  1817. return -EOPNOTSUPP;
  1818. crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
  1819. if (!crtc)
  1820. return -ENOENT;
  1821. flags = queue_seq->flags;
  1822. /* Check valid flag bits */
  1823. if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
  1824. DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
  1825. return -EINVAL;
  1826. pipe = drm_crtc_index(crtc);
  1827. vblank = &dev->vblank[pipe];
  1828. e = kzalloc(sizeof(*e), GFP_KERNEL);
  1829. if (e == NULL)
  1830. return -ENOMEM;
  1831. ret = drm_crtc_vblank_get(crtc);
  1832. if (ret) {
  1833. drm_dbg_core(dev,
  1834. "crtc %d failed to acquire vblank counter, %d\n",
  1835. pipe, ret);
  1836. goto err_free;
  1837. }
  1838. seq = drm_vblank_count_and_time(dev, pipe, &now);
  1839. req_seq = queue_seq->sequence;
  1840. if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
  1841. req_seq += seq;
  1842. if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, req_seq))
  1843. req_seq = seq + 1;
  1844. e->pipe = pipe;
  1845. e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
  1846. e->event.base.length = sizeof(e->event.seq);
  1847. e->event.seq.user_data = queue_seq->user_data;
  1848. spin_lock_irq(&dev->event_lock);
  1849. /*
  1850. * drm_crtc_vblank_off() might have been called after we called
  1851. * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
  1852. * vblank disable, so no need for further locking. The reference from
  1853. * drm_crtc_vblank_get() protects against vblank disable from another source.
  1854. */
  1855. if (!READ_ONCE(vblank->enabled)) {
  1856. ret = -EINVAL;
  1857. goto err_unlock;
  1858. }
  1859. ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
  1860. &e->event.base);
  1861. if (ret)
  1862. goto err_unlock;
  1863. e->sequence = req_seq;
  1864. if (drm_vblank_passed(seq, req_seq)) {
  1865. drm_crtc_vblank_put(crtc);
  1866. send_vblank_event(dev, e, seq, now);
  1867. queue_seq->sequence = seq;
  1868. } else {
  1869. /* drm_handle_vblank_events will call drm_vblank_put */
  1870. list_add_tail(&e->base.link, &dev->vblank_event_list);
  1871. queue_seq->sequence = req_seq;
  1872. }
  1873. spin_unlock_irq(&dev->event_lock);
  1874. return 0;
  1875. err_unlock:
  1876. spin_unlock_irq(&dev->event_lock);
  1877. drm_crtc_vblank_put(crtc);
  1878. err_free:
  1879. kfree(e);
  1880. return ret;
  1881. }