relay.c 30 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247
  1. /*
  2. * Public API and common code for kernel->userspace relay file support.
  3. *
  4. * See Documentation/filesystems/relay.rst for an overview.
  5. *
  6. * Copyright (C) 2002-2005 - Tom Zanussi ([email protected]), IBM Corp
  7. * Copyright (C) 1999-2005 - Karim Yaghmour ([email protected])
  8. *
  9. * Moved to kernel/relay.c by Paul Mundt, 2006.
  10. * November 2006 - CPU hotplug support by Mathieu Desnoyers
  11. * ([email protected])
  12. *
  13. * This file is released under the GPL.
  14. */
  15. #include <linux/errno.h>
  16. #include <linux/stddef.h>
  17. #include <linux/slab.h>
  18. #include <linux/export.h>
  19. #include <linux/string.h>
  20. #include <linux/relay.h>
  21. #include <linux/vmalloc.h>
  22. #include <linux/mm.h>
  23. #include <linux/cpu.h>
  24. #include <linux/splice.h>
  25. /* list of open channels, for cpu hotplug */
  26. static DEFINE_MUTEX(relay_channels_mutex);
  27. static LIST_HEAD(relay_channels);
  28. /*
  29. * fault() vm_op implementation for relay file mapping.
  30. */
  31. static vm_fault_t relay_buf_fault(struct vm_fault *vmf)
  32. {
  33. struct page *page;
  34. struct rchan_buf *buf = vmf->vma->vm_private_data;
  35. pgoff_t pgoff = vmf->pgoff;
  36. if (!buf)
  37. return VM_FAULT_OOM;
  38. page = vmalloc_to_page(buf->start + (pgoff << PAGE_SHIFT));
  39. if (!page)
  40. return VM_FAULT_SIGBUS;
  41. get_page(page);
  42. vmf->page = page;
  43. return 0;
  44. }
  45. /*
  46. * vm_ops for relay file mappings.
  47. */
  48. static const struct vm_operations_struct relay_file_mmap_ops = {
  49. .fault = relay_buf_fault,
  50. };
  51. /*
  52. * allocate an array of pointers of struct page
  53. */
  54. static struct page **relay_alloc_page_array(unsigned int n_pages)
  55. {
  56. return kvcalloc(n_pages, sizeof(struct page *), GFP_KERNEL);
  57. }
  58. /*
  59. * free an array of pointers of struct page
  60. */
  61. static void relay_free_page_array(struct page **array)
  62. {
  63. kvfree(array);
  64. }
  65. /**
  66. * relay_mmap_buf: - mmap channel buffer to process address space
  67. * @buf: relay channel buffer
  68. * @vma: vm_area_struct describing memory to be mapped
  69. *
  70. * Returns 0 if ok, negative on error
  71. *
  72. * Caller should already have grabbed mmap_lock.
  73. */
  74. static int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma)
  75. {
  76. unsigned long length = vma->vm_end - vma->vm_start;
  77. if (!buf)
  78. return -EBADF;
  79. if (length != (unsigned long)buf->chan->alloc_size)
  80. return -EINVAL;
  81. vma->vm_ops = &relay_file_mmap_ops;
  82. vm_flags_set(vma, VM_DONTEXPAND);
  83. vma->vm_private_data = buf;
  84. return 0;
  85. }
  86. /**
  87. * relay_alloc_buf - allocate a channel buffer
  88. * @buf: the buffer struct
  89. * @size: total size of the buffer
  90. *
  91. * Returns a pointer to the resulting buffer, %NULL if unsuccessful. The
  92. * passed in size will get page aligned, if it isn't already.
  93. */
  94. static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size)
  95. {
  96. void *mem;
  97. unsigned int i, j, n_pages;
  98. *size = PAGE_ALIGN(*size);
  99. n_pages = *size >> PAGE_SHIFT;
  100. buf->page_array = relay_alloc_page_array(n_pages);
  101. if (!buf->page_array)
  102. return NULL;
  103. for (i = 0; i < n_pages; i++) {
  104. buf->page_array[i] = alloc_page(GFP_KERNEL);
  105. if (unlikely(!buf->page_array[i]))
  106. goto depopulate;
  107. set_page_private(buf->page_array[i], (unsigned long)buf);
  108. }
  109. mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL);
  110. if (!mem)
  111. goto depopulate;
  112. memset(mem, 0, *size);
  113. buf->page_count = n_pages;
  114. return mem;
  115. depopulate:
  116. for (j = 0; j < i; j++)
  117. __free_page(buf->page_array[j]);
  118. relay_free_page_array(buf->page_array);
  119. return NULL;
  120. }
  121. /**
  122. * relay_create_buf - allocate and initialize a channel buffer
  123. * @chan: the relay channel
  124. *
  125. * Returns channel buffer if successful, %NULL otherwise.
  126. */
  127. static struct rchan_buf *relay_create_buf(struct rchan *chan)
  128. {
  129. struct rchan_buf *buf;
  130. if (chan->n_subbufs > KMALLOC_MAX_SIZE / sizeof(size_t))
  131. return NULL;
  132. buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL);
  133. if (!buf)
  134. return NULL;
  135. buf->padding = kmalloc_array(chan->n_subbufs, sizeof(size_t),
  136. GFP_KERNEL);
  137. if (!buf->padding)
  138. goto free_buf;
  139. buf->start = relay_alloc_buf(buf, &chan->alloc_size);
  140. if (!buf->start)
  141. goto free_buf;
  142. buf->chan = chan;
  143. kref_get(&buf->chan->kref);
  144. return buf;
  145. free_buf:
  146. kfree(buf->padding);
  147. kfree(buf);
  148. return NULL;
  149. }
  150. /**
  151. * relay_destroy_channel - free the channel struct
  152. * @kref: target kernel reference that contains the relay channel
  153. *
  154. * Should only be called from kref_put().
  155. */
  156. static void relay_destroy_channel(struct kref *kref)
  157. {
  158. struct rchan *chan = container_of(kref, struct rchan, kref);
  159. free_percpu(chan->buf);
  160. kfree(chan);
  161. }
  162. /**
  163. * relay_destroy_buf - destroy an rchan_buf struct and associated buffer
  164. * @buf: the buffer struct
  165. */
  166. static void relay_destroy_buf(struct rchan_buf *buf)
  167. {
  168. struct rchan *chan = buf->chan;
  169. unsigned int i;
  170. if (likely(buf->start)) {
  171. vunmap(buf->start);
  172. for (i = 0; i < buf->page_count; i++)
  173. __free_page(buf->page_array[i]);
  174. relay_free_page_array(buf->page_array);
  175. }
  176. *per_cpu_ptr(chan->buf, buf->cpu) = NULL;
  177. kfree(buf->padding);
  178. kfree(buf);
  179. kref_put(&chan->kref, relay_destroy_channel);
  180. }
  181. /**
  182. * relay_remove_buf - remove a channel buffer
  183. * @kref: target kernel reference that contains the relay buffer
  184. *
  185. * Removes the file from the filesystem, which also frees the
  186. * rchan_buf_struct and the channel buffer. Should only be called from
  187. * kref_put().
  188. */
  189. static void relay_remove_buf(struct kref *kref)
  190. {
  191. struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref);
  192. relay_destroy_buf(buf);
  193. }
  194. /**
  195. * relay_buf_empty - boolean, is the channel buffer empty?
  196. * @buf: channel buffer
  197. *
  198. * Returns 1 if the buffer is empty, 0 otherwise.
  199. */
  200. static int relay_buf_empty(struct rchan_buf *buf)
  201. {
  202. return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1;
  203. }
  204. /**
  205. * relay_buf_full - boolean, is the channel buffer full?
  206. * @buf: channel buffer
  207. *
  208. * Returns 1 if the buffer is full, 0 otherwise.
  209. */
  210. int relay_buf_full(struct rchan_buf *buf)
  211. {
  212. size_t ready = buf->subbufs_produced - buf->subbufs_consumed;
  213. return (ready >= buf->chan->n_subbufs) ? 1 : 0;
  214. }
  215. EXPORT_SYMBOL_GPL(relay_buf_full);
  216. /*
  217. * High-level relay kernel API and associated functions.
  218. */
  219. static int relay_subbuf_start(struct rchan_buf *buf, void *subbuf,
  220. void *prev_subbuf, size_t prev_padding)
  221. {
  222. if (!buf->chan->cb->subbuf_start)
  223. return !relay_buf_full(buf);
  224. return buf->chan->cb->subbuf_start(buf, subbuf,
  225. prev_subbuf, prev_padding);
  226. }
  227. /**
  228. * wakeup_readers - wake up readers waiting on a channel
  229. * @work: contains the channel buffer
  230. *
  231. * This is the function used to defer reader waking
  232. */
  233. static void wakeup_readers(struct irq_work *work)
  234. {
  235. struct rchan_buf *buf;
  236. buf = container_of(work, struct rchan_buf, wakeup_work);
  237. wake_up_interruptible(&buf->read_wait);
  238. }
  239. /**
  240. * __relay_reset - reset a channel buffer
  241. * @buf: the channel buffer
  242. * @init: 1 if this is a first-time initialization
  243. *
  244. * See relay_reset() for description of effect.
  245. */
  246. static void __relay_reset(struct rchan_buf *buf, unsigned int init)
  247. {
  248. size_t i;
  249. if (init) {
  250. init_waitqueue_head(&buf->read_wait);
  251. kref_init(&buf->kref);
  252. init_irq_work(&buf->wakeup_work, wakeup_readers);
  253. } else {
  254. irq_work_sync(&buf->wakeup_work);
  255. }
  256. buf->subbufs_produced = 0;
  257. buf->subbufs_consumed = 0;
  258. buf->bytes_consumed = 0;
  259. buf->finalized = 0;
  260. buf->data = buf->start;
  261. buf->offset = 0;
  262. for (i = 0; i < buf->chan->n_subbufs; i++)
  263. buf->padding[i] = 0;
  264. relay_subbuf_start(buf, buf->data, NULL, 0);
  265. }
  266. /**
  267. * relay_reset - reset the channel
  268. * @chan: the channel
  269. *
  270. * This has the effect of erasing all data from all channel buffers
  271. * and restarting the channel in its initial state. The buffers
  272. * are not freed, so any mappings are still in effect.
  273. *
  274. * NOTE. Care should be taken that the channel isn't actually
  275. * being used by anything when this call is made.
  276. */
  277. void relay_reset(struct rchan *chan)
  278. {
  279. struct rchan_buf *buf;
  280. unsigned int i;
  281. if (!chan)
  282. return;
  283. if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) {
  284. __relay_reset(buf, 0);
  285. return;
  286. }
  287. mutex_lock(&relay_channels_mutex);
  288. for_each_possible_cpu(i)
  289. if ((buf = *per_cpu_ptr(chan->buf, i)))
  290. __relay_reset(buf, 0);
  291. mutex_unlock(&relay_channels_mutex);
  292. }
  293. EXPORT_SYMBOL_GPL(relay_reset);
  294. static inline void relay_set_buf_dentry(struct rchan_buf *buf,
  295. struct dentry *dentry)
  296. {
  297. buf->dentry = dentry;
  298. d_inode(buf->dentry)->i_size = buf->early_bytes;
  299. }
  300. static struct dentry *relay_create_buf_file(struct rchan *chan,
  301. struct rchan_buf *buf,
  302. unsigned int cpu)
  303. {
  304. struct dentry *dentry;
  305. char *tmpname;
  306. tmpname = kzalloc(NAME_MAX + 1, GFP_KERNEL);
  307. if (!tmpname)
  308. return NULL;
  309. snprintf(tmpname, NAME_MAX, "%s%d", chan->base_filename, cpu);
  310. /* Create file in fs */
  311. dentry = chan->cb->create_buf_file(tmpname, chan->parent,
  312. S_IRUSR, buf,
  313. &chan->is_global);
  314. if (IS_ERR(dentry))
  315. dentry = NULL;
  316. kfree(tmpname);
  317. return dentry;
  318. }
  319. /*
  320. * relay_open_buf - create a new relay channel buffer
  321. *
  322. * used by relay_open() and CPU hotplug.
  323. */
  324. static struct rchan_buf *relay_open_buf(struct rchan *chan, unsigned int cpu)
  325. {
  326. struct rchan_buf *buf = NULL;
  327. struct dentry *dentry;
  328. if (chan->is_global)
  329. return *per_cpu_ptr(chan->buf, 0);
  330. buf = relay_create_buf(chan);
  331. if (!buf)
  332. return NULL;
  333. if (chan->has_base_filename) {
  334. dentry = relay_create_buf_file(chan, buf, cpu);
  335. if (!dentry)
  336. goto free_buf;
  337. relay_set_buf_dentry(buf, dentry);
  338. } else {
  339. /* Only retrieve global info, nothing more, nothing less */
  340. dentry = chan->cb->create_buf_file(NULL, NULL,
  341. S_IRUSR, buf,
  342. &chan->is_global);
  343. if (IS_ERR_OR_NULL(dentry))
  344. goto free_buf;
  345. }
  346. buf->cpu = cpu;
  347. __relay_reset(buf, 1);
  348. if(chan->is_global) {
  349. *per_cpu_ptr(chan->buf, 0) = buf;
  350. buf->cpu = 0;
  351. }
  352. return buf;
  353. free_buf:
  354. relay_destroy_buf(buf);
  355. return NULL;
  356. }
  357. /**
  358. * relay_close_buf - close a channel buffer
  359. * @buf: channel buffer
  360. *
  361. * Marks the buffer finalized and restores the default callbacks.
  362. * The channel buffer and channel buffer data structure are then freed
  363. * automatically when the last reference is given up.
  364. */
  365. static void relay_close_buf(struct rchan_buf *buf)
  366. {
  367. buf->finalized = 1;
  368. irq_work_sync(&buf->wakeup_work);
  369. buf->chan->cb->remove_buf_file(buf->dentry);
  370. kref_put(&buf->kref, relay_remove_buf);
  371. }
  372. int relay_prepare_cpu(unsigned int cpu)
  373. {
  374. struct rchan *chan;
  375. struct rchan_buf *buf;
  376. mutex_lock(&relay_channels_mutex);
  377. list_for_each_entry(chan, &relay_channels, list) {
  378. if (*per_cpu_ptr(chan->buf, cpu))
  379. continue;
  380. buf = relay_open_buf(chan, cpu);
  381. if (!buf) {
  382. pr_err("relay: cpu %d buffer creation failed\n", cpu);
  383. mutex_unlock(&relay_channels_mutex);
  384. return -ENOMEM;
  385. }
  386. *per_cpu_ptr(chan->buf, cpu) = buf;
  387. }
  388. mutex_unlock(&relay_channels_mutex);
  389. return 0;
  390. }
  391. /**
  392. * relay_open - create a new relay channel
  393. * @base_filename: base name of files to create, %NULL for buffering only
  394. * @parent: dentry of parent directory, %NULL for root directory or buffer
  395. * @subbuf_size: size of sub-buffers
  396. * @n_subbufs: number of sub-buffers
  397. * @cb: client callback functions
  398. * @private_data: user-defined data
  399. *
  400. * Returns channel pointer if successful, %NULL otherwise.
  401. *
  402. * Creates a channel buffer for each cpu using the sizes and
  403. * attributes specified. The created channel buffer files
  404. * will be named base_filename0...base_filenameN-1. File
  405. * permissions will be %S_IRUSR.
  406. *
  407. * If opening a buffer (@parent = NULL) that you later wish to register
  408. * in a filesystem, call relay_late_setup_files() once the @parent dentry
  409. * is available.
  410. */
  411. struct rchan *relay_open(const char *base_filename,
  412. struct dentry *parent,
  413. size_t subbuf_size,
  414. size_t n_subbufs,
  415. const struct rchan_callbacks *cb,
  416. void *private_data)
  417. {
  418. unsigned int i;
  419. struct rchan *chan;
  420. struct rchan_buf *buf;
  421. if (!(subbuf_size && n_subbufs))
  422. return NULL;
  423. if (subbuf_size > UINT_MAX / n_subbufs)
  424. return NULL;
  425. if (!cb || !cb->create_buf_file || !cb->remove_buf_file)
  426. return NULL;
  427. chan = kzalloc(sizeof(struct rchan), GFP_KERNEL);
  428. if (!chan)
  429. return NULL;
  430. chan->buf = alloc_percpu(struct rchan_buf *);
  431. if (!chan->buf) {
  432. kfree(chan);
  433. return NULL;
  434. }
  435. chan->version = RELAYFS_CHANNEL_VERSION;
  436. chan->n_subbufs = n_subbufs;
  437. chan->subbuf_size = subbuf_size;
  438. chan->alloc_size = PAGE_ALIGN(subbuf_size * n_subbufs);
  439. chan->parent = parent;
  440. chan->private_data = private_data;
  441. if (base_filename) {
  442. chan->has_base_filename = 1;
  443. strlcpy(chan->base_filename, base_filename, NAME_MAX);
  444. }
  445. chan->cb = cb;
  446. kref_init(&chan->kref);
  447. mutex_lock(&relay_channels_mutex);
  448. for_each_online_cpu(i) {
  449. buf = relay_open_buf(chan, i);
  450. if (!buf)
  451. goto free_bufs;
  452. *per_cpu_ptr(chan->buf, i) = buf;
  453. }
  454. list_add(&chan->list, &relay_channels);
  455. mutex_unlock(&relay_channels_mutex);
  456. return chan;
  457. free_bufs:
  458. for_each_possible_cpu(i) {
  459. if ((buf = *per_cpu_ptr(chan->buf, i)))
  460. relay_close_buf(buf);
  461. }
  462. kref_put(&chan->kref, relay_destroy_channel);
  463. mutex_unlock(&relay_channels_mutex);
  464. return NULL;
  465. }
  466. EXPORT_SYMBOL_GPL(relay_open);
  467. struct rchan_percpu_buf_dispatcher {
  468. struct rchan_buf *buf;
  469. struct dentry *dentry;
  470. };
  471. /* Called in atomic context. */
  472. static void __relay_set_buf_dentry(void *info)
  473. {
  474. struct rchan_percpu_buf_dispatcher *p = info;
  475. relay_set_buf_dentry(p->buf, p->dentry);
  476. }
  477. /**
  478. * relay_late_setup_files - triggers file creation
  479. * @chan: channel to operate on
  480. * @base_filename: base name of files to create
  481. * @parent: dentry of parent directory, %NULL for root directory
  482. *
  483. * Returns 0 if successful, non-zero otherwise.
  484. *
  485. * Use to setup files for a previously buffer-only channel created
  486. * by relay_open() with a NULL parent dentry.
  487. *
  488. * For example, this is useful for perfomring early tracing in kernel,
  489. * before VFS is up and then exposing the early results once the dentry
  490. * is available.
  491. */
  492. int relay_late_setup_files(struct rchan *chan,
  493. const char *base_filename,
  494. struct dentry *parent)
  495. {
  496. int err = 0;
  497. unsigned int i, curr_cpu;
  498. unsigned long flags;
  499. struct dentry *dentry;
  500. struct rchan_buf *buf;
  501. struct rchan_percpu_buf_dispatcher disp;
  502. if (!chan || !base_filename)
  503. return -EINVAL;
  504. strlcpy(chan->base_filename, base_filename, NAME_MAX);
  505. mutex_lock(&relay_channels_mutex);
  506. /* Is chan already set up? */
  507. if (unlikely(chan->has_base_filename)) {
  508. mutex_unlock(&relay_channels_mutex);
  509. return -EEXIST;
  510. }
  511. chan->has_base_filename = 1;
  512. chan->parent = parent;
  513. if (chan->is_global) {
  514. err = -EINVAL;
  515. buf = *per_cpu_ptr(chan->buf, 0);
  516. if (!WARN_ON_ONCE(!buf)) {
  517. dentry = relay_create_buf_file(chan, buf, 0);
  518. if (dentry && !WARN_ON_ONCE(!chan->is_global)) {
  519. relay_set_buf_dentry(buf, dentry);
  520. err = 0;
  521. }
  522. }
  523. mutex_unlock(&relay_channels_mutex);
  524. return err;
  525. }
  526. curr_cpu = get_cpu();
  527. /*
  528. * The CPU hotplug notifier ran before us and created buffers with
  529. * no files associated. So it's safe to call relay_setup_buf_file()
  530. * on all currently online CPUs.
  531. */
  532. for_each_online_cpu(i) {
  533. buf = *per_cpu_ptr(chan->buf, i);
  534. if (unlikely(!buf)) {
  535. WARN_ONCE(1, KERN_ERR "CPU has no buffer!\n");
  536. err = -EINVAL;
  537. break;
  538. }
  539. dentry = relay_create_buf_file(chan, buf, i);
  540. if (unlikely(!dentry)) {
  541. err = -EINVAL;
  542. break;
  543. }
  544. if (curr_cpu == i) {
  545. local_irq_save(flags);
  546. relay_set_buf_dentry(buf, dentry);
  547. local_irq_restore(flags);
  548. } else {
  549. disp.buf = buf;
  550. disp.dentry = dentry;
  551. smp_mb();
  552. /* relay_channels_mutex must be held, so wait. */
  553. err = smp_call_function_single(i,
  554. __relay_set_buf_dentry,
  555. &disp, 1);
  556. }
  557. if (unlikely(err))
  558. break;
  559. }
  560. put_cpu();
  561. mutex_unlock(&relay_channels_mutex);
  562. return err;
  563. }
  564. EXPORT_SYMBOL_GPL(relay_late_setup_files);
  565. /**
  566. * relay_switch_subbuf - switch to a new sub-buffer
  567. * @buf: channel buffer
  568. * @length: size of current event
  569. *
  570. * Returns either the length passed in or 0 if full.
  571. *
  572. * Performs sub-buffer-switch tasks such as invoking callbacks,
  573. * updating padding counts, waking up readers, etc.
  574. */
  575. size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length)
  576. {
  577. void *old, *new;
  578. size_t old_subbuf, new_subbuf;
  579. if (unlikely(length > buf->chan->subbuf_size))
  580. goto toobig;
  581. if (buf->offset != buf->chan->subbuf_size + 1) {
  582. buf->prev_padding = buf->chan->subbuf_size - buf->offset;
  583. old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
  584. buf->padding[old_subbuf] = buf->prev_padding;
  585. buf->subbufs_produced++;
  586. if (buf->dentry)
  587. d_inode(buf->dentry)->i_size +=
  588. buf->chan->subbuf_size -
  589. buf->padding[old_subbuf];
  590. else
  591. buf->early_bytes += buf->chan->subbuf_size -
  592. buf->padding[old_subbuf];
  593. smp_mb();
  594. if (waitqueue_active(&buf->read_wait)) {
  595. /*
  596. * Calling wake_up_interruptible() from here
  597. * will deadlock if we happen to be logging
  598. * from the scheduler (trying to re-grab
  599. * rq->lock), so defer it.
  600. */
  601. irq_work_queue(&buf->wakeup_work);
  602. }
  603. }
  604. old = buf->data;
  605. new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
  606. new = buf->start + new_subbuf * buf->chan->subbuf_size;
  607. buf->offset = 0;
  608. if (!relay_subbuf_start(buf, new, old, buf->prev_padding)) {
  609. buf->offset = buf->chan->subbuf_size + 1;
  610. return 0;
  611. }
  612. buf->data = new;
  613. buf->padding[new_subbuf] = 0;
  614. if (unlikely(length + buf->offset > buf->chan->subbuf_size))
  615. goto toobig;
  616. return length;
  617. toobig:
  618. buf->chan->last_toobig = length;
  619. return 0;
  620. }
  621. EXPORT_SYMBOL_GPL(relay_switch_subbuf);
  622. /**
  623. * relay_subbufs_consumed - update the buffer's sub-buffers-consumed count
  624. * @chan: the channel
  625. * @cpu: the cpu associated with the channel buffer to update
  626. * @subbufs_consumed: number of sub-buffers to add to current buf's count
  627. *
  628. * Adds to the channel buffer's consumed sub-buffer count.
  629. * subbufs_consumed should be the number of sub-buffers newly consumed,
  630. * not the total consumed.
  631. *
  632. * NOTE. Kernel clients don't need to call this function if the channel
  633. * mode is 'overwrite'.
  634. */
  635. void relay_subbufs_consumed(struct rchan *chan,
  636. unsigned int cpu,
  637. size_t subbufs_consumed)
  638. {
  639. struct rchan_buf *buf;
  640. if (!chan || cpu >= NR_CPUS)
  641. return;
  642. buf = *per_cpu_ptr(chan->buf, cpu);
  643. if (!buf || subbufs_consumed > chan->n_subbufs)
  644. return;
  645. if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed)
  646. buf->subbufs_consumed = buf->subbufs_produced;
  647. else
  648. buf->subbufs_consumed += subbufs_consumed;
  649. }
  650. EXPORT_SYMBOL_GPL(relay_subbufs_consumed);
  651. /**
  652. * relay_close - close the channel
  653. * @chan: the channel
  654. *
  655. * Closes all channel buffers and frees the channel.
  656. */
  657. void relay_close(struct rchan *chan)
  658. {
  659. struct rchan_buf *buf;
  660. unsigned int i;
  661. if (!chan)
  662. return;
  663. mutex_lock(&relay_channels_mutex);
  664. if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0)))
  665. relay_close_buf(buf);
  666. else
  667. for_each_possible_cpu(i)
  668. if ((buf = *per_cpu_ptr(chan->buf, i)))
  669. relay_close_buf(buf);
  670. if (chan->last_toobig)
  671. printk(KERN_WARNING "relay: one or more items not logged "
  672. "[item size (%zd) > sub-buffer size (%zd)]\n",
  673. chan->last_toobig, chan->subbuf_size);
  674. list_del(&chan->list);
  675. kref_put(&chan->kref, relay_destroy_channel);
  676. mutex_unlock(&relay_channels_mutex);
  677. }
  678. EXPORT_SYMBOL_GPL(relay_close);
  679. /**
  680. * relay_flush - close the channel
  681. * @chan: the channel
  682. *
  683. * Flushes all channel buffers, i.e. forces buffer switch.
  684. */
  685. void relay_flush(struct rchan *chan)
  686. {
  687. struct rchan_buf *buf;
  688. unsigned int i;
  689. if (!chan)
  690. return;
  691. if (chan->is_global && (buf = *per_cpu_ptr(chan->buf, 0))) {
  692. relay_switch_subbuf(buf, 0);
  693. return;
  694. }
  695. mutex_lock(&relay_channels_mutex);
  696. for_each_possible_cpu(i)
  697. if ((buf = *per_cpu_ptr(chan->buf, i)))
  698. relay_switch_subbuf(buf, 0);
  699. mutex_unlock(&relay_channels_mutex);
  700. }
  701. EXPORT_SYMBOL_GPL(relay_flush);
  702. /**
  703. * relay_file_open - open file op for relay files
  704. * @inode: the inode
  705. * @filp: the file
  706. *
  707. * Increments the channel buffer refcount.
  708. */
  709. static int relay_file_open(struct inode *inode, struct file *filp)
  710. {
  711. struct rchan_buf *buf = inode->i_private;
  712. kref_get(&buf->kref);
  713. filp->private_data = buf;
  714. return nonseekable_open(inode, filp);
  715. }
  716. /**
  717. * relay_file_mmap - mmap file op for relay files
  718. * @filp: the file
  719. * @vma: the vma describing what to map
  720. *
  721. * Calls upon relay_mmap_buf() to map the file into user space.
  722. */
  723. static int relay_file_mmap(struct file *filp, struct vm_area_struct *vma)
  724. {
  725. struct rchan_buf *buf = filp->private_data;
  726. return relay_mmap_buf(buf, vma);
  727. }
  728. /**
  729. * relay_file_poll - poll file op for relay files
  730. * @filp: the file
  731. * @wait: poll table
  732. *
  733. * Poll implemention.
  734. */
  735. static __poll_t relay_file_poll(struct file *filp, poll_table *wait)
  736. {
  737. __poll_t mask = 0;
  738. struct rchan_buf *buf = filp->private_data;
  739. if (buf->finalized)
  740. return EPOLLERR;
  741. if (filp->f_mode & FMODE_READ) {
  742. poll_wait(filp, &buf->read_wait, wait);
  743. if (!relay_buf_empty(buf))
  744. mask |= EPOLLIN | EPOLLRDNORM;
  745. }
  746. return mask;
  747. }
  748. /**
  749. * relay_file_release - release file op for relay files
  750. * @inode: the inode
  751. * @filp: the file
  752. *
  753. * Decrements the channel refcount, as the filesystem is
  754. * no longer using it.
  755. */
  756. static int relay_file_release(struct inode *inode, struct file *filp)
  757. {
  758. struct rchan_buf *buf = filp->private_data;
  759. kref_put(&buf->kref, relay_remove_buf);
  760. return 0;
  761. }
  762. /*
  763. * relay_file_read_consume - update the consumed count for the buffer
  764. */
  765. static void relay_file_read_consume(struct rchan_buf *buf,
  766. size_t read_pos,
  767. size_t bytes_consumed)
  768. {
  769. size_t subbuf_size = buf->chan->subbuf_size;
  770. size_t n_subbufs = buf->chan->n_subbufs;
  771. size_t read_subbuf;
  772. if (buf->subbufs_produced == buf->subbufs_consumed &&
  773. buf->offset == buf->bytes_consumed)
  774. return;
  775. if (buf->bytes_consumed + bytes_consumed > subbuf_size) {
  776. relay_subbufs_consumed(buf->chan, buf->cpu, 1);
  777. buf->bytes_consumed = 0;
  778. }
  779. buf->bytes_consumed += bytes_consumed;
  780. if (!read_pos)
  781. read_subbuf = buf->subbufs_consumed % n_subbufs;
  782. else
  783. read_subbuf = read_pos / buf->chan->subbuf_size;
  784. if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) {
  785. if ((read_subbuf == buf->subbufs_produced % n_subbufs) &&
  786. (buf->offset == subbuf_size))
  787. return;
  788. relay_subbufs_consumed(buf->chan, buf->cpu, 1);
  789. buf->bytes_consumed = 0;
  790. }
  791. }
  792. /*
  793. * relay_file_read_avail - boolean, are there unconsumed bytes available?
  794. */
  795. static int relay_file_read_avail(struct rchan_buf *buf)
  796. {
  797. size_t subbuf_size = buf->chan->subbuf_size;
  798. size_t n_subbufs = buf->chan->n_subbufs;
  799. size_t produced = buf->subbufs_produced;
  800. size_t consumed;
  801. relay_file_read_consume(buf, 0, 0);
  802. consumed = buf->subbufs_consumed;
  803. if (unlikely(buf->offset > subbuf_size)) {
  804. if (produced == consumed)
  805. return 0;
  806. return 1;
  807. }
  808. if (unlikely(produced - consumed >= n_subbufs)) {
  809. consumed = produced - n_subbufs + 1;
  810. buf->subbufs_consumed = consumed;
  811. buf->bytes_consumed = 0;
  812. }
  813. produced = (produced % n_subbufs) * subbuf_size + buf->offset;
  814. consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed;
  815. if (consumed > produced)
  816. produced += n_subbufs * subbuf_size;
  817. if (consumed == produced) {
  818. if (buf->offset == subbuf_size &&
  819. buf->subbufs_produced > buf->subbufs_consumed)
  820. return 1;
  821. return 0;
  822. }
  823. return 1;
  824. }
  825. /**
  826. * relay_file_read_subbuf_avail - return bytes available in sub-buffer
  827. * @read_pos: file read position
  828. * @buf: relay channel buffer
  829. */
  830. static size_t relay_file_read_subbuf_avail(size_t read_pos,
  831. struct rchan_buf *buf)
  832. {
  833. size_t padding, avail = 0;
  834. size_t read_subbuf, read_offset, write_subbuf, write_offset;
  835. size_t subbuf_size = buf->chan->subbuf_size;
  836. write_subbuf = (buf->data - buf->start) / subbuf_size;
  837. write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset;
  838. read_subbuf = read_pos / subbuf_size;
  839. read_offset = read_pos % subbuf_size;
  840. padding = buf->padding[read_subbuf];
  841. if (read_subbuf == write_subbuf) {
  842. if (read_offset + padding < write_offset)
  843. avail = write_offset - (read_offset + padding);
  844. } else
  845. avail = (subbuf_size - padding) - read_offset;
  846. return avail;
  847. }
  848. /**
  849. * relay_file_read_start_pos - find the first available byte to read
  850. * @buf: relay channel buffer
  851. *
  852. * If the read_pos is in the middle of padding, return the
  853. * position of the first actually available byte, otherwise
  854. * return the original value.
  855. */
  856. static size_t relay_file_read_start_pos(struct rchan_buf *buf)
  857. {
  858. size_t read_subbuf, padding, padding_start, padding_end;
  859. size_t subbuf_size = buf->chan->subbuf_size;
  860. size_t n_subbufs = buf->chan->n_subbufs;
  861. size_t consumed = buf->subbufs_consumed % n_subbufs;
  862. size_t read_pos = (consumed * subbuf_size + buf->bytes_consumed)
  863. % (n_subbufs * subbuf_size);
  864. read_subbuf = read_pos / subbuf_size;
  865. padding = buf->padding[read_subbuf];
  866. padding_start = (read_subbuf + 1) * subbuf_size - padding;
  867. padding_end = (read_subbuf + 1) * subbuf_size;
  868. if (read_pos >= padding_start && read_pos < padding_end) {
  869. read_subbuf = (read_subbuf + 1) % n_subbufs;
  870. read_pos = read_subbuf * subbuf_size;
  871. }
  872. return read_pos;
  873. }
  874. /**
  875. * relay_file_read_end_pos - return the new read position
  876. * @read_pos: file read position
  877. * @buf: relay channel buffer
  878. * @count: number of bytes to be read
  879. */
  880. static size_t relay_file_read_end_pos(struct rchan_buf *buf,
  881. size_t read_pos,
  882. size_t count)
  883. {
  884. size_t read_subbuf, padding, end_pos;
  885. size_t subbuf_size = buf->chan->subbuf_size;
  886. size_t n_subbufs = buf->chan->n_subbufs;
  887. read_subbuf = read_pos / subbuf_size;
  888. padding = buf->padding[read_subbuf];
  889. if (read_pos % subbuf_size + count + padding == subbuf_size)
  890. end_pos = (read_subbuf + 1) * subbuf_size;
  891. else
  892. end_pos = read_pos + count;
  893. if (end_pos >= subbuf_size * n_subbufs)
  894. end_pos = 0;
  895. return end_pos;
  896. }
  897. static ssize_t relay_file_read(struct file *filp,
  898. char __user *buffer,
  899. size_t count,
  900. loff_t *ppos)
  901. {
  902. struct rchan_buf *buf = filp->private_data;
  903. size_t read_start, avail;
  904. size_t written = 0;
  905. int ret;
  906. if (!count)
  907. return 0;
  908. inode_lock(file_inode(filp));
  909. do {
  910. void *from;
  911. if (!relay_file_read_avail(buf))
  912. break;
  913. read_start = relay_file_read_start_pos(buf);
  914. avail = relay_file_read_subbuf_avail(read_start, buf);
  915. if (!avail)
  916. break;
  917. avail = min(count, avail);
  918. from = buf->start + read_start;
  919. ret = avail;
  920. if (copy_to_user(buffer, from, avail))
  921. break;
  922. buffer += ret;
  923. written += ret;
  924. count -= ret;
  925. relay_file_read_consume(buf, read_start, ret);
  926. *ppos = relay_file_read_end_pos(buf, read_start, ret);
  927. } while (count);
  928. inode_unlock(file_inode(filp));
  929. return written;
  930. }
  931. static void relay_consume_bytes(struct rchan_buf *rbuf, int bytes_consumed)
  932. {
  933. rbuf->bytes_consumed += bytes_consumed;
  934. if (rbuf->bytes_consumed >= rbuf->chan->subbuf_size) {
  935. relay_subbufs_consumed(rbuf->chan, rbuf->cpu, 1);
  936. rbuf->bytes_consumed %= rbuf->chan->subbuf_size;
  937. }
  938. }
  939. static void relay_pipe_buf_release(struct pipe_inode_info *pipe,
  940. struct pipe_buffer *buf)
  941. {
  942. struct rchan_buf *rbuf;
  943. rbuf = (struct rchan_buf *)page_private(buf->page);
  944. relay_consume_bytes(rbuf, buf->private);
  945. }
  946. static const struct pipe_buf_operations relay_pipe_buf_ops = {
  947. .release = relay_pipe_buf_release,
  948. .try_steal = generic_pipe_buf_try_steal,
  949. .get = generic_pipe_buf_get,
  950. };
  951. static void relay_page_release(struct splice_pipe_desc *spd, unsigned int i)
  952. {
  953. }
  954. /*
  955. * subbuf_splice_actor - splice up to one subbuf's worth of data
  956. */
  957. static ssize_t subbuf_splice_actor(struct file *in,
  958. loff_t *ppos,
  959. struct pipe_inode_info *pipe,
  960. size_t len,
  961. unsigned int flags,
  962. int *nonpad_ret)
  963. {
  964. unsigned int pidx, poff, total_len, subbuf_pages, nr_pages;
  965. struct rchan_buf *rbuf = in->private_data;
  966. unsigned int subbuf_size = rbuf->chan->subbuf_size;
  967. uint64_t pos = (uint64_t) *ppos;
  968. uint32_t alloc_size = (uint32_t) rbuf->chan->alloc_size;
  969. size_t read_start = (size_t) do_div(pos, alloc_size);
  970. size_t read_subbuf = read_start / subbuf_size;
  971. size_t padding = rbuf->padding[read_subbuf];
  972. size_t nonpad_end = read_subbuf * subbuf_size + subbuf_size - padding;
  973. struct page *pages[PIPE_DEF_BUFFERS];
  974. struct partial_page partial[PIPE_DEF_BUFFERS];
  975. struct splice_pipe_desc spd = {
  976. .pages = pages,
  977. .nr_pages = 0,
  978. .nr_pages_max = PIPE_DEF_BUFFERS,
  979. .partial = partial,
  980. .ops = &relay_pipe_buf_ops,
  981. .spd_release = relay_page_release,
  982. };
  983. ssize_t ret;
  984. if (rbuf->subbufs_produced == rbuf->subbufs_consumed)
  985. return 0;
  986. if (splice_grow_spd(pipe, &spd))
  987. return -ENOMEM;
  988. /*
  989. * Adjust read len, if longer than what is available
  990. */
  991. if (len > (subbuf_size - read_start % subbuf_size))
  992. len = subbuf_size - read_start % subbuf_size;
  993. subbuf_pages = rbuf->chan->alloc_size >> PAGE_SHIFT;
  994. pidx = (read_start / PAGE_SIZE) % subbuf_pages;
  995. poff = read_start & ~PAGE_MASK;
  996. nr_pages = min_t(unsigned int, subbuf_pages, spd.nr_pages_max);
  997. for (total_len = 0; spd.nr_pages < nr_pages; spd.nr_pages++) {
  998. unsigned int this_len, this_end, private;
  999. unsigned int cur_pos = read_start + total_len;
  1000. if (!len)
  1001. break;
  1002. this_len = min_t(unsigned long, len, PAGE_SIZE - poff);
  1003. private = this_len;
  1004. spd.pages[spd.nr_pages] = rbuf->page_array[pidx];
  1005. spd.partial[spd.nr_pages].offset = poff;
  1006. this_end = cur_pos + this_len;
  1007. if (this_end >= nonpad_end) {
  1008. this_len = nonpad_end - cur_pos;
  1009. private = this_len + padding;
  1010. }
  1011. spd.partial[spd.nr_pages].len = this_len;
  1012. spd.partial[spd.nr_pages].private = private;
  1013. len -= this_len;
  1014. total_len += this_len;
  1015. poff = 0;
  1016. pidx = (pidx + 1) % subbuf_pages;
  1017. if (this_end >= nonpad_end) {
  1018. spd.nr_pages++;
  1019. break;
  1020. }
  1021. }
  1022. ret = 0;
  1023. if (!spd.nr_pages)
  1024. goto out;
  1025. ret = *nonpad_ret = splice_to_pipe(pipe, &spd);
  1026. if (ret < 0 || ret < total_len)
  1027. goto out;
  1028. if (read_start + ret == nonpad_end)
  1029. ret += padding;
  1030. out:
  1031. splice_shrink_spd(&spd);
  1032. return ret;
  1033. }
  1034. static ssize_t relay_file_splice_read(struct file *in,
  1035. loff_t *ppos,
  1036. struct pipe_inode_info *pipe,
  1037. size_t len,
  1038. unsigned int flags)
  1039. {
  1040. ssize_t spliced;
  1041. int ret;
  1042. int nonpad_ret = 0;
  1043. ret = 0;
  1044. spliced = 0;
  1045. while (len && !spliced) {
  1046. ret = subbuf_splice_actor(in, ppos, pipe, len, flags, &nonpad_ret);
  1047. if (ret < 0)
  1048. break;
  1049. else if (!ret) {
  1050. if (flags & SPLICE_F_NONBLOCK)
  1051. ret = -EAGAIN;
  1052. break;
  1053. }
  1054. *ppos += ret;
  1055. if (ret > len)
  1056. len = 0;
  1057. else
  1058. len -= ret;
  1059. spliced += nonpad_ret;
  1060. nonpad_ret = 0;
  1061. }
  1062. if (spliced)
  1063. return spliced;
  1064. return ret;
  1065. }
  1066. const struct file_operations relay_file_operations = {
  1067. .open = relay_file_open,
  1068. .poll = relay_file_poll,
  1069. .mmap = relay_file_mmap,
  1070. .read = relay_file_read,
  1071. .llseek = no_llseek,
  1072. .release = relay_file_release,
  1073. .splice_read = relay_file_splice_read,
  1074. };
  1075. EXPORT_SYMBOL_GPL(relay_file_operations);