internal.h 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _KERNEL_EVENTS_INTERNAL_H
  3. #define _KERNEL_EVENTS_INTERNAL_H
  4. #include <linux/hardirq.h>
  5. #include <linux/uaccess.h>
  6. #include <linux/refcount.h>
  7. /* Buffer handling */
  8. #define RING_BUFFER_WRITABLE 0x01
  9. struct perf_buffer {
  10. refcount_t refcount;
  11. struct rcu_head rcu_head;
  12. #ifdef CONFIG_PERF_USE_VMALLOC
  13. struct work_struct work;
  14. int page_order; /* allocation order */
  15. #endif
  16. int nr_pages; /* nr of data pages */
  17. int overwrite; /* can overwrite itself */
  18. int paused; /* can write into ring buffer */
  19. atomic_t poll; /* POLL_ for wakeups */
  20. local_t head; /* write position */
  21. unsigned int nest; /* nested writers */
  22. local_t events; /* event limit */
  23. local_t wakeup; /* wakeup stamp */
  24. local_t lost; /* nr records lost */
  25. long watermark; /* wakeup watermark */
  26. long aux_watermark;
  27. /* poll crap */
  28. spinlock_t event_lock;
  29. struct list_head event_list;
  30. atomic_t mmap_count;
  31. unsigned long mmap_locked;
  32. struct user_struct *mmap_user;
  33. /* AUX area */
  34. long aux_head;
  35. unsigned int aux_nest;
  36. long aux_wakeup; /* last aux_watermark boundary crossed by aux_head */
  37. unsigned long aux_pgoff;
  38. int aux_nr_pages;
  39. int aux_overwrite;
  40. atomic_t aux_mmap_count;
  41. unsigned long aux_mmap_locked;
  42. void (*free_aux)(void *);
  43. refcount_t aux_refcount;
  44. int aux_in_sampling;
  45. void **aux_pages;
  46. void *aux_priv;
  47. struct perf_event_mmap_page *user_page;
  48. void *data_pages[];
  49. };
  50. extern void rb_free(struct perf_buffer *rb);
  51. static inline void rb_free_rcu(struct rcu_head *rcu_head)
  52. {
  53. struct perf_buffer *rb;
  54. rb = container_of(rcu_head, struct perf_buffer, rcu_head);
  55. rb_free(rb);
  56. }
  57. static inline void rb_toggle_paused(struct perf_buffer *rb, bool pause)
  58. {
  59. if (!pause && rb->nr_pages)
  60. rb->paused = 0;
  61. else
  62. rb->paused = 1;
  63. }
  64. extern struct perf_buffer *
  65. rb_alloc(int nr_pages, long watermark, int cpu, int flags);
  66. extern void perf_event_wakeup(struct perf_event *event);
  67. extern int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event,
  68. pgoff_t pgoff, int nr_pages, long watermark, int flags);
  69. extern void rb_free_aux(struct perf_buffer *rb);
  70. extern struct perf_buffer *ring_buffer_get(struct perf_event *event);
  71. extern void ring_buffer_put(struct perf_buffer *rb);
  72. static inline bool rb_has_aux(struct perf_buffer *rb)
  73. {
  74. return !!rb->aux_nr_pages;
  75. }
  76. void perf_event_aux_event(struct perf_event *event, unsigned long head,
  77. unsigned long size, u64 flags);
  78. extern struct page *
  79. perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff);
  80. #ifdef CONFIG_PERF_USE_VMALLOC
  81. /*
  82. * Back perf_mmap() with vmalloc memory.
  83. *
  84. * Required for architectures that have d-cache aliasing issues.
  85. */
  86. static inline int page_order(struct perf_buffer *rb)
  87. {
  88. return rb->page_order;
  89. }
  90. #else
  91. static inline int page_order(struct perf_buffer *rb)
  92. {
  93. return 0;
  94. }
  95. #endif
  96. static inline int data_page_nr(struct perf_buffer *rb)
  97. {
  98. return rb->nr_pages << page_order(rb);
  99. }
  100. static inline unsigned long perf_data_size(struct perf_buffer *rb)
  101. {
  102. return rb->nr_pages << (PAGE_SHIFT + page_order(rb));
  103. }
  104. static inline unsigned long perf_aux_size(struct perf_buffer *rb)
  105. {
  106. return rb->aux_nr_pages << PAGE_SHIFT;
  107. }
  108. #define __DEFINE_OUTPUT_COPY_BODY(advance_buf, memcpy_func, ...) \
  109. { \
  110. unsigned long size, written; \
  111. \
  112. do { \
  113. size = min(handle->size, len); \
  114. written = memcpy_func(__VA_ARGS__); \
  115. written = size - written; \
  116. \
  117. len -= written; \
  118. handle->addr += written; \
  119. if (advance_buf) \
  120. buf += written; \
  121. handle->size -= written; \
  122. if (!handle->size) { \
  123. struct perf_buffer *rb = handle->rb; \
  124. \
  125. handle->page++; \
  126. handle->page &= rb->nr_pages - 1; \
  127. handle->addr = rb->data_pages[handle->page]; \
  128. handle->size = PAGE_SIZE << page_order(rb); \
  129. } \
  130. } while (len && written == size); \
  131. \
  132. return len; \
  133. }
  134. #define DEFINE_OUTPUT_COPY(func_name, memcpy_func) \
  135. static inline unsigned long \
  136. func_name(struct perf_output_handle *handle, \
  137. const void *buf, unsigned long len) \
  138. __DEFINE_OUTPUT_COPY_BODY(true, memcpy_func, handle->addr, buf, size)
  139. static inline unsigned long
  140. __output_custom(struct perf_output_handle *handle, perf_copy_f copy_func,
  141. const void *buf, unsigned long len)
  142. {
  143. unsigned long orig_len = len;
  144. __DEFINE_OUTPUT_COPY_BODY(false, copy_func, handle->addr, buf,
  145. orig_len - len, size)
  146. }
  147. static inline unsigned long
  148. memcpy_common(void *dst, const void *src, unsigned long n)
  149. {
  150. memcpy(dst, src, n);
  151. return 0;
  152. }
  153. DEFINE_OUTPUT_COPY(__output_copy, memcpy_common)
  154. static inline unsigned long
  155. memcpy_skip(void *dst, const void *src, unsigned long n)
  156. {
  157. return 0;
  158. }
  159. DEFINE_OUTPUT_COPY(__output_skip, memcpy_skip)
  160. #ifndef arch_perf_out_copy_user
  161. #define arch_perf_out_copy_user arch_perf_out_copy_user
  162. static inline unsigned long
  163. arch_perf_out_copy_user(void *dst, const void *src, unsigned long n)
  164. {
  165. unsigned long ret;
  166. pagefault_disable();
  167. ret = __copy_from_user_inatomic(dst, src, n);
  168. pagefault_enable();
  169. return ret;
  170. }
  171. #endif
  172. DEFINE_OUTPUT_COPY(__output_copy_user, arch_perf_out_copy_user)
  173. static inline int get_recursion_context(int *recursion)
  174. {
  175. unsigned char rctx = interrupt_context_level();
  176. if (recursion[rctx])
  177. return -1;
  178. recursion[rctx]++;
  179. barrier();
  180. return rctx;
  181. }
  182. static inline void put_recursion_context(int *recursion, int rctx)
  183. {
  184. barrier();
  185. recursion[rctx]--;
  186. }
  187. #ifdef CONFIG_HAVE_PERF_USER_STACK_DUMP
  188. static inline bool arch_perf_have_user_stack_dump(void)
  189. {
  190. return true;
  191. }
  192. #define perf_user_stack_pointer(regs) user_stack_pointer(regs)
  193. #else
  194. static inline bool arch_perf_have_user_stack_dump(void)
  195. {
  196. return false;
  197. }
  198. #define perf_user_stack_pointer(regs) 0
  199. #endif /* CONFIG_HAVE_PERF_USER_STACK_DUMP */
  200. #endif /* _KERNEL_EVENTS_INTERNAL_H */