uprobes.c 57 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * User-space Probes (UProbes)
  4. *
  5. * Copyright (C) IBM Corporation, 2008-2012
  6. * Authors:
  7. * Srikar Dronamraju
  8. * Jim Keniston
  9. * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
  10. */
  11. #include <linux/kernel.h>
  12. #include <linux/highmem.h>
  13. #include <linux/pagemap.h> /* read_mapping_page */
  14. #include <linux/slab.h>
  15. #include <linux/sched.h>
  16. #include <linux/sched/mm.h>
  17. #include <linux/sched/coredump.h>
  18. #include <linux/export.h>
  19. #include <linux/rmap.h> /* anon_vma_prepare */
  20. #include <linux/mmu_notifier.h> /* set_pte_at_notify */
  21. #include <linux/swap.h> /* folio_free_swap */
  22. #include <linux/ptrace.h> /* user_enable_single_step */
  23. #include <linux/kdebug.h> /* notifier mechanism */
  24. #include "../../mm/internal.h" /* munlock_vma_page */
  25. #include <linux/percpu-rwsem.h>
  26. #include <linux/task_work.h>
  27. #include <linux/shmem_fs.h>
  28. #include <linux/khugepaged.h>
  29. #include <linux/uprobes.h>
  30. #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
  31. #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE
  32. static struct rb_root uprobes_tree = RB_ROOT;
  33. /*
  34. * allows us to skip the uprobe_mmap if there are no uprobe events active
  35. * at this time. Probably a fine grained per inode count is better?
  36. */
  37. #define no_uprobe_events() RB_EMPTY_ROOT(&uprobes_tree)
  38. static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */
  39. #define UPROBES_HASH_SZ 13
  40. /* serialize uprobe->pending_list */
  41. static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
  42. #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
  43. DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
  44. /* Have a copy of original instruction */
  45. #define UPROBE_COPY_INSN 0
  46. struct uprobe {
  47. struct rb_node rb_node; /* node in the rb tree */
  48. refcount_t ref;
  49. struct rw_semaphore register_rwsem;
  50. struct rw_semaphore consumer_rwsem;
  51. struct list_head pending_list;
  52. struct uprobe_consumer *consumers;
  53. struct inode *inode; /* Also hold a ref to inode */
  54. loff_t offset;
  55. loff_t ref_ctr_offset;
  56. unsigned long flags;
  57. /*
  58. * The generic code assumes that it has two members of unknown type
  59. * owned by the arch-specific code:
  60. *
  61. * insn - copy_insn() saves the original instruction here for
  62. * arch_uprobe_analyze_insn().
  63. *
  64. * ixol - potentially modified instruction to execute out of
  65. * line, copied to xol_area by xol_get_insn_slot().
  66. */
  67. struct arch_uprobe arch;
  68. };
  69. struct delayed_uprobe {
  70. struct list_head list;
  71. struct uprobe *uprobe;
  72. struct mm_struct *mm;
  73. };
  74. static DEFINE_MUTEX(delayed_uprobe_lock);
  75. static LIST_HEAD(delayed_uprobe_list);
  76. /*
  77. * Execute out of line area: anonymous executable mapping installed
  78. * by the probed task to execute the copy of the original instruction
  79. * mangled by set_swbp().
  80. *
  81. * On a breakpoint hit, thread contests for a slot. It frees the
  82. * slot after singlestep. Currently a fixed number of slots are
  83. * allocated.
  84. */
  85. struct xol_area {
  86. wait_queue_head_t wq; /* if all slots are busy */
  87. atomic_t slot_count; /* number of in-use slots */
  88. unsigned long *bitmap; /* 0 = free slot */
  89. struct vm_special_mapping xol_mapping;
  90. struct page *pages[2];
  91. /*
  92. * We keep the vma's vm_start rather than a pointer to the vma
  93. * itself. The probed process or a naughty kernel module could make
  94. * the vma go away, and we must handle that reasonably gracefully.
  95. */
  96. unsigned long vaddr; /* Page(s) of instruction slots */
  97. };
  98. /*
  99. * valid_vma: Verify if the specified vma is an executable vma
  100. * Relax restrictions while unregistering: vm_flags might have
  101. * changed after breakpoint was inserted.
  102. * - is_register: indicates if we are in register context.
  103. * - Return 1 if the specified virtual address is in an
  104. * executable vma.
  105. */
  106. static bool valid_vma(struct vm_area_struct *vma, bool is_register)
  107. {
  108. vm_flags_t flags = VM_HUGETLB | VM_MAYEXEC | VM_MAYSHARE;
  109. if (is_register)
  110. flags |= VM_WRITE;
  111. return vma->vm_file && (vma->vm_flags & flags) == VM_MAYEXEC;
  112. }
  113. static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
  114. {
  115. return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
  116. }
  117. static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
  118. {
  119. return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
  120. }
  121. /**
  122. * __replace_page - replace page in vma by new page.
  123. * based on replace_page in mm/ksm.c
  124. *
  125. * @vma: vma that holds the pte pointing to page
  126. * @addr: address the old @page is mapped at
  127. * @old_page: the page we are replacing by new_page
  128. * @new_page: the modified page we replace page by
  129. *
  130. * If @new_page is NULL, only unmap @old_page.
  131. *
  132. * Returns 0 on success, negative error code otherwise.
  133. */
  134. static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
  135. struct page *old_page, struct page *new_page)
  136. {
  137. struct folio *old_folio = page_folio(old_page);
  138. struct folio *new_folio;
  139. struct mm_struct *mm = vma->vm_mm;
  140. DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, vma, addr, 0);
  141. int err;
  142. struct mmu_notifier_range range;
  143. mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr,
  144. addr + PAGE_SIZE);
  145. if (new_page) {
  146. new_folio = page_folio(new_page);
  147. err = mem_cgroup_charge(new_folio, vma->vm_mm, GFP_KERNEL);
  148. if (err)
  149. return err;
  150. }
  151. /* For folio_free_swap() below */
  152. folio_lock(old_folio);
  153. mmu_notifier_invalidate_range_start(&range);
  154. err = -EAGAIN;
  155. if (!page_vma_mapped_walk(&pvmw))
  156. goto unlock;
  157. VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
  158. if (new_page) {
  159. folio_get(new_folio);
  160. page_add_new_anon_rmap(new_page, vma, addr);
  161. folio_add_lru_vma(new_folio, vma);
  162. } else
  163. /* no new page, just dec_mm_counter for old_page */
  164. dec_mm_counter(mm, MM_ANONPAGES);
  165. if (!folio_test_anon(old_folio)) {
  166. dec_mm_counter(mm, mm_counter_file(old_page));
  167. inc_mm_counter(mm, MM_ANONPAGES);
  168. }
  169. flush_cache_page(vma, addr, pte_pfn(*pvmw.pte));
  170. ptep_clear_flush_notify(vma, addr, pvmw.pte);
  171. if (new_page)
  172. set_pte_at_notify(mm, addr, pvmw.pte,
  173. mk_pte(new_page, vma->vm_page_prot));
  174. page_remove_rmap(old_page, vma, false);
  175. if (!folio_mapped(old_folio))
  176. folio_free_swap(old_folio);
  177. page_vma_mapped_walk_done(&pvmw);
  178. folio_put(old_folio);
  179. err = 0;
  180. unlock:
  181. mmu_notifier_invalidate_range_end(&range);
  182. folio_unlock(old_folio);
  183. return err;
  184. }
  185. /**
  186. * is_swbp_insn - check if instruction is breakpoint instruction.
  187. * @insn: instruction to be checked.
  188. * Default implementation of is_swbp_insn
  189. * Returns true if @insn is a breakpoint instruction.
  190. */
  191. bool __weak is_swbp_insn(uprobe_opcode_t *insn)
  192. {
  193. return *insn == UPROBE_SWBP_INSN;
  194. }
  195. /**
  196. * is_trap_insn - check if instruction is breakpoint instruction.
  197. * @insn: instruction to be checked.
  198. * Default implementation of is_trap_insn
  199. * Returns true if @insn is a breakpoint instruction.
  200. *
  201. * This function is needed for the case where an architecture has multiple
  202. * trap instructions (like powerpc).
  203. */
  204. bool __weak is_trap_insn(uprobe_opcode_t *insn)
  205. {
  206. return is_swbp_insn(insn);
  207. }
  208. static void copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len)
  209. {
  210. void *kaddr = kmap_atomic(page);
  211. memcpy(dst, kaddr + (vaddr & ~PAGE_MASK), len);
  212. kunmap_atomic(kaddr);
  213. }
  214. static void copy_to_page(struct page *page, unsigned long vaddr, const void *src, int len)
  215. {
  216. void *kaddr = kmap_atomic(page);
  217. memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len);
  218. kunmap_atomic(kaddr);
  219. }
  220. static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode)
  221. {
  222. uprobe_opcode_t old_opcode;
  223. bool is_swbp;
  224. /*
  225. * Note: We only check if the old_opcode is UPROBE_SWBP_INSN here.
  226. * We do not check if it is any other 'trap variant' which could
  227. * be conditional trap instruction such as the one powerpc supports.
  228. *
  229. * The logic is that we do not care if the underlying instruction
  230. * is a trap variant; uprobes always wins over any other (gdb)
  231. * breakpoint.
  232. */
  233. copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE);
  234. is_swbp = is_swbp_insn(&old_opcode);
  235. if (is_swbp_insn(new_opcode)) {
  236. if (is_swbp) /* register: already installed? */
  237. return 0;
  238. } else {
  239. if (!is_swbp) /* unregister: was it changed by us? */
  240. return 0;
  241. }
  242. return 1;
  243. }
  244. static struct delayed_uprobe *
  245. delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm)
  246. {
  247. struct delayed_uprobe *du;
  248. list_for_each_entry(du, &delayed_uprobe_list, list)
  249. if (du->uprobe == uprobe && du->mm == mm)
  250. return du;
  251. return NULL;
  252. }
  253. static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm)
  254. {
  255. struct delayed_uprobe *du;
  256. if (delayed_uprobe_check(uprobe, mm))
  257. return 0;
  258. du = kzalloc(sizeof(*du), GFP_KERNEL);
  259. if (!du)
  260. return -ENOMEM;
  261. du->uprobe = uprobe;
  262. du->mm = mm;
  263. list_add(&du->list, &delayed_uprobe_list);
  264. return 0;
  265. }
  266. static void delayed_uprobe_delete(struct delayed_uprobe *du)
  267. {
  268. if (WARN_ON(!du))
  269. return;
  270. list_del(&du->list);
  271. kfree(du);
  272. }
  273. static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm)
  274. {
  275. struct list_head *pos, *q;
  276. struct delayed_uprobe *du;
  277. if (!uprobe && !mm)
  278. return;
  279. list_for_each_safe(pos, q, &delayed_uprobe_list) {
  280. du = list_entry(pos, struct delayed_uprobe, list);
  281. if (uprobe && du->uprobe != uprobe)
  282. continue;
  283. if (mm && du->mm != mm)
  284. continue;
  285. delayed_uprobe_delete(du);
  286. }
  287. }
  288. static bool valid_ref_ctr_vma(struct uprobe *uprobe,
  289. struct vm_area_struct *vma)
  290. {
  291. unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset);
  292. return uprobe->ref_ctr_offset &&
  293. vma->vm_file &&
  294. file_inode(vma->vm_file) == uprobe->inode &&
  295. (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
  296. vma->vm_start <= vaddr &&
  297. vma->vm_end > vaddr;
  298. }
  299. static struct vm_area_struct *
  300. find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
  301. {
  302. VMA_ITERATOR(vmi, mm, 0);
  303. struct vm_area_struct *tmp;
  304. for_each_vma(vmi, tmp)
  305. if (valid_ref_ctr_vma(uprobe, tmp))
  306. return tmp;
  307. return NULL;
  308. }
  309. static int
  310. __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
  311. {
  312. void *kaddr;
  313. struct page *page;
  314. struct vm_area_struct *vma;
  315. int ret;
  316. short *ptr;
  317. if (!vaddr || !d)
  318. return -EINVAL;
  319. ret = get_user_pages_remote(mm, vaddr, 1,
  320. FOLL_WRITE, &page, &vma, NULL);
  321. if (unlikely(ret <= 0)) {
  322. /*
  323. * We are asking for 1 page. If get_user_pages_remote() fails,
  324. * it may return 0, in that case we have to return error.
  325. */
  326. return ret == 0 ? -EBUSY : ret;
  327. }
  328. kaddr = kmap_atomic(page);
  329. ptr = kaddr + (vaddr & ~PAGE_MASK);
  330. if (unlikely(*ptr + d < 0)) {
  331. pr_warn("ref_ctr going negative. vaddr: 0x%lx, "
  332. "curr val: %d, delta: %d\n", vaddr, *ptr, d);
  333. ret = -EINVAL;
  334. goto out;
  335. }
  336. *ptr += d;
  337. ret = 0;
  338. out:
  339. kunmap_atomic(kaddr);
  340. put_page(page);
  341. return ret;
  342. }
  343. static void update_ref_ctr_warn(struct uprobe *uprobe,
  344. struct mm_struct *mm, short d)
  345. {
  346. pr_warn("ref_ctr %s failed for inode: 0x%lx offset: "
  347. "0x%llx ref_ctr_offset: 0x%llx of mm: 0x%pK\n",
  348. d > 0 ? "increment" : "decrement", uprobe->inode->i_ino,
  349. (unsigned long long) uprobe->offset,
  350. (unsigned long long) uprobe->ref_ctr_offset, mm);
  351. }
  352. static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm,
  353. short d)
  354. {
  355. struct vm_area_struct *rc_vma;
  356. unsigned long rc_vaddr;
  357. int ret = 0;
  358. rc_vma = find_ref_ctr_vma(uprobe, mm);
  359. if (rc_vma) {
  360. rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset);
  361. ret = __update_ref_ctr(mm, rc_vaddr, d);
  362. if (ret)
  363. update_ref_ctr_warn(uprobe, mm, d);
  364. if (d > 0)
  365. return ret;
  366. }
  367. mutex_lock(&delayed_uprobe_lock);
  368. if (d > 0)
  369. ret = delayed_uprobe_add(uprobe, mm);
  370. else
  371. delayed_uprobe_remove(uprobe, mm);
  372. mutex_unlock(&delayed_uprobe_lock);
  373. return ret;
  374. }
  375. /*
  376. * NOTE:
  377. * Expect the breakpoint instruction to be the smallest size instruction for
  378. * the architecture. If an arch has variable length instruction and the
  379. * breakpoint instruction is not of the smallest length instruction
  380. * supported by that architecture then we need to modify is_trap_at_addr and
  381. * uprobe_write_opcode accordingly. This would never be a problem for archs
  382. * that have fixed length instructions.
  383. *
  384. * uprobe_write_opcode - write the opcode at a given virtual address.
  385. * @auprobe: arch specific probepoint information.
  386. * @mm: the probed process address space.
  387. * @vaddr: the virtual address to store the opcode.
  388. * @opcode: opcode to be written at @vaddr.
  389. *
  390. * Called with mm->mmap_lock held for write.
  391. * Return 0 (success) or a negative errno.
  392. */
  393. int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
  394. unsigned long vaddr, uprobe_opcode_t opcode)
  395. {
  396. struct uprobe *uprobe;
  397. struct page *old_page, *new_page;
  398. struct vm_area_struct *vma;
  399. int ret, is_register, ref_ctr_updated = 0;
  400. bool orig_page_huge = false;
  401. unsigned int gup_flags = FOLL_FORCE;
  402. is_register = is_swbp_insn(&opcode);
  403. uprobe = container_of(auprobe, struct uprobe, arch);
  404. retry:
  405. if (is_register)
  406. gup_flags |= FOLL_SPLIT_PMD;
  407. /* Read the page with vaddr into memory */
  408. ret = get_user_pages_remote(mm, vaddr, 1, gup_flags,
  409. &old_page, &vma, NULL);
  410. if (ret <= 0)
  411. return ret;
  412. ret = verify_opcode(old_page, vaddr, &opcode);
  413. if (ret <= 0)
  414. goto put_old;
  415. if (WARN(!is_register && PageCompound(old_page),
  416. "uprobe unregister should never work on compound page\n")) {
  417. ret = -EINVAL;
  418. goto put_old;
  419. }
  420. /* We are going to replace instruction, update ref_ctr. */
  421. if (!ref_ctr_updated && uprobe->ref_ctr_offset) {
  422. ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);
  423. if (ret)
  424. goto put_old;
  425. ref_ctr_updated = 1;
  426. }
  427. ret = 0;
  428. if (!is_register && !PageAnon(old_page))
  429. goto put_old;
  430. ret = anon_vma_prepare(vma);
  431. if (ret)
  432. goto put_old;
  433. ret = -ENOMEM;
  434. new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
  435. if (!new_page)
  436. goto put_old;
  437. __SetPageUptodate(new_page);
  438. copy_highpage(new_page, old_page);
  439. copy_to_page(new_page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
  440. if (!is_register) {
  441. struct page *orig_page;
  442. pgoff_t index;
  443. VM_BUG_ON_PAGE(!PageAnon(old_page), old_page);
  444. index = vaddr_to_offset(vma, vaddr & PAGE_MASK) >> PAGE_SHIFT;
  445. orig_page = find_get_page(vma->vm_file->f_inode->i_mapping,
  446. index);
  447. if (orig_page) {
  448. if (PageUptodate(orig_page) &&
  449. pages_identical(new_page, orig_page)) {
  450. /* let go new_page */
  451. put_page(new_page);
  452. new_page = NULL;
  453. if (PageCompound(orig_page))
  454. orig_page_huge = true;
  455. }
  456. put_page(orig_page);
  457. }
  458. }
  459. ret = __replace_page(vma, vaddr, old_page, new_page);
  460. if (new_page)
  461. put_page(new_page);
  462. put_old:
  463. put_page(old_page);
  464. if (unlikely(ret == -EAGAIN))
  465. goto retry;
  466. /* Revert back reference counter if instruction update failed. */
  467. if (ret && is_register && ref_ctr_updated)
  468. update_ref_ctr(uprobe, mm, -1);
  469. /* try collapse pmd for compound page */
  470. if (!ret && orig_page_huge)
  471. collapse_pte_mapped_thp(mm, vaddr, false);
  472. return ret;
  473. }
  474. /**
  475. * set_swbp - store breakpoint at a given address.
  476. * @auprobe: arch specific probepoint information.
  477. * @mm: the probed process address space.
  478. * @vaddr: the virtual address to insert the opcode.
  479. *
  480. * For mm @mm, store the breakpoint instruction at @vaddr.
  481. * Return 0 (success) or a negative errno.
  482. */
  483. int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
  484. {
  485. return uprobe_write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN);
  486. }
  487. /**
  488. * set_orig_insn - Restore the original instruction.
  489. * @mm: the probed process address space.
  490. * @auprobe: arch specific probepoint information.
  491. * @vaddr: the virtual address to insert the opcode.
  492. *
  493. * For mm @mm, restore the original opcode (opcode) at @vaddr.
  494. * Return 0 (success) or a negative errno.
  495. */
  496. int __weak
  497. set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
  498. {
  499. return uprobe_write_opcode(auprobe, mm, vaddr,
  500. *(uprobe_opcode_t *)&auprobe->insn);
  501. }
  502. static struct uprobe *get_uprobe(struct uprobe *uprobe)
  503. {
  504. refcount_inc(&uprobe->ref);
  505. return uprobe;
  506. }
  507. static void put_uprobe(struct uprobe *uprobe)
  508. {
  509. if (refcount_dec_and_test(&uprobe->ref)) {
  510. /*
  511. * If application munmap(exec_vma) before uprobe_unregister()
  512. * gets called, we don't get a chance to remove uprobe from
  513. * delayed_uprobe_list from remove_breakpoint(). Do it here.
  514. */
  515. mutex_lock(&delayed_uprobe_lock);
  516. delayed_uprobe_remove(uprobe, NULL);
  517. mutex_unlock(&delayed_uprobe_lock);
  518. kfree(uprobe);
  519. }
  520. }
  521. static __always_inline
  522. int uprobe_cmp(const struct inode *l_inode, const loff_t l_offset,
  523. const struct uprobe *r)
  524. {
  525. if (l_inode < r->inode)
  526. return -1;
  527. if (l_inode > r->inode)
  528. return 1;
  529. if (l_offset < r->offset)
  530. return -1;
  531. if (l_offset > r->offset)
  532. return 1;
  533. return 0;
  534. }
  535. #define __node_2_uprobe(node) \
  536. rb_entry((node), struct uprobe, rb_node)
  537. struct __uprobe_key {
  538. struct inode *inode;
  539. loff_t offset;
  540. };
  541. static inline int __uprobe_cmp_key(const void *key, const struct rb_node *b)
  542. {
  543. const struct __uprobe_key *a = key;
  544. return uprobe_cmp(a->inode, a->offset, __node_2_uprobe(b));
  545. }
  546. static inline int __uprobe_cmp(struct rb_node *a, const struct rb_node *b)
  547. {
  548. struct uprobe *u = __node_2_uprobe(a);
  549. return uprobe_cmp(u->inode, u->offset, __node_2_uprobe(b));
  550. }
  551. static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)
  552. {
  553. struct __uprobe_key key = {
  554. .inode = inode,
  555. .offset = offset,
  556. };
  557. struct rb_node *node = rb_find(&key, &uprobes_tree, __uprobe_cmp_key);
  558. if (node)
  559. return get_uprobe(__node_2_uprobe(node));
  560. return NULL;
  561. }
  562. /*
  563. * Find a uprobe corresponding to a given inode:offset
  564. * Acquires uprobes_treelock
  565. */
  566. static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
  567. {
  568. struct uprobe *uprobe;
  569. spin_lock(&uprobes_treelock);
  570. uprobe = __find_uprobe(inode, offset);
  571. spin_unlock(&uprobes_treelock);
  572. return uprobe;
  573. }
  574. static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
  575. {
  576. struct rb_node *node;
  577. node = rb_find_add(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp);
  578. if (node)
  579. return get_uprobe(__node_2_uprobe(node));
  580. /* get access + creation ref */
  581. refcount_set(&uprobe->ref, 2);
  582. return NULL;
  583. }
  584. /*
  585. * Acquire uprobes_treelock.
  586. * Matching uprobe already exists in rbtree;
  587. * increment (access refcount) and return the matching uprobe.
  588. *
  589. * No matching uprobe; insert the uprobe in rb_tree;
  590. * get a double refcount (access + creation) and return NULL.
  591. */
  592. static struct uprobe *insert_uprobe(struct uprobe *uprobe)
  593. {
  594. struct uprobe *u;
  595. spin_lock(&uprobes_treelock);
  596. u = __insert_uprobe(uprobe);
  597. spin_unlock(&uprobes_treelock);
  598. return u;
  599. }
  600. static void
  601. ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
  602. {
  603. pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx "
  604. "ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n",
  605. uprobe->inode->i_ino, (unsigned long long) uprobe->offset,
  606. (unsigned long long) cur_uprobe->ref_ctr_offset,
  607. (unsigned long long) uprobe->ref_ctr_offset);
  608. }
  609. static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset,
  610. loff_t ref_ctr_offset)
  611. {
  612. struct uprobe *uprobe, *cur_uprobe;
  613. uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
  614. if (!uprobe)
  615. return NULL;
  616. uprobe->inode = inode;
  617. uprobe->offset = offset;
  618. uprobe->ref_ctr_offset = ref_ctr_offset;
  619. init_rwsem(&uprobe->register_rwsem);
  620. init_rwsem(&uprobe->consumer_rwsem);
  621. /* add to uprobes_tree, sorted on inode:offset */
  622. cur_uprobe = insert_uprobe(uprobe);
  623. /* a uprobe exists for this inode:offset combination */
  624. if (cur_uprobe) {
  625. if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) {
  626. ref_ctr_mismatch_warn(cur_uprobe, uprobe);
  627. put_uprobe(cur_uprobe);
  628. kfree(uprobe);
  629. return ERR_PTR(-EINVAL);
  630. }
  631. kfree(uprobe);
  632. uprobe = cur_uprobe;
  633. }
  634. return uprobe;
  635. }
  636. static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
  637. {
  638. down_write(&uprobe->consumer_rwsem);
  639. uc->next = uprobe->consumers;
  640. uprobe->consumers = uc;
  641. up_write(&uprobe->consumer_rwsem);
  642. }
  643. /*
  644. * For uprobe @uprobe, delete the consumer @uc.
  645. * Return true if the @uc is deleted successfully
  646. * or return false.
  647. */
  648. static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
  649. {
  650. struct uprobe_consumer **con;
  651. bool ret = false;
  652. down_write(&uprobe->consumer_rwsem);
  653. for (con = &uprobe->consumers; *con; con = &(*con)->next) {
  654. if (*con == uc) {
  655. *con = uc->next;
  656. ret = true;
  657. break;
  658. }
  659. }
  660. up_write(&uprobe->consumer_rwsem);
  661. return ret;
  662. }
  663. static int __copy_insn(struct address_space *mapping, struct file *filp,
  664. void *insn, int nbytes, loff_t offset)
  665. {
  666. struct page *page;
  667. /*
  668. * Ensure that the page that has the original instruction is populated
  669. * and in page-cache. If ->read_folio == NULL it must be shmem_mapping(),
  670. * see uprobe_register().
  671. */
  672. if (mapping->a_ops->read_folio)
  673. page = read_mapping_page(mapping, offset >> PAGE_SHIFT, filp);
  674. else
  675. page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
  676. if (IS_ERR(page))
  677. return PTR_ERR(page);
  678. copy_from_page(page, offset, insn, nbytes);
  679. put_page(page);
  680. return 0;
  681. }
  682. static int copy_insn(struct uprobe *uprobe, struct file *filp)
  683. {
  684. struct address_space *mapping = uprobe->inode->i_mapping;
  685. loff_t offs = uprobe->offset;
  686. void *insn = &uprobe->arch.insn;
  687. int size = sizeof(uprobe->arch.insn);
  688. int len, err = -EIO;
  689. /* Copy only available bytes, -EIO if nothing was read */
  690. do {
  691. if (offs >= i_size_read(uprobe->inode))
  692. break;
  693. len = min_t(int, size, PAGE_SIZE - (offs & ~PAGE_MASK));
  694. err = __copy_insn(mapping, filp, insn, len, offs);
  695. if (err)
  696. break;
  697. insn += len;
  698. offs += len;
  699. size -= len;
  700. } while (size);
  701. return err;
  702. }
  703. static int prepare_uprobe(struct uprobe *uprobe, struct file *file,
  704. struct mm_struct *mm, unsigned long vaddr)
  705. {
  706. int ret = 0;
  707. if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
  708. return ret;
  709. /* TODO: move this into _register, until then we abuse this sem. */
  710. down_write(&uprobe->consumer_rwsem);
  711. if (test_bit(UPROBE_COPY_INSN, &uprobe->flags))
  712. goto out;
  713. ret = copy_insn(uprobe, file);
  714. if (ret)
  715. goto out;
  716. ret = -ENOTSUPP;
  717. if (is_trap_insn((uprobe_opcode_t *)&uprobe->arch.insn))
  718. goto out;
  719. ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);
  720. if (ret)
  721. goto out;
  722. smp_wmb(); /* pairs with the smp_rmb() in handle_swbp() */
  723. set_bit(UPROBE_COPY_INSN, &uprobe->flags);
  724. out:
  725. up_write(&uprobe->consumer_rwsem);
  726. return ret;
  727. }
  728. static inline bool consumer_filter(struct uprobe_consumer *uc,
  729. enum uprobe_filter_ctx ctx, struct mm_struct *mm)
  730. {
  731. return !uc->filter || uc->filter(uc, ctx, mm);
  732. }
  733. static bool filter_chain(struct uprobe *uprobe,
  734. enum uprobe_filter_ctx ctx, struct mm_struct *mm)
  735. {
  736. struct uprobe_consumer *uc;
  737. bool ret = false;
  738. down_read(&uprobe->consumer_rwsem);
  739. for (uc = uprobe->consumers; uc; uc = uc->next) {
  740. ret = consumer_filter(uc, ctx, mm);
  741. if (ret)
  742. break;
  743. }
  744. up_read(&uprobe->consumer_rwsem);
  745. return ret;
  746. }
  747. static int
  748. install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
  749. struct vm_area_struct *vma, unsigned long vaddr)
  750. {
  751. bool first_uprobe;
  752. int ret;
  753. ret = prepare_uprobe(uprobe, vma->vm_file, mm, vaddr);
  754. if (ret)
  755. return ret;
  756. /*
  757. * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
  758. * the task can hit this breakpoint right after __replace_page().
  759. */
  760. first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);
  761. if (first_uprobe)
  762. set_bit(MMF_HAS_UPROBES, &mm->flags);
  763. ret = set_swbp(&uprobe->arch, mm, vaddr);
  764. if (!ret)
  765. clear_bit(MMF_RECALC_UPROBES, &mm->flags);
  766. else if (first_uprobe)
  767. clear_bit(MMF_HAS_UPROBES, &mm->flags);
  768. return ret;
  769. }
  770. static int
  771. remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)
  772. {
  773. set_bit(MMF_RECALC_UPROBES, &mm->flags);
  774. return set_orig_insn(&uprobe->arch, mm, vaddr);
  775. }
  776. static inline bool uprobe_is_active(struct uprobe *uprobe)
  777. {
  778. return !RB_EMPTY_NODE(&uprobe->rb_node);
  779. }
  780. /*
  781. * There could be threads that have already hit the breakpoint. They
  782. * will recheck the current insn and restart if find_uprobe() fails.
  783. * See find_active_uprobe().
  784. */
  785. static void delete_uprobe(struct uprobe *uprobe)
  786. {
  787. if (WARN_ON(!uprobe_is_active(uprobe)))
  788. return;
  789. spin_lock(&uprobes_treelock);
  790. rb_erase(&uprobe->rb_node, &uprobes_tree);
  791. spin_unlock(&uprobes_treelock);
  792. RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */
  793. put_uprobe(uprobe);
  794. }
  795. struct map_info {
  796. struct map_info *next;
  797. struct mm_struct *mm;
  798. unsigned long vaddr;
  799. };
  800. static inline struct map_info *free_map_info(struct map_info *info)
  801. {
  802. struct map_info *next = info->next;
  803. kfree(info);
  804. return next;
  805. }
  806. static struct map_info *
  807. build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
  808. {
  809. unsigned long pgoff = offset >> PAGE_SHIFT;
  810. struct vm_area_struct *vma;
  811. struct map_info *curr = NULL;
  812. struct map_info *prev = NULL;
  813. struct map_info *info;
  814. int more = 0;
  815. again:
  816. i_mmap_lock_read(mapping);
  817. vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
  818. if (!valid_vma(vma, is_register))
  819. continue;
  820. if (!prev && !more) {
  821. /*
  822. * Needs GFP_NOWAIT to avoid i_mmap_rwsem recursion through
  823. * reclaim. This is optimistic, no harm done if it fails.
  824. */
  825. prev = kmalloc(sizeof(struct map_info),
  826. GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
  827. if (prev)
  828. prev->next = NULL;
  829. }
  830. if (!prev) {
  831. more++;
  832. continue;
  833. }
  834. if (!mmget_not_zero(vma->vm_mm))
  835. continue;
  836. info = prev;
  837. prev = prev->next;
  838. info->next = curr;
  839. curr = info;
  840. info->mm = vma->vm_mm;
  841. info->vaddr = offset_to_vaddr(vma, offset);
  842. }
  843. i_mmap_unlock_read(mapping);
  844. if (!more)
  845. goto out;
  846. prev = curr;
  847. while (curr) {
  848. mmput(curr->mm);
  849. curr = curr->next;
  850. }
  851. do {
  852. info = kmalloc(sizeof(struct map_info), GFP_KERNEL);
  853. if (!info) {
  854. curr = ERR_PTR(-ENOMEM);
  855. goto out;
  856. }
  857. info->next = prev;
  858. prev = info;
  859. } while (--more);
  860. goto again;
  861. out:
  862. while (prev)
  863. prev = free_map_info(prev);
  864. return curr;
  865. }
  866. static int
  867. register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
  868. {
  869. bool is_register = !!new;
  870. struct map_info *info;
  871. int err = 0;
  872. percpu_down_write(&dup_mmap_sem);
  873. info = build_map_info(uprobe->inode->i_mapping,
  874. uprobe->offset, is_register);
  875. if (IS_ERR(info)) {
  876. err = PTR_ERR(info);
  877. goto out;
  878. }
  879. while (info) {
  880. struct mm_struct *mm = info->mm;
  881. struct vm_area_struct *vma;
  882. if (err && is_register)
  883. goto free;
  884. mmap_write_lock(mm);
  885. vma = find_vma(mm, info->vaddr);
  886. if (!vma || !valid_vma(vma, is_register) ||
  887. file_inode(vma->vm_file) != uprobe->inode)
  888. goto unlock;
  889. if (vma->vm_start > info->vaddr ||
  890. vaddr_to_offset(vma, info->vaddr) != uprobe->offset)
  891. goto unlock;
  892. if (is_register) {
  893. /* consult only the "caller", new consumer. */
  894. if (consumer_filter(new,
  895. UPROBE_FILTER_REGISTER, mm))
  896. err = install_breakpoint(uprobe, mm, vma, info->vaddr);
  897. } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) {
  898. if (!filter_chain(uprobe,
  899. UPROBE_FILTER_UNREGISTER, mm))
  900. err |= remove_breakpoint(uprobe, mm, info->vaddr);
  901. }
  902. unlock:
  903. mmap_write_unlock(mm);
  904. free:
  905. mmput(mm);
  906. info = free_map_info(info);
  907. }
  908. out:
  909. percpu_up_write(&dup_mmap_sem);
  910. return err;
  911. }
  912. static void
  913. __uprobe_unregister(struct uprobe *uprobe, struct uprobe_consumer *uc)
  914. {
  915. int err;
  916. if (WARN_ON(!consumer_del(uprobe, uc)))
  917. return;
  918. err = register_for_each_vma(uprobe, NULL);
  919. /* TODO : cant unregister? schedule a worker thread */
  920. if (!uprobe->consumers && !err)
  921. delete_uprobe(uprobe);
  922. }
  923. /*
  924. * uprobe_unregister - unregister an already registered probe.
  925. * @inode: the file in which the probe has to be removed.
  926. * @offset: offset from the start of the file.
  927. * @uc: identify which probe if multiple probes are colocated.
  928. */
  929. void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
  930. {
  931. struct uprobe *uprobe;
  932. uprobe = find_uprobe(inode, offset);
  933. if (WARN_ON(!uprobe))
  934. return;
  935. down_write(&uprobe->register_rwsem);
  936. __uprobe_unregister(uprobe, uc);
  937. up_write(&uprobe->register_rwsem);
  938. put_uprobe(uprobe);
  939. }
  940. EXPORT_SYMBOL_GPL(uprobe_unregister);
  941. /*
  942. * __uprobe_register - register a probe
  943. * @inode: the file in which the probe has to be placed.
  944. * @offset: offset from the start of the file.
  945. * @uc: information on howto handle the probe..
  946. *
  947. * Apart from the access refcount, __uprobe_register() takes a creation
  948. * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
  949. * inserted into the rbtree (i.e first consumer for a @inode:@offset
  950. * tuple). Creation refcount stops uprobe_unregister from freeing the
  951. * @uprobe even before the register operation is complete. Creation
  952. * refcount is released when the last @uc for the @uprobe
  953. * unregisters. Caller of __uprobe_register() is required to keep @inode
  954. * (and the containing mount) referenced.
  955. *
  956. * Return errno if it cannot successully install probes
  957. * else return 0 (success)
  958. */
  959. static int __uprobe_register(struct inode *inode, loff_t offset,
  960. loff_t ref_ctr_offset, struct uprobe_consumer *uc)
  961. {
  962. struct uprobe *uprobe;
  963. int ret;
  964. /* Uprobe must have at least one set consumer */
  965. if (!uc->handler && !uc->ret_handler)
  966. return -EINVAL;
  967. /* copy_insn() uses read_mapping_page() or shmem_read_mapping_page() */
  968. if (!inode->i_mapping->a_ops->read_folio &&
  969. !shmem_mapping(inode->i_mapping))
  970. return -EIO;
  971. /* Racy, just to catch the obvious mistakes */
  972. if (offset > i_size_read(inode))
  973. return -EINVAL;
  974. /*
  975. * This ensures that copy_from_page(), copy_to_page() and
  976. * __update_ref_ctr() can't cross page boundary.
  977. */
  978. if (!IS_ALIGNED(offset, UPROBE_SWBP_INSN_SIZE))
  979. return -EINVAL;
  980. if (!IS_ALIGNED(ref_ctr_offset, sizeof(short)))
  981. return -EINVAL;
  982. retry:
  983. uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
  984. if (!uprobe)
  985. return -ENOMEM;
  986. if (IS_ERR(uprobe))
  987. return PTR_ERR(uprobe);
  988. /*
  989. * We can race with uprobe_unregister()->delete_uprobe().
  990. * Check uprobe_is_active() and retry if it is false.
  991. */
  992. down_write(&uprobe->register_rwsem);
  993. ret = -EAGAIN;
  994. if (likely(uprobe_is_active(uprobe))) {
  995. consumer_add(uprobe, uc);
  996. ret = register_for_each_vma(uprobe, uc);
  997. if (ret)
  998. __uprobe_unregister(uprobe, uc);
  999. }
  1000. up_write(&uprobe->register_rwsem);
  1001. put_uprobe(uprobe);
  1002. if (unlikely(ret == -EAGAIN))
  1003. goto retry;
  1004. return ret;
  1005. }
  1006. int uprobe_register(struct inode *inode, loff_t offset,
  1007. struct uprobe_consumer *uc)
  1008. {
  1009. return __uprobe_register(inode, offset, 0, uc);
  1010. }
  1011. EXPORT_SYMBOL_GPL(uprobe_register);
  1012. int uprobe_register_refctr(struct inode *inode, loff_t offset,
  1013. loff_t ref_ctr_offset, struct uprobe_consumer *uc)
  1014. {
  1015. return __uprobe_register(inode, offset, ref_ctr_offset, uc);
  1016. }
  1017. EXPORT_SYMBOL_GPL(uprobe_register_refctr);
  1018. /*
  1019. * uprobe_apply - unregister an already registered probe.
  1020. * @inode: the file in which the probe has to be removed.
  1021. * @offset: offset from the start of the file.
  1022. * @uc: consumer which wants to add more or remove some breakpoints
  1023. * @add: add or remove the breakpoints
  1024. */
  1025. int uprobe_apply(struct inode *inode, loff_t offset,
  1026. struct uprobe_consumer *uc, bool add)
  1027. {
  1028. struct uprobe *uprobe;
  1029. struct uprobe_consumer *con;
  1030. int ret = -ENOENT;
  1031. uprobe = find_uprobe(inode, offset);
  1032. if (WARN_ON(!uprobe))
  1033. return ret;
  1034. down_write(&uprobe->register_rwsem);
  1035. for (con = uprobe->consumers; con && con != uc ; con = con->next)
  1036. ;
  1037. if (con)
  1038. ret = register_for_each_vma(uprobe, add ? uc : NULL);
  1039. up_write(&uprobe->register_rwsem);
  1040. put_uprobe(uprobe);
  1041. return ret;
  1042. }
  1043. static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
  1044. {
  1045. VMA_ITERATOR(vmi, mm, 0);
  1046. struct vm_area_struct *vma;
  1047. int err = 0;
  1048. mmap_read_lock(mm);
  1049. for_each_vma(vmi, vma) {
  1050. unsigned long vaddr;
  1051. loff_t offset;
  1052. if (!valid_vma(vma, false) ||
  1053. file_inode(vma->vm_file) != uprobe->inode)
  1054. continue;
  1055. offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
  1056. if (uprobe->offset < offset ||
  1057. uprobe->offset >= offset + vma->vm_end - vma->vm_start)
  1058. continue;
  1059. vaddr = offset_to_vaddr(vma, uprobe->offset);
  1060. err |= remove_breakpoint(uprobe, mm, vaddr);
  1061. }
  1062. mmap_read_unlock(mm);
  1063. return err;
  1064. }
  1065. static struct rb_node *
  1066. find_node_in_range(struct inode *inode, loff_t min, loff_t max)
  1067. {
  1068. struct rb_node *n = uprobes_tree.rb_node;
  1069. while (n) {
  1070. struct uprobe *u = rb_entry(n, struct uprobe, rb_node);
  1071. if (inode < u->inode) {
  1072. n = n->rb_left;
  1073. } else if (inode > u->inode) {
  1074. n = n->rb_right;
  1075. } else {
  1076. if (max < u->offset)
  1077. n = n->rb_left;
  1078. else if (min > u->offset)
  1079. n = n->rb_right;
  1080. else
  1081. break;
  1082. }
  1083. }
  1084. return n;
  1085. }
  1086. /*
  1087. * For a given range in vma, build a list of probes that need to be inserted.
  1088. */
  1089. static void build_probe_list(struct inode *inode,
  1090. struct vm_area_struct *vma,
  1091. unsigned long start, unsigned long end,
  1092. struct list_head *head)
  1093. {
  1094. loff_t min, max;
  1095. struct rb_node *n, *t;
  1096. struct uprobe *u;
  1097. INIT_LIST_HEAD(head);
  1098. min = vaddr_to_offset(vma, start);
  1099. max = min + (end - start) - 1;
  1100. spin_lock(&uprobes_treelock);
  1101. n = find_node_in_range(inode, min, max);
  1102. if (n) {
  1103. for (t = n; t; t = rb_prev(t)) {
  1104. u = rb_entry(t, struct uprobe, rb_node);
  1105. if (u->inode != inode || u->offset < min)
  1106. break;
  1107. list_add(&u->pending_list, head);
  1108. get_uprobe(u);
  1109. }
  1110. for (t = n; (t = rb_next(t)); ) {
  1111. u = rb_entry(t, struct uprobe, rb_node);
  1112. if (u->inode != inode || u->offset > max)
  1113. break;
  1114. list_add(&u->pending_list, head);
  1115. get_uprobe(u);
  1116. }
  1117. }
  1118. spin_unlock(&uprobes_treelock);
  1119. }
  1120. /* @vma contains reference counter, not the probed instruction. */
  1121. static int delayed_ref_ctr_inc(struct vm_area_struct *vma)
  1122. {
  1123. struct list_head *pos, *q;
  1124. struct delayed_uprobe *du;
  1125. unsigned long vaddr;
  1126. int ret = 0, err = 0;
  1127. mutex_lock(&delayed_uprobe_lock);
  1128. list_for_each_safe(pos, q, &delayed_uprobe_list) {
  1129. du = list_entry(pos, struct delayed_uprobe, list);
  1130. if (du->mm != vma->vm_mm ||
  1131. !valid_ref_ctr_vma(du->uprobe, vma))
  1132. continue;
  1133. vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset);
  1134. ret = __update_ref_ctr(vma->vm_mm, vaddr, 1);
  1135. if (ret) {
  1136. update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1);
  1137. if (!err)
  1138. err = ret;
  1139. }
  1140. delayed_uprobe_delete(du);
  1141. }
  1142. mutex_unlock(&delayed_uprobe_lock);
  1143. return err;
  1144. }
  1145. /*
  1146. * Called from mmap_region/vma_adjust with mm->mmap_lock acquired.
  1147. *
  1148. * Currently we ignore all errors and always return 0, the callers
  1149. * can't handle the failure anyway.
  1150. */
  1151. int uprobe_mmap(struct vm_area_struct *vma)
  1152. {
  1153. struct list_head tmp_list;
  1154. struct uprobe *uprobe, *u;
  1155. struct inode *inode;
  1156. if (no_uprobe_events())
  1157. return 0;
  1158. if (vma->vm_file &&
  1159. (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
  1160. test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags))
  1161. delayed_ref_ctr_inc(vma);
  1162. if (!valid_vma(vma, true))
  1163. return 0;
  1164. inode = file_inode(vma->vm_file);
  1165. if (!inode)
  1166. return 0;
  1167. mutex_lock(uprobes_mmap_hash(inode));
  1168. build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);
  1169. /*
  1170. * We can race with uprobe_unregister(), this uprobe can be already
  1171. * removed. But in this case filter_chain() must return false, all
  1172. * consumers have gone away.
  1173. */
  1174. list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {
  1175. if (!fatal_signal_pending(current) &&
  1176. filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {
  1177. unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
  1178. install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);
  1179. }
  1180. put_uprobe(uprobe);
  1181. }
  1182. mutex_unlock(uprobes_mmap_hash(inode));
  1183. return 0;
  1184. }
  1185. static bool
  1186. vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  1187. {
  1188. loff_t min, max;
  1189. struct inode *inode;
  1190. struct rb_node *n;
  1191. inode = file_inode(vma->vm_file);
  1192. min = vaddr_to_offset(vma, start);
  1193. max = min + (end - start) - 1;
  1194. spin_lock(&uprobes_treelock);
  1195. n = find_node_in_range(inode, min, max);
  1196. spin_unlock(&uprobes_treelock);
  1197. return !!n;
  1198. }
  1199. /*
  1200. * Called in context of a munmap of a vma.
  1201. */
  1202. void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  1203. {
  1204. if (no_uprobe_events() || !valid_vma(vma, false))
  1205. return;
  1206. if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
  1207. return;
  1208. if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||
  1209. test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))
  1210. return;
  1211. if (vma_has_uprobes(vma, start, end))
  1212. set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);
  1213. }
  1214. /* Slot allocation for XOL */
  1215. static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
  1216. {
  1217. struct vm_area_struct *vma;
  1218. int ret;
  1219. if (mmap_write_lock_killable(mm))
  1220. return -EINTR;
  1221. if (mm->uprobes_state.xol_area) {
  1222. ret = -EALREADY;
  1223. goto fail;
  1224. }
  1225. if (!area->vaddr) {
  1226. /* Try to map as high as possible, this is only a hint. */
  1227. area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
  1228. PAGE_SIZE, 0, 0);
  1229. if (IS_ERR_VALUE(area->vaddr)) {
  1230. ret = area->vaddr;
  1231. goto fail;
  1232. }
  1233. }
  1234. vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,
  1235. VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO,
  1236. &area->xol_mapping);
  1237. if (IS_ERR(vma)) {
  1238. ret = PTR_ERR(vma);
  1239. goto fail;
  1240. }
  1241. ret = 0;
  1242. /* pairs with get_xol_area() */
  1243. smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
  1244. fail:
  1245. mmap_write_unlock(mm);
  1246. return ret;
  1247. }
  1248. static struct xol_area *__create_xol_area(unsigned long vaddr)
  1249. {
  1250. struct mm_struct *mm = current->mm;
  1251. uprobe_opcode_t insn = UPROBE_SWBP_INSN;
  1252. struct xol_area *area;
  1253. area = kmalloc(sizeof(*area), GFP_KERNEL);
  1254. if (unlikely(!area))
  1255. goto out;
  1256. area->bitmap = kcalloc(BITS_TO_LONGS(UINSNS_PER_PAGE), sizeof(long),
  1257. GFP_KERNEL);
  1258. if (!area->bitmap)
  1259. goto free_area;
  1260. area->xol_mapping.name = "[uprobes]";
  1261. area->xol_mapping.fault = NULL;
  1262. area->xol_mapping.pages = area->pages;
  1263. area->pages[0] = alloc_page(GFP_HIGHUSER);
  1264. if (!area->pages[0])
  1265. goto free_bitmap;
  1266. area->pages[1] = NULL;
  1267. area->vaddr = vaddr;
  1268. init_waitqueue_head(&area->wq);
  1269. /* Reserve the 1st slot for get_trampoline_vaddr() */
  1270. set_bit(0, area->bitmap);
  1271. atomic_set(&area->slot_count, 1);
  1272. arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);
  1273. if (!xol_add_vma(mm, area))
  1274. return area;
  1275. __free_page(area->pages[0]);
  1276. free_bitmap:
  1277. kfree(area->bitmap);
  1278. free_area:
  1279. kfree(area);
  1280. out:
  1281. return NULL;
  1282. }
  1283. /*
  1284. * get_xol_area - Allocate process's xol_area if necessary.
  1285. * This area will be used for storing instructions for execution out of line.
  1286. *
  1287. * Returns the allocated area or NULL.
  1288. */
  1289. static struct xol_area *get_xol_area(void)
  1290. {
  1291. struct mm_struct *mm = current->mm;
  1292. struct xol_area *area;
  1293. if (!mm->uprobes_state.xol_area)
  1294. __create_xol_area(0);
  1295. /* Pairs with xol_add_vma() smp_store_release() */
  1296. area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */
  1297. return area;
  1298. }
  1299. /*
  1300. * uprobe_clear_state - Free the area allocated for slots.
  1301. */
  1302. void uprobe_clear_state(struct mm_struct *mm)
  1303. {
  1304. struct xol_area *area = mm->uprobes_state.xol_area;
  1305. mutex_lock(&delayed_uprobe_lock);
  1306. delayed_uprobe_remove(NULL, mm);
  1307. mutex_unlock(&delayed_uprobe_lock);
  1308. if (!area)
  1309. return;
  1310. put_page(area->pages[0]);
  1311. kfree(area->bitmap);
  1312. kfree(area);
  1313. }
  1314. void uprobe_start_dup_mmap(void)
  1315. {
  1316. percpu_down_read(&dup_mmap_sem);
  1317. }
  1318. void uprobe_end_dup_mmap(void)
  1319. {
  1320. percpu_up_read(&dup_mmap_sem);
  1321. }
  1322. void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
  1323. {
  1324. if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {
  1325. set_bit(MMF_HAS_UPROBES, &newmm->flags);
  1326. /* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */
  1327. set_bit(MMF_RECALC_UPROBES, &newmm->flags);
  1328. }
  1329. }
  1330. /*
  1331. * - search for a free slot.
  1332. */
  1333. static unsigned long xol_take_insn_slot(struct xol_area *area)
  1334. {
  1335. unsigned long slot_addr;
  1336. int slot_nr;
  1337. do {
  1338. slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
  1339. if (slot_nr < UINSNS_PER_PAGE) {
  1340. if (!test_and_set_bit(slot_nr, area->bitmap))
  1341. break;
  1342. slot_nr = UINSNS_PER_PAGE;
  1343. continue;
  1344. }
  1345. wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
  1346. } while (slot_nr >= UINSNS_PER_PAGE);
  1347. slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
  1348. atomic_inc(&area->slot_count);
  1349. return slot_addr;
  1350. }
  1351. /*
  1352. * xol_get_insn_slot - allocate a slot for xol.
  1353. * Returns the allocated slot address or 0.
  1354. */
  1355. static unsigned long xol_get_insn_slot(struct uprobe *uprobe)
  1356. {
  1357. struct xol_area *area;
  1358. unsigned long xol_vaddr;
  1359. area = get_xol_area();
  1360. if (!area)
  1361. return 0;
  1362. xol_vaddr = xol_take_insn_slot(area);
  1363. if (unlikely(!xol_vaddr))
  1364. return 0;
  1365. arch_uprobe_copy_ixol(area->pages[0], xol_vaddr,
  1366. &uprobe->arch.ixol, sizeof(uprobe->arch.ixol));
  1367. return xol_vaddr;
  1368. }
  1369. /*
  1370. * xol_free_insn_slot - If slot was earlier allocated by
  1371. * @xol_get_insn_slot(), make the slot available for
  1372. * subsequent requests.
  1373. */
  1374. static void xol_free_insn_slot(struct task_struct *tsk)
  1375. {
  1376. struct xol_area *area;
  1377. unsigned long vma_end;
  1378. unsigned long slot_addr;
  1379. if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)
  1380. return;
  1381. slot_addr = tsk->utask->xol_vaddr;
  1382. if (unlikely(!slot_addr))
  1383. return;
  1384. area = tsk->mm->uprobes_state.xol_area;
  1385. vma_end = area->vaddr + PAGE_SIZE;
  1386. if (area->vaddr <= slot_addr && slot_addr < vma_end) {
  1387. unsigned long offset;
  1388. int slot_nr;
  1389. offset = slot_addr - area->vaddr;
  1390. slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
  1391. if (slot_nr >= UINSNS_PER_PAGE)
  1392. return;
  1393. clear_bit(slot_nr, area->bitmap);
  1394. atomic_dec(&area->slot_count);
  1395. smp_mb__after_atomic(); /* pairs with prepare_to_wait() */
  1396. if (waitqueue_active(&area->wq))
  1397. wake_up(&area->wq);
  1398. tsk->utask->xol_vaddr = 0;
  1399. }
  1400. }
  1401. void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
  1402. void *src, unsigned long len)
  1403. {
  1404. /* Initialize the slot */
  1405. copy_to_page(page, vaddr, src, len);
  1406. /*
  1407. * We probably need flush_icache_user_page() but it needs vma.
  1408. * This should work on most of architectures by default. If
  1409. * architecture needs to do something different it can define
  1410. * its own version of the function.
  1411. */
  1412. flush_dcache_page(page);
  1413. }
  1414. /**
  1415. * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
  1416. * @regs: Reflects the saved state of the task after it has hit a breakpoint
  1417. * instruction.
  1418. * Return the address of the breakpoint instruction.
  1419. */
  1420. unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
  1421. {
  1422. return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
  1423. }
  1424. unsigned long uprobe_get_trap_addr(struct pt_regs *regs)
  1425. {
  1426. struct uprobe_task *utask = current->utask;
  1427. if (unlikely(utask && utask->active_uprobe))
  1428. return utask->vaddr;
  1429. return instruction_pointer(regs);
  1430. }
  1431. static struct return_instance *free_ret_instance(struct return_instance *ri)
  1432. {
  1433. struct return_instance *next = ri->next;
  1434. put_uprobe(ri->uprobe);
  1435. kfree(ri);
  1436. return next;
  1437. }
  1438. /*
  1439. * Called with no locks held.
  1440. * Called in context of an exiting or an exec-ing thread.
  1441. */
  1442. void uprobe_free_utask(struct task_struct *t)
  1443. {
  1444. struct uprobe_task *utask = t->utask;
  1445. struct return_instance *ri;
  1446. if (!utask)
  1447. return;
  1448. if (utask->active_uprobe)
  1449. put_uprobe(utask->active_uprobe);
  1450. ri = utask->return_instances;
  1451. while (ri)
  1452. ri = free_ret_instance(ri);
  1453. xol_free_insn_slot(t);
  1454. kfree(utask);
  1455. t->utask = NULL;
  1456. }
  1457. /*
  1458. * Allocate a uprobe_task object for the task if necessary.
  1459. * Called when the thread hits a breakpoint.
  1460. *
  1461. * Returns:
  1462. * - pointer to new uprobe_task on success
  1463. * - NULL otherwise
  1464. */
  1465. static struct uprobe_task *get_utask(void)
  1466. {
  1467. if (!current->utask)
  1468. current->utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
  1469. return current->utask;
  1470. }
  1471. static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask)
  1472. {
  1473. struct uprobe_task *n_utask;
  1474. struct return_instance **p, *o, *n;
  1475. n_utask = kzalloc(sizeof(struct uprobe_task), GFP_KERNEL);
  1476. if (!n_utask)
  1477. return -ENOMEM;
  1478. t->utask = n_utask;
  1479. p = &n_utask->return_instances;
  1480. for (o = o_utask->return_instances; o; o = o->next) {
  1481. n = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
  1482. if (!n)
  1483. return -ENOMEM;
  1484. *n = *o;
  1485. get_uprobe(n->uprobe);
  1486. n->next = NULL;
  1487. *p = n;
  1488. p = &n->next;
  1489. n_utask->depth++;
  1490. }
  1491. return 0;
  1492. }
  1493. static void uprobe_warn(struct task_struct *t, const char *msg)
  1494. {
  1495. pr_warn("uprobe: %s:%d failed to %s\n",
  1496. current->comm, current->pid, msg);
  1497. }
  1498. static void dup_xol_work(struct callback_head *work)
  1499. {
  1500. if (current->flags & PF_EXITING)
  1501. return;
  1502. if (!__create_xol_area(current->utask->dup_xol_addr) &&
  1503. !fatal_signal_pending(current))
  1504. uprobe_warn(current, "dup xol area");
  1505. }
  1506. /*
  1507. * Called in context of a new clone/fork from copy_process.
  1508. */
  1509. void uprobe_copy_process(struct task_struct *t, unsigned long flags)
  1510. {
  1511. struct uprobe_task *utask = current->utask;
  1512. struct mm_struct *mm = current->mm;
  1513. struct xol_area *area;
  1514. t->utask = NULL;
  1515. if (!utask || !utask->return_instances)
  1516. return;
  1517. if (mm == t->mm && !(flags & CLONE_VFORK))
  1518. return;
  1519. if (dup_utask(t, utask))
  1520. return uprobe_warn(t, "dup ret instances");
  1521. /* The task can fork() after dup_xol_work() fails */
  1522. area = mm->uprobes_state.xol_area;
  1523. if (!area)
  1524. return uprobe_warn(t, "dup xol area");
  1525. if (mm == t->mm)
  1526. return;
  1527. t->utask->dup_xol_addr = area->vaddr;
  1528. init_task_work(&t->utask->dup_xol_work, dup_xol_work);
  1529. task_work_add(t, &t->utask->dup_xol_work, TWA_RESUME);
  1530. }
  1531. /*
  1532. * Current area->vaddr notion assume the trampoline address is always
  1533. * equal area->vaddr.
  1534. *
  1535. * Returns -1 in case the xol_area is not allocated.
  1536. */
  1537. static unsigned long get_trampoline_vaddr(void)
  1538. {
  1539. struct xol_area *area;
  1540. unsigned long trampoline_vaddr = -1;
  1541. /* Pairs with xol_add_vma() smp_store_release() */
  1542. area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */
  1543. if (area)
  1544. trampoline_vaddr = area->vaddr;
  1545. return trampoline_vaddr;
  1546. }
  1547. static void cleanup_return_instances(struct uprobe_task *utask, bool chained,
  1548. struct pt_regs *regs)
  1549. {
  1550. struct return_instance *ri = utask->return_instances;
  1551. enum rp_check ctx = chained ? RP_CHECK_CHAIN_CALL : RP_CHECK_CALL;
  1552. while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) {
  1553. ri = free_ret_instance(ri);
  1554. utask->depth--;
  1555. }
  1556. utask->return_instances = ri;
  1557. }
  1558. static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs)
  1559. {
  1560. struct return_instance *ri;
  1561. struct uprobe_task *utask;
  1562. unsigned long orig_ret_vaddr, trampoline_vaddr;
  1563. bool chained;
  1564. if (!get_xol_area())
  1565. return;
  1566. utask = get_utask();
  1567. if (!utask)
  1568. return;
  1569. if (utask->depth >= MAX_URETPROBE_DEPTH) {
  1570. printk_ratelimited(KERN_INFO "uprobe: omit uretprobe due to"
  1571. " nestedness limit pid/tgid=%d/%d\n",
  1572. current->pid, current->tgid);
  1573. return;
  1574. }
  1575. ri = kmalloc(sizeof(struct return_instance), GFP_KERNEL);
  1576. if (!ri)
  1577. return;
  1578. trampoline_vaddr = get_trampoline_vaddr();
  1579. orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
  1580. if (orig_ret_vaddr == -1)
  1581. goto fail;
  1582. /* drop the entries invalidated by longjmp() */
  1583. chained = (orig_ret_vaddr == trampoline_vaddr);
  1584. cleanup_return_instances(utask, chained, regs);
  1585. /*
  1586. * We don't want to keep trampoline address in stack, rather keep the
  1587. * original return address of first caller thru all the consequent
  1588. * instances. This also makes breakpoint unwrapping easier.
  1589. */
  1590. if (chained) {
  1591. if (!utask->return_instances) {
  1592. /*
  1593. * This situation is not possible. Likely we have an
  1594. * attack from user-space.
  1595. */
  1596. uprobe_warn(current, "handle tail call");
  1597. goto fail;
  1598. }
  1599. orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
  1600. }
  1601. ri->uprobe = get_uprobe(uprobe);
  1602. ri->func = instruction_pointer(regs);
  1603. ri->stack = user_stack_pointer(regs);
  1604. ri->orig_ret_vaddr = orig_ret_vaddr;
  1605. ri->chained = chained;
  1606. utask->depth++;
  1607. ri->next = utask->return_instances;
  1608. utask->return_instances = ri;
  1609. return;
  1610. fail:
  1611. kfree(ri);
  1612. }
  1613. /* Prepare to single-step probed instruction out of line. */
  1614. static int
  1615. pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long bp_vaddr)
  1616. {
  1617. struct uprobe_task *utask;
  1618. unsigned long xol_vaddr;
  1619. int err;
  1620. utask = get_utask();
  1621. if (!utask)
  1622. return -ENOMEM;
  1623. xol_vaddr = xol_get_insn_slot(uprobe);
  1624. if (!xol_vaddr)
  1625. return -ENOMEM;
  1626. utask->xol_vaddr = xol_vaddr;
  1627. utask->vaddr = bp_vaddr;
  1628. err = arch_uprobe_pre_xol(&uprobe->arch, regs);
  1629. if (unlikely(err)) {
  1630. xol_free_insn_slot(current);
  1631. return err;
  1632. }
  1633. utask->active_uprobe = uprobe;
  1634. utask->state = UTASK_SSTEP;
  1635. return 0;
  1636. }
  1637. /*
  1638. * If we are singlestepping, then ensure this thread is not connected to
  1639. * non-fatal signals until completion of singlestep. When xol insn itself
  1640. * triggers the signal, restart the original insn even if the task is
  1641. * already SIGKILL'ed (since coredump should report the correct ip). This
  1642. * is even more important if the task has a handler for SIGSEGV/etc, The
  1643. * _same_ instruction should be repeated again after return from the signal
  1644. * handler, and SSTEP can never finish in this case.
  1645. */
  1646. bool uprobe_deny_signal(void)
  1647. {
  1648. struct task_struct *t = current;
  1649. struct uprobe_task *utask = t->utask;
  1650. if (likely(!utask || !utask->active_uprobe))
  1651. return false;
  1652. WARN_ON_ONCE(utask->state != UTASK_SSTEP);
  1653. if (task_sigpending(t)) {
  1654. spin_lock_irq(&t->sighand->siglock);
  1655. clear_tsk_thread_flag(t, TIF_SIGPENDING);
  1656. spin_unlock_irq(&t->sighand->siglock);
  1657. if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
  1658. utask->state = UTASK_SSTEP_TRAPPED;
  1659. set_tsk_thread_flag(t, TIF_UPROBE);
  1660. }
  1661. }
  1662. return true;
  1663. }
  1664. static void mmf_recalc_uprobes(struct mm_struct *mm)
  1665. {
  1666. VMA_ITERATOR(vmi, mm, 0);
  1667. struct vm_area_struct *vma;
  1668. for_each_vma(vmi, vma) {
  1669. if (!valid_vma(vma, false))
  1670. continue;
  1671. /*
  1672. * This is not strictly accurate, we can race with
  1673. * uprobe_unregister() and see the already removed
  1674. * uprobe if delete_uprobe() was not yet called.
  1675. * Or this uprobe can be filtered out.
  1676. */
  1677. if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))
  1678. return;
  1679. }
  1680. clear_bit(MMF_HAS_UPROBES, &mm->flags);
  1681. }
  1682. static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
  1683. {
  1684. struct page *page;
  1685. uprobe_opcode_t opcode;
  1686. int result;
  1687. if (WARN_ON_ONCE(!IS_ALIGNED(vaddr, UPROBE_SWBP_INSN_SIZE)))
  1688. return -EINVAL;
  1689. pagefault_disable();
  1690. result = __get_user(opcode, (uprobe_opcode_t __user *)vaddr);
  1691. pagefault_enable();
  1692. if (likely(result == 0))
  1693. goto out;
  1694. /*
  1695. * The NULL 'tsk' here ensures that any faults that occur here
  1696. * will not be accounted to the task. 'mm' *is* current->mm,
  1697. * but we treat this as a 'remote' access since it is
  1698. * essentially a kernel access to the memory.
  1699. */
  1700. result = get_user_pages_remote(mm, vaddr, 1, FOLL_FORCE, &page,
  1701. NULL, NULL);
  1702. if (result < 0)
  1703. return result;
  1704. copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
  1705. put_page(page);
  1706. out:
  1707. /* This needs to return true for any variant of the trap insn */
  1708. return is_trap_insn(&opcode);
  1709. }
  1710. static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
  1711. {
  1712. struct mm_struct *mm = current->mm;
  1713. struct uprobe *uprobe = NULL;
  1714. struct vm_area_struct *vma;
  1715. mmap_read_lock(mm);
  1716. vma = vma_lookup(mm, bp_vaddr);
  1717. if (vma) {
  1718. if (valid_vma(vma, false)) {
  1719. struct inode *inode = file_inode(vma->vm_file);
  1720. loff_t offset = vaddr_to_offset(vma, bp_vaddr);
  1721. uprobe = find_uprobe(inode, offset);
  1722. }
  1723. if (!uprobe)
  1724. *is_swbp = is_trap_at_addr(mm, bp_vaddr);
  1725. } else {
  1726. *is_swbp = -EFAULT;
  1727. }
  1728. if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))
  1729. mmf_recalc_uprobes(mm);
  1730. mmap_read_unlock(mm);
  1731. return uprobe;
  1732. }
  1733. static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
  1734. {
  1735. struct uprobe_consumer *uc;
  1736. int remove = UPROBE_HANDLER_REMOVE;
  1737. bool need_prep = false; /* prepare return uprobe, when needed */
  1738. down_read(&uprobe->register_rwsem);
  1739. for (uc = uprobe->consumers; uc; uc = uc->next) {
  1740. int rc = 0;
  1741. if (uc->handler) {
  1742. rc = uc->handler(uc, regs);
  1743. WARN(rc & ~UPROBE_HANDLER_MASK,
  1744. "bad rc=0x%x from %ps()\n", rc, uc->handler);
  1745. }
  1746. if (uc->ret_handler)
  1747. need_prep = true;
  1748. remove &= rc;
  1749. }
  1750. if (need_prep && !remove)
  1751. prepare_uretprobe(uprobe, regs); /* put bp at return */
  1752. if (remove && uprobe->consumers) {
  1753. WARN_ON(!uprobe_is_active(uprobe));
  1754. unapply_uprobe(uprobe, current->mm);
  1755. }
  1756. up_read(&uprobe->register_rwsem);
  1757. }
  1758. static void
  1759. handle_uretprobe_chain(struct return_instance *ri, struct pt_regs *regs)
  1760. {
  1761. struct uprobe *uprobe = ri->uprobe;
  1762. struct uprobe_consumer *uc;
  1763. down_read(&uprobe->register_rwsem);
  1764. for (uc = uprobe->consumers; uc; uc = uc->next) {
  1765. if (uc->ret_handler)
  1766. uc->ret_handler(uc, ri->func, regs);
  1767. }
  1768. up_read(&uprobe->register_rwsem);
  1769. }
  1770. static struct return_instance *find_next_ret_chain(struct return_instance *ri)
  1771. {
  1772. bool chained;
  1773. do {
  1774. chained = ri->chained;
  1775. ri = ri->next; /* can't be NULL if chained */
  1776. } while (chained);
  1777. return ri;
  1778. }
  1779. static void handle_trampoline(struct pt_regs *regs)
  1780. {
  1781. struct uprobe_task *utask;
  1782. struct return_instance *ri, *next;
  1783. bool valid;
  1784. utask = current->utask;
  1785. if (!utask)
  1786. goto sigill;
  1787. ri = utask->return_instances;
  1788. if (!ri)
  1789. goto sigill;
  1790. do {
  1791. /*
  1792. * We should throw out the frames invalidated by longjmp().
  1793. * If this chain is valid, then the next one should be alive
  1794. * or NULL; the latter case means that nobody but ri->func
  1795. * could hit this trampoline on return. TODO: sigaltstack().
  1796. */
  1797. next = find_next_ret_chain(ri);
  1798. valid = !next || arch_uretprobe_is_alive(next, RP_CHECK_RET, regs);
  1799. instruction_pointer_set(regs, ri->orig_ret_vaddr);
  1800. do {
  1801. if (valid)
  1802. handle_uretprobe_chain(ri, regs);
  1803. ri = free_ret_instance(ri);
  1804. utask->depth--;
  1805. } while (ri != next);
  1806. } while (!valid);
  1807. utask->return_instances = ri;
  1808. return;
  1809. sigill:
  1810. uprobe_warn(current, "handle uretprobe, sending SIGILL.");
  1811. force_sig(SIGILL);
  1812. }
  1813. bool __weak arch_uprobe_ignore(struct arch_uprobe *aup, struct pt_regs *regs)
  1814. {
  1815. return false;
  1816. }
  1817. bool __weak arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
  1818. struct pt_regs *regs)
  1819. {
  1820. return true;
  1821. }
  1822. /*
  1823. * Run handler and ask thread to singlestep.
  1824. * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
  1825. */
  1826. static void handle_swbp(struct pt_regs *regs)
  1827. {
  1828. struct uprobe *uprobe;
  1829. unsigned long bp_vaddr;
  1830. int is_swbp;
  1831. bp_vaddr = uprobe_get_swbp_addr(regs);
  1832. if (bp_vaddr == get_trampoline_vaddr())
  1833. return handle_trampoline(regs);
  1834. uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
  1835. if (!uprobe) {
  1836. if (is_swbp > 0) {
  1837. /* No matching uprobe; signal SIGTRAP. */
  1838. force_sig(SIGTRAP);
  1839. } else {
  1840. /*
  1841. * Either we raced with uprobe_unregister() or we can't
  1842. * access this memory. The latter is only possible if
  1843. * another thread plays with our ->mm. In both cases
  1844. * we can simply restart. If this vma was unmapped we
  1845. * can pretend this insn was not executed yet and get
  1846. * the (correct) SIGSEGV after restart.
  1847. */
  1848. instruction_pointer_set(regs, bp_vaddr);
  1849. }
  1850. return;
  1851. }
  1852. /* change it in advance for ->handler() and restart */
  1853. instruction_pointer_set(regs, bp_vaddr);
  1854. /*
  1855. * TODO: move copy_insn/etc into _register and remove this hack.
  1856. * After we hit the bp, _unregister + _register can install the
  1857. * new and not-yet-analyzed uprobe at the same address, restart.
  1858. */
  1859. if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
  1860. goto out;
  1861. /*
  1862. * Pairs with the smp_wmb() in prepare_uprobe().
  1863. *
  1864. * Guarantees that if we see the UPROBE_COPY_INSN bit set, then
  1865. * we must also see the stores to &uprobe->arch performed by the
  1866. * prepare_uprobe() call.
  1867. */
  1868. smp_rmb();
  1869. /* Tracing handlers use ->utask to communicate with fetch methods */
  1870. if (!get_utask())
  1871. goto out;
  1872. if (arch_uprobe_ignore(&uprobe->arch, regs))
  1873. goto out;
  1874. handler_chain(uprobe, regs);
  1875. if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
  1876. goto out;
  1877. if (!pre_ssout(uprobe, regs, bp_vaddr))
  1878. return;
  1879. /* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
  1880. out:
  1881. put_uprobe(uprobe);
  1882. }
  1883. /*
  1884. * Perform required fix-ups and disable singlestep.
  1885. * Allow pending signals to take effect.
  1886. */
  1887. static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
  1888. {
  1889. struct uprobe *uprobe;
  1890. int err = 0;
  1891. uprobe = utask->active_uprobe;
  1892. if (utask->state == UTASK_SSTEP_ACK)
  1893. err = arch_uprobe_post_xol(&uprobe->arch, regs);
  1894. else if (utask->state == UTASK_SSTEP_TRAPPED)
  1895. arch_uprobe_abort_xol(&uprobe->arch, regs);
  1896. else
  1897. WARN_ON_ONCE(1);
  1898. put_uprobe(uprobe);
  1899. utask->active_uprobe = NULL;
  1900. utask->state = UTASK_RUNNING;
  1901. xol_free_insn_slot(current);
  1902. spin_lock_irq(&current->sighand->siglock);
  1903. recalc_sigpending(); /* see uprobe_deny_signal() */
  1904. spin_unlock_irq(&current->sighand->siglock);
  1905. if (unlikely(err)) {
  1906. uprobe_warn(current, "execute the probed insn, sending SIGILL.");
  1907. force_sig(SIGILL);
  1908. }
  1909. }
  1910. /*
  1911. * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
  1912. * allows the thread to return from interrupt. After that handle_swbp()
  1913. * sets utask->active_uprobe.
  1914. *
  1915. * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
  1916. * and allows the thread to return from interrupt.
  1917. *
  1918. * While returning to userspace, thread notices the TIF_UPROBE flag and calls
  1919. * uprobe_notify_resume().
  1920. */
  1921. void uprobe_notify_resume(struct pt_regs *regs)
  1922. {
  1923. struct uprobe_task *utask;
  1924. clear_thread_flag(TIF_UPROBE);
  1925. utask = current->utask;
  1926. if (utask && utask->active_uprobe)
  1927. handle_singlestep(utask, regs);
  1928. else
  1929. handle_swbp(regs);
  1930. }
  1931. /*
  1932. * uprobe_pre_sstep_notifier gets called from interrupt context as part of
  1933. * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
  1934. */
  1935. int uprobe_pre_sstep_notifier(struct pt_regs *regs)
  1936. {
  1937. if (!current->mm)
  1938. return 0;
  1939. if (!test_bit(MMF_HAS_UPROBES, &current->mm->flags) &&
  1940. (!current->utask || !current->utask->return_instances))
  1941. return 0;
  1942. set_thread_flag(TIF_UPROBE);
  1943. return 1;
  1944. }
  1945. /*
  1946. * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
  1947. * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
  1948. */
  1949. int uprobe_post_sstep_notifier(struct pt_regs *regs)
  1950. {
  1951. struct uprobe_task *utask = current->utask;
  1952. if (!current->mm || !utask || !utask->active_uprobe)
  1953. /* task is currently not uprobed */
  1954. return 0;
  1955. utask->state = UTASK_SSTEP_ACK;
  1956. set_thread_flag(TIF_UPROBE);
  1957. return 1;
  1958. }
  1959. static struct notifier_block uprobe_exception_nb = {
  1960. .notifier_call = arch_uprobe_exception_notify,
  1961. .priority = INT_MAX-1, /* notified after kprobes, kgdb */
  1962. };
  1963. void __init uprobes_init(void)
  1964. {
  1965. int i;
  1966. for (i = 0; i < UPROBES_HASH_SZ; i++)
  1967. mutex_init(&uprobes_mmap_mutex[i]);
  1968. BUG_ON(register_die_notifier(&uprobe_exception_nb));
  1969. }