sem.c 63 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * linux/ipc/sem.c
  4. * Copyright (C) 1992 Krishna Balasubramanian
  5. * Copyright (C) 1995 Eric Schenk, Bruno Haible
  6. *
  7. * /proc/sysvipc/sem support (c) 1999 Dragos Acostachioaie <[email protected]>
  8. *
  9. * SMP-threaded, sysctl's added
  10. * (c) 1999 Manfred Spraul <[email protected]>
  11. * Enforced range limit on SEM_UNDO
  12. * (c) 2001 Red Hat Inc
  13. * Lockless wakeup
  14. * (c) 2003 Manfred Spraul <[email protected]>
  15. * (c) 2016 Davidlohr Bueso <[email protected]>
  16. * Further wakeup optimizations, documentation
  17. * (c) 2010 Manfred Spraul <[email protected]>
  18. *
  19. * support for audit of ipc object properties and permission changes
  20. * Dustin Kirkland <[email protected]>
  21. *
  22. * namespaces support
  23. * OpenVZ, SWsoft Inc.
  24. * Pavel Emelianov <[email protected]>
  25. *
  26. * Implementation notes: (May 2010)
  27. * This file implements System V semaphores.
  28. *
  29. * User space visible behavior:
  30. * - FIFO ordering for semop() operations (just FIFO, not starvation
  31. * protection)
  32. * - multiple semaphore operations that alter the same semaphore in
  33. * one semop() are handled.
  34. * - sem_ctime (time of last semctl()) is updated in the IPC_SET, SETVAL and
  35. * SETALL calls.
  36. * - two Linux specific semctl() commands: SEM_STAT, SEM_INFO.
  37. * - undo adjustments at process exit are limited to 0..SEMVMX.
  38. * - namespace are supported.
  39. * - SEMMSL, SEMMNS, SEMOPM and SEMMNI can be configured at runtime by writing
  40. * to /proc/sys/kernel/sem.
  41. * - statistics about the usage are reported in /proc/sysvipc/sem.
  42. *
  43. * Internals:
  44. * - scalability:
  45. * - all global variables are read-mostly.
  46. * - semop() calls and semctl(RMID) are synchronized by RCU.
  47. * - most operations do write operations (actually: spin_lock calls) to
  48. * the per-semaphore array structure.
  49. * Thus: Perfect SMP scaling between independent semaphore arrays.
  50. * If multiple semaphores in one array are used, then cache line
  51. * trashing on the semaphore array spinlock will limit the scaling.
  52. * - semncnt and semzcnt are calculated on demand in count_semcnt()
  53. * - the task that performs a successful semop() scans the list of all
  54. * sleeping tasks and completes any pending operations that can be fulfilled.
  55. * Semaphores are actively given to waiting tasks (necessary for FIFO).
  56. * (see update_queue())
  57. * - To improve the scalability, the actual wake-up calls are performed after
  58. * dropping all locks. (see wake_up_sem_queue_prepare())
  59. * - All work is done by the waker, the woken up task does not have to do
  60. * anything - not even acquiring a lock or dropping a refcount.
  61. * - A woken up task may not even touch the semaphore array anymore, it may
  62. * have been destroyed already by a semctl(RMID).
  63. * - UNDO values are stored in an array (one per process and per
  64. * semaphore array, lazily allocated). For backwards compatibility, multiple
  65. * modes for the UNDO variables are supported (per process, per thread)
  66. * (see copy_semundo, CLONE_SYSVSEM)
  67. * - There are two lists of the pending operations: a per-array list
  68. * and per-semaphore list (stored in the array). This allows to achieve FIFO
  69. * ordering without always scanning all pending operations.
  70. * The worst-case behavior is nevertheless O(N^2) for N wakeups.
  71. */
  72. #include <linux/compat.h>
  73. #include <linux/slab.h>
  74. #include <linux/spinlock.h>
  75. #include <linux/init.h>
  76. #include <linux/proc_fs.h>
  77. #include <linux/time.h>
  78. #include <linux/security.h>
  79. #include <linux/syscalls.h>
  80. #include <linux/audit.h>
  81. #include <linux/capability.h>
  82. #include <linux/seq_file.h>
  83. #include <linux/rwsem.h>
  84. #include <linux/nsproxy.h>
  85. #include <linux/ipc_namespace.h>
  86. #include <linux/sched/wake_q.h>
  87. #include <linux/nospec.h>
  88. #include <linux/rhashtable.h>
  89. #include <linux/uaccess.h>
  90. #include "util.h"
  91. /* One semaphore structure for each semaphore in the system. */
  92. struct sem {
  93. int semval; /* current value */
  94. /*
  95. * PID of the process that last modified the semaphore. For
  96. * Linux, specifically these are:
  97. * - semop
  98. * - semctl, via SETVAL and SETALL.
  99. * - at task exit when performing undo adjustments (see exit_sem).
  100. */
  101. struct pid *sempid;
  102. spinlock_t lock; /* spinlock for fine-grained semtimedop */
  103. struct list_head pending_alter; /* pending single-sop operations */
  104. /* that alter the semaphore */
  105. struct list_head pending_const; /* pending single-sop operations */
  106. /* that do not alter the semaphore*/
  107. time64_t sem_otime; /* candidate for sem_otime */
  108. } ____cacheline_aligned_in_smp;
  109. /* One sem_array data structure for each set of semaphores in the system. */
  110. struct sem_array {
  111. struct kern_ipc_perm sem_perm; /* permissions .. see ipc.h */
  112. time64_t sem_ctime; /* create/last semctl() time */
  113. struct list_head pending_alter; /* pending operations */
  114. /* that alter the array */
  115. struct list_head pending_const; /* pending complex operations */
  116. /* that do not alter semvals */
  117. struct list_head list_id; /* undo requests on this array */
  118. int sem_nsems; /* no. of semaphores in array */
  119. int complex_count; /* pending complex operations */
  120. unsigned int use_global_lock;/* >0: global lock required */
  121. struct sem sems[];
  122. } __randomize_layout;
  123. /* One queue for each sleeping process in the system. */
  124. struct sem_queue {
  125. struct list_head list; /* queue of pending operations */
  126. struct task_struct *sleeper; /* this process */
  127. struct sem_undo *undo; /* undo structure */
  128. struct pid *pid; /* process id of requesting process */
  129. int status; /* completion status of operation */
  130. struct sembuf *sops; /* array of pending operations */
  131. struct sembuf *blocking; /* the operation that blocked */
  132. int nsops; /* number of operations */
  133. bool alter; /* does *sops alter the array? */
  134. bool dupsop; /* sops on more than one sem_num */
  135. };
  136. /* Each task has a list of undo requests. They are executed automatically
  137. * when the process exits.
  138. */
  139. struct sem_undo {
  140. struct list_head list_proc; /* per-process list: *
  141. * all undos from one process
  142. * rcu protected */
  143. struct rcu_head rcu; /* rcu struct for sem_undo */
  144. struct sem_undo_list *ulp; /* back ptr to sem_undo_list */
  145. struct list_head list_id; /* per semaphore array list:
  146. * all undos for one array */
  147. int semid; /* semaphore set identifier */
  148. short *semadj; /* array of adjustments */
  149. /* one per semaphore */
  150. };
  151. /* sem_undo_list controls shared access to the list of sem_undo structures
  152. * that may be shared among all a CLONE_SYSVSEM task group.
  153. */
  154. struct sem_undo_list {
  155. refcount_t refcnt;
  156. spinlock_t lock;
  157. struct list_head list_proc;
  158. };
  159. #define sem_ids(ns) ((ns)->ids[IPC_SEM_IDS])
  160. static int newary(struct ipc_namespace *, struct ipc_params *);
  161. static void freeary(struct ipc_namespace *, struct kern_ipc_perm *);
  162. #ifdef CONFIG_PROC_FS
  163. static int sysvipc_sem_proc_show(struct seq_file *s, void *it);
  164. #endif
  165. #define SEMMSL_FAST 256 /* 512 bytes on stack */
  166. #define SEMOPM_FAST 64 /* ~ 372 bytes on stack */
  167. /*
  168. * Switching from the mode suitable for simple ops
  169. * to the mode for complex ops is costly. Therefore:
  170. * use some hysteresis
  171. */
  172. #define USE_GLOBAL_LOCK_HYSTERESIS 10
  173. /*
  174. * Locking:
  175. * a) global sem_lock() for read/write
  176. * sem_undo.id_next,
  177. * sem_array.complex_count,
  178. * sem_array.pending{_alter,_const},
  179. * sem_array.sem_undo
  180. *
  181. * b) global or semaphore sem_lock() for read/write:
  182. * sem_array.sems[i].pending_{const,alter}:
  183. *
  184. * c) special:
  185. * sem_undo_list.list_proc:
  186. * * undo_list->lock for write
  187. * * rcu for read
  188. * use_global_lock:
  189. * * global sem_lock() for write
  190. * * either local or global sem_lock() for read.
  191. *
  192. * Memory ordering:
  193. * Most ordering is enforced by using spin_lock() and spin_unlock().
  194. *
  195. * Exceptions:
  196. * 1) use_global_lock: (SEM_BARRIER_1)
  197. * Setting it from non-zero to 0 is a RELEASE, this is ensured by
  198. * using smp_store_release(): Immediately after setting it to 0,
  199. * a simple op can start.
  200. * Testing if it is non-zero is an ACQUIRE, this is ensured by using
  201. * smp_load_acquire().
  202. * Setting it from 0 to non-zero must be ordered with regards to
  203. * this smp_load_acquire(), this is guaranteed because the smp_load_acquire()
  204. * is inside a spin_lock() and after a write from 0 to non-zero a
  205. * spin_lock()+spin_unlock() is done.
  206. * To prevent the compiler/cpu temporarily writing 0 to use_global_lock,
  207. * READ_ONCE()/WRITE_ONCE() is used.
  208. *
  209. * 2) queue.status: (SEM_BARRIER_2)
  210. * Initialization is done while holding sem_lock(), so no further barrier is
  211. * required.
  212. * Setting it to a result code is a RELEASE, this is ensured by both a
  213. * smp_store_release() (for case a) and while holding sem_lock()
  214. * (for case b).
  215. * The ACQUIRE when reading the result code without holding sem_lock() is
  216. * achieved by using READ_ONCE() + smp_acquire__after_ctrl_dep().
  217. * (case a above).
  218. * Reading the result code while holding sem_lock() needs no further barriers,
  219. * the locks inside sem_lock() enforce ordering (case b above)
  220. *
  221. * 3) current->state:
  222. * current->state is set to TASK_INTERRUPTIBLE while holding sem_lock().
  223. * The wakeup is handled using the wake_q infrastructure. wake_q wakeups may
  224. * happen immediately after calling wake_q_add. As wake_q_add_safe() is called
  225. * when holding sem_lock(), no further barriers are required.
  226. *
  227. * See also ipc/mqueue.c for more details on the covered races.
  228. */
  229. #define sc_semmsl sem_ctls[0]
  230. #define sc_semmns sem_ctls[1]
  231. #define sc_semopm sem_ctls[2]
  232. #define sc_semmni sem_ctls[3]
  233. void sem_init_ns(struct ipc_namespace *ns)
  234. {
  235. ns->sc_semmsl = SEMMSL;
  236. ns->sc_semmns = SEMMNS;
  237. ns->sc_semopm = SEMOPM;
  238. ns->sc_semmni = SEMMNI;
  239. ns->used_sems = 0;
  240. ipc_init_ids(&ns->ids[IPC_SEM_IDS]);
  241. }
  242. #ifdef CONFIG_IPC_NS
  243. void sem_exit_ns(struct ipc_namespace *ns)
  244. {
  245. free_ipcs(ns, &sem_ids(ns), freeary);
  246. idr_destroy(&ns->ids[IPC_SEM_IDS].ipcs_idr);
  247. rhashtable_destroy(&ns->ids[IPC_SEM_IDS].key_ht);
  248. }
  249. #endif
  250. void __init sem_init(void)
  251. {
  252. sem_init_ns(&init_ipc_ns);
  253. ipc_init_proc_interface("sysvipc/sem",
  254. " key semid perms nsems uid gid cuid cgid otime ctime\n",
  255. IPC_SEM_IDS, sysvipc_sem_proc_show);
  256. }
  257. /**
  258. * unmerge_queues - unmerge queues, if possible.
  259. * @sma: semaphore array
  260. *
  261. * The function unmerges the wait queues if complex_count is 0.
  262. * It must be called prior to dropping the global semaphore array lock.
  263. */
  264. static void unmerge_queues(struct sem_array *sma)
  265. {
  266. struct sem_queue *q, *tq;
  267. /* complex operations still around? */
  268. if (sma->complex_count)
  269. return;
  270. /*
  271. * We will switch back to simple mode.
  272. * Move all pending operation back into the per-semaphore
  273. * queues.
  274. */
  275. list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {
  276. struct sem *curr;
  277. curr = &sma->sems[q->sops[0].sem_num];
  278. list_add_tail(&q->list, &curr->pending_alter);
  279. }
  280. INIT_LIST_HEAD(&sma->pending_alter);
  281. }
  282. /**
  283. * merge_queues - merge single semop queues into global queue
  284. * @sma: semaphore array
  285. *
  286. * This function merges all per-semaphore queues into the global queue.
  287. * It is necessary to achieve FIFO ordering for the pending single-sop
  288. * operations when a multi-semop operation must sleep.
  289. * Only the alter operations must be moved, the const operations can stay.
  290. */
  291. static void merge_queues(struct sem_array *sma)
  292. {
  293. int i;
  294. for (i = 0; i < sma->sem_nsems; i++) {
  295. struct sem *sem = &sma->sems[i];
  296. list_splice_init(&sem->pending_alter, &sma->pending_alter);
  297. }
  298. }
  299. static void sem_rcu_free(struct rcu_head *head)
  300. {
  301. struct kern_ipc_perm *p = container_of(head, struct kern_ipc_perm, rcu);
  302. struct sem_array *sma = container_of(p, struct sem_array, sem_perm);
  303. security_sem_free(&sma->sem_perm);
  304. kvfree(sma);
  305. }
  306. /*
  307. * Enter the mode suitable for non-simple operations:
  308. * Caller must own sem_perm.lock.
  309. */
  310. static void complexmode_enter(struct sem_array *sma)
  311. {
  312. int i;
  313. struct sem *sem;
  314. if (sma->use_global_lock > 0) {
  315. /*
  316. * We are already in global lock mode.
  317. * Nothing to do, just reset the
  318. * counter until we return to simple mode.
  319. */
  320. WRITE_ONCE(sma->use_global_lock, USE_GLOBAL_LOCK_HYSTERESIS);
  321. return;
  322. }
  323. WRITE_ONCE(sma->use_global_lock, USE_GLOBAL_LOCK_HYSTERESIS);
  324. for (i = 0; i < sma->sem_nsems; i++) {
  325. sem = &sma->sems[i];
  326. spin_lock(&sem->lock);
  327. spin_unlock(&sem->lock);
  328. }
  329. }
  330. /*
  331. * Try to leave the mode that disallows simple operations:
  332. * Caller must own sem_perm.lock.
  333. */
  334. static void complexmode_tryleave(struct sem_array *sma)
  335. {
  336. if (sma->complex_count) {
  337. /* Complex ops are sleeping.
  338. * We must stay in complex mode
  339. */
  340. return;
  341. }
  342. if (sma->use_global_lock == 1) {
  343. /* See SEM_BARRIER_1 for purpose/pairing */
  344. smp_store_release(&sma->use_global_lock, 0);
  345. } else {
  346. WRITE_ONCE(sma->use_global_lock,
  347. sma->use_global_lock-1);
  348. }
  349. }
  350. #define SEM_GLOBAL_LOCK (-1)
  351. /*
  352. * If the request contains only one semaphore operation, and there are
  353. * no complex transactions pending, lock only the semaphore involved.
  354. * Otherwise, lock the entire semaphore array, since we either have
  355. * multiple semaphores in our own semops, or we need to look at
  356. * semaphores from other pending complex operations.
  357. */
  358. static inline int sem_lock(struct sem_array *sma, struct sembuf *sops,
  359. int nsops)
  360. {
  361. struct sem *sem;
  362. int idx;
  363. if (nsops != 1) {
  364. /* Complex operation - acquire a full lock */
  365. ipc_lock_object(&sma->sem_perm);
  366. /* Prevent parallel simple ops */
  367. complexmode_enter(sma);
  368. return SEM_GLOBAL_LOCK;
  369. }
  370. /*
  371. * Only one semaphore affected - try to optimize locking.
  372. * Optimized locking is possible if no complex operation
  373. * is either enqueued or processed right now.
  374. *
  375. * Both facts are tracked by use_global_mode.
  376. */
  377. idx = array_index_nospec(sops->sem_num, sma->sem_nsems);
  378. sem = &sma->sems[idx];
  379. /*
  380. * Initial check for use_global_lock. Just an optimization,
  381. * no locking, no memory barrier.
  382. */
  383. if (!READ_ONCE(sma->use_global_lock)) {
  384. /*
  385. * It appears that no complex operation is around.
  386. * Acquire the per-semaphore lock.
  387. */
  388. spin_lock(&sem->lock);
  389. /* see SEM_BARRIER_1 for purpose/pairing */
  390. if (!smp_load_acquire(&sma->use_global_lock)) {
  391. /* fast path successful! */
  392. return sops->sem_num;
  393. }
  394. spin_unlock(&sem->lock);
  395. }
  396. /* slow path: acquire the full lock */
  397. ipc_lock_object(&sma->sem_perm);
  398. if (sma->use_global_lock == 0) {
  399. /*
  400. * The use_global_lock mode ended while we waited for
  401. * sma->sem_perm.lock. Thus we must switch to locking
  402. * with sem->lock.
  403. * Unlike in the fast path, there is no need to recheck
  404. * sma->use_global_lock after we have acquired sem->lock:
  405. * We own sma->sem_perm.lock, thus use_global_lock cannot
  406. * change.
  407. */
  408. spin_lock(&sem->lock);
  409. ipc_unlock_object(&sma->sem_perm);
  410. return sops->sem_num;
  411. } else {
  412. /*
  413. * Not a false alarm, thus continue to use the global lock
  414. * mode. No need for complexmode_enter(), this was done by
  415. * the caller that has set use_global_mode to non-zero.
  416. */
  417. return SEM_GLOBAL_LOCK;
  418. }
  419. }
  420. static inline void sem_unlock(struct sem_array *sma, int locknum)
  421. {
  422. if (locknum == SEM_GLOBAL_LOCK) {
  423. unmerge_queues(sma);
  424. complexmode_tryleave(sma);
  425. ipc_unlock_object(&sma->sem_perm);
  426. } else {
  427. struct sem *sem = &sma->sems[locknum];
  428. spin_unlock(&sem->lock);
  429. }
  430. }
  431. /*
  432. * sem_lock_(check_) routines are called in the paths where the rwsem
  433. * is not held.
  434. *
  435. * The caller holds the RCU read lock.
  436. */
  437. static inline struct sem_array *sem_obtain_object(struct ipc_namespace *ns, int id)
  438. {
  439. struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&sem_ids(ns), id);
  440. if (IS_ERR(ipcp))
  441. return ERR_CAST(ipcp);
  442. return container_of(ipcp, struct sem_array, sem_perm);
  443. }
  444. static inline struct sem_array *sem_obtain_object_check(struct ipc_namespace *ns,
  445. int id)
  446. {
  447. struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&sem_ids(ns), id);
  448. if (IS_ERR(ipcp))
  449. return ERR_CAST(ipcp);
  450. return container_of(ipcp, struct sem_array, sem_perm);
  451. }
  452. static inline void sem_lock_and_putref(struct sem_array *sma)
  453. {
  454. sem_lock(sma, NULL, -1);
  455. ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
  456. }
  457. static inline void sem_rmid(struct ipc_namespace *ns, struct sem_array *s)
  458. {
  459. ipc_rmid(&sem_ids(ns), &s->sem_perm);
  460. }
  461. static struct sem_array *sem_alloc(size_t nsems)
  462. {
  463. struct sem_array *sma;
  464. if (nsems > (INT_MAX - sizeof(*sma)) / sizeof(sma->sems[0]))
  465. return NULL;
  466. sma = kvzalloc(struct_size(sma, sems, nsems), GFP_KERNEL_ACCOUNT);
  467. if (unlikely(!sma))
  468. return NULL;
  469. return sma;
  470. }
  471. /**
  472. * newary - Create a new semaphore set
  473. * @ns: namespace
  474. * @params: ptr to the structure that contains key, semflg and nsems
  475. *
  476. * Called with sem_ids.rwsem held (as a writer)
  477. */
  478. static int newary(struct ipc_namespace *ns, struct ipc_params *params)
  479. {
  480. int retval;
  481. struct sem_array *sma;
  482. key_t key = params->key;
  483. int nsems = params->u.nsems;
  484. int semflg = params->flg;
  485. int i;
  486. if (!nsems)
  487. return -EINVAL;
  488. if (ns->used_sems + nsems > ns->sc_semmns)
  489. return -ENOSPC;
  490. sma = sem_alloc(nsems);
  491. if (!sma)
  492. return -ENOMEM;
  493. sma->sem_perm.mode = (semflg & S_IRWXUGO);
  494. sma->sem_perm.key = key;
  495. sma->sem_perm.security = NULL;
  496. retval = security_sem_alloc(&sma->sem_perm);
  497. if (retval) {
  498. kvfree(sma);
  499. return retval;
  500. }
  501. for (i = 0; i < nsems; i++) {
  502. INIT_LIST_HEAD(&sma->sems[i].pending_alter);
  503. INIT_LIST_HEAD(&sma->sems[i].pending_const);
  504. spin_lock_init(&sma->sems[i].lock);
  505. }
  506. sma->complex_count = 0;
  507. sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS;
  508. INIT_LIST_HEAD(&sma->pending_alter);
  509. INIT_LIST_HEAD(&sma->pending_const);
  510. INIT_LIST_HEAD(&sma->list_id);
  511. sma->sem_nsems = nsems;
  512. sma->sem_ctime = ktime_get_real_seconds();
  513. /* ipc_addid() locks sma upon success. */
  514. retval = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni);
  515. if (retval < 0) {
  516. ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
  517. return retval;
  518. }
  519. ns->used_sems += nsems;
  520. sem_unlock(sma, -1);
  521. rcu_read_unlock();
  522. return sma->sem_perm.id;
  523. }
  524. /*
  525. * Called with sem_ids.rwsem and ipcp locked.
  526. */
  527. static int sem_more_checks(struct kern_ipc_perm *ipcp, struct ipc_params *params)
  528. {
  529. struct sem_array *sma;
  530. sma = container_of(ipcp, struct sem_array, sem_perm);
  531. if (params->u.nsems > sma->sem_nsems)
  532. return -EINVAL;
  533. return 0;
  534. }
  535. long ksys_semget(key_t key, int nsems, int semflg)
  536. {
  537. struct ipc_namespace *ns;
  538. static const struct ipc_ops sem_ops = {
  539. .getnew = newary,
  540. .associate = security_sem_associate,
  541. .more_checks = sem_more_checks,
  542. };
  543. struct ipc_params sem_params;
  544. ns = current->nsproxy->ipc_ns;
  545. if (nsems < 0 || nsems > ns->sc_semmsl)
  546. return -EINVAL;
  547. sem_params.key = key;
  548. sem_params.flg = semflg;
  549. sem_params.u.nsems = nsems;
  550. return ipcget(ns, &sem_ids(ns), &sem_ops, &sem_params);
  551. }
  552. SYSCALL_DEFINE3(semget, key_t, key, int, nsems, int, semflg)
  553. {
  554. return ksys_semget(key, nsems, semflg);
  555. }
  556. /**
  557. * perform_atomic_semop[_slow] - Attempt to perform semaphore
  558. * operations on a given array.
  559. * @sma: semaphore array
  560. * @q: struct sem_queue that describes the operation
  561. *
  562. * Caller blocking are as follows, based the value
  563. * indicated by the semaphore operation (sem_op):
  564. *
  565. * (1) >0 never blocks.
  566. * (2) 0 (wait-for-zero operation): semval is non-zero.
  567. * (3) <0 attempting to decrement semval to a value smaller than zero.
  568. *
  569. * Returns 0 if the operation was possible.
  570. * Returns 1 if the operation is impossible, the caller must sleep.
  571. * Returns <0 for error codes.
  572. */
  573. static int perform_atomic_semop_slow(struct sem_array *sma, struct sem_queue *q)
  574. {
  575. int result, sem_op, nsops;
  576. struct pid *pid;
  577. struct sembuf *sop;
  578. struct sem *curr;
  579. struct sembuf *sops;
  580. struct sem_undo *un;
  581. sops = q->sops;
  582. nsops = q->nsops;
  583. un = q->undo;
  584. for (sop = sops; sop < sops + nsops; sop++) {
  585. int idx = array_index_nospec(sop->sem_num, sma->sem_nsems);
  586. curr = &sma->sems[idx];
  587. sem_op = sop->sem_op;
  588. result = curr->semval;
  589. if (!sem_op && result)
  590. goto would_block;
  591. result += sem_op;
  592. if (result < 0)
  593. goto would_block;
  594. if (result > SEMVMX)
  595. goto out_of_range;
  596. if (sop->sem_flg & SEM_UNDO) {
  597. int undo = un->semadj[sop->sem_num] - sem_op;
  598. /* Exceeding the undo range is an error. */
  599. if (undo < (-SEMAEM - 1) || undo > SEMAEM)
  600. goto out_of_range;
  601. un->semadj[sop->sem_num] = undo;
  602. }
  603. curr->semval = result;
  604. }
  605. sop--;
  606. pid = q->pid;
  607. while (sop >= sops) {
  608. ipc_update_pid(&sma->sems[sop->sem_num].sempid, pid);
  609. sop--;
  610. }
  611. return 0;
  612. out_of_range:
  613. result = -ERANGE;
  614. goto undo;
  615. would_block:
  616. q->blocking = sop;
  617. if (sop->sem_flg & IPC_NOWAIT)
  618. result = -EAGAIN;
  619. else
  620. result = 1;
  621. undo:
  622. sop--;
  623. while (sop >= sops) {
  624. sem_op = sop->sem_op;
  625. sma->sems[sop->sem_num].semval -= sem_op;
  626. if (sop->sem_flg & SEM_UNDO)
  627. un->semadj[sop->sem_num] += sem_op;
  628. sop--;
  629. }
  630. return result;
  631. }
  632. static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q)
  633. {
  634. int result, sem_op, nsops;
  635. struct sembuf *sop;
  636. struct sem *curr;
  637. struct sembuf *sops;
  638. struct sem_undo *un;
  639. sops = q->sops;
  640. nsops = q->nsops;
  641. un = q->undo;
  642. if (unlikely(q->dupsop))
  643. return perform_atomic_semop_slow(sma, q);
  644. /*
  645. * We scan the semaphore set twice, first to ensure that the entire
  646. * operation can succeed, therefore avoiding any pointless writes
  647. * to shared memory and having to undo such changes in order to block
  648. * until the operations can go through.
  649. */
  650. for (sop = sops; sop < sops + nsops; sop++) {
  651. int idx = array_index_nospec(sop->sem_num, sma->sem_nsems);
  652. curr = &sma->sems[idx];
  653. sem_op = sop->sem_op;
  654. result = curr->semval;
  655. if (!sem_op && result)
  656. goto would_block; /* wait-for-zero */
  657. result += sem_op;
  658. if (result < 0)
  659. goto would_block;
  660. if (result > SEMVMX)
  661. return -ERANGE;
  662. if (sop->sem_flg & SEM_UNDO) {
  663. int undo = un->semadj[sop->sem_num] - sem_op;
  664. /* Exceeding the undo range is an error. */
  665. if (undo < (-SEMAEM - 1) || undo > SEMAEM)
  666. return -ERANGE;
  667. }
  668. }
  669. for (sop = sops; sop < sops + nsops; sop++) {
  670. curr = &sma->sems[sop->sem_num];
  671. sem_op = sop->sem_op;
  672. if (sop->sem_flg & SEM_UNDO) {
  673. int undo = un->semadj[sop->sem_num] - sem_op;
  674. un->semadj[sop->sem_num] = undo;
  675. }
  676. curr->semval += sem_op;
  677. ipc_update_pid(&curr->sempid, q->pid);
  678. }
  679. return 0;
  680. would_block:
  681. q->blocking = sop;
  682. return sop->sem_flg & IPC_NOWAIT ? -EAGAIN : 1;
  683. }
  684. static inline void wake_up_sem_queue_prepare(struct sem_queue *q, int error,
  685. struct wake_q_head *wake_q)
  686. {
  687. struct task_struct *sleeper;
  688. sleeper = get_task_struct(q->sleeper);
  689. /* see SEM_BARRIER_2 for purpose/pairing */
  690. smp_store_release(&q->status, error);
  691. wake_q_add_safe(wake_q, sleeper);
  692. }
  693. static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
  694. {
  695. list_del(&q->list);
  696. if (q->nsops > 1)
  697. sma->complex_count--;
  698. }
  699. /** check_restart(sma, q)
  700. * @sma: semaphore array
  701. * @q: the operation that just completed
  702. *
  703. * update_queue is O(N^2) when it restarts scanning the whole queue of
  704. * waiting operations. Therefore this function checks if the restart is
  705. * really necessary. It is called after a previously waiting operation
  706. * modified the array.
  707. * Note that wait-for-zero operations are handled without restart.
  708. */
  709. static inline int check_restart(struct sem_array *sma, struct sem_queue *q)
  710. {
  711. /* pending complex alter operations are too difficult to analyse */
  712. if (!list_empty(&sma->pending_alter))
  713. return 1;
  714. /* we were a sleeping complex operation. Too difficult */
  715. if (q->nsops > 1)
  716. return 1;
  717. /* It is impossible that someone waits for the new value:
  718. * - complex operations always restart.
  719. * - wait-for-zero are handled separately.
  720. * - q is a previously sleeping simple operation that
  721. * altered the array. It must be a decrement, because
  722. * simple increments never sleep.
  723. * - If there are older (higher priority) decrements
  724. * in the queue, then they have observed the original
  725. * semval value and couldn't proceed. The operation
  726. * decremented to value - thus they won't proceed either.
  727. */
  728. return 0;
  729. }
  730. /**
  731. * wake_const_ops - wake up non-alter tasks
  732. * @sma: semaphore array.
  733. * @semnum: semaphore that was modified.
  734. * @wake_q: lockless wake-queue head.
  735. *
  736. * wake_const_ops must be called after a semaphore in a semaphore array
  737. * was set to 0. If complex const operations are pending, wake_const_ops must
  738. * be called with semnum = -1, as well as with the number of each modified
  739. * semaphore.
  740. * The tasks that must be woken up are added to @wake_q. The return code
  741. * is stored in q->pid.
  742. * The function returns 1 if at least one operation was completed successfully.
  743. */
  744. static int wake_const_ops(struct sem_array *sma, int semnum,
  745. struct wake_q_head *wake_q)
  746. {
  747. struct sem_queue *q, *tmp;
  748. struct list_head *pending_list;
  749. int semop_completed = 0;
  750. if (semnum == -1)
  751. pending_list = &sma->pending_const;
  752. else
  753. pending_list = &sma->sems[semnum].pending_const;
  754. list_for_each_entry_safe(q, tmp, pending_list, list) {
  755. int error = perform_atomic_semop(sma, q);
  756. if (error > 0)
  757. continue;
  758. /* operation completed, remove from queue & wakeup */
  759. unlink_queue(sma, q);
  760. wake_up_sem_queue_prepare(q, error, wake_q);
  761. if (error == 0)
  762. semop_completed = 1;
  763. }
  764. return semop_completed;
  765. }
  766. /**
  767. * do_smart_wakeup_zero - wakeup all wait for zero tasks
  768. * @sma: semaphore array
  769. * @sops: operations that were performed
  770. * @nsops: number of operations
  771. * @wake_q: lockless wake-queue head
  772. *
  773. * Checks all required queue for wait-for-zero operations, based
  774. * on the actual changes that were performed on the semaphore array.
  775. * The function returns 1 if at least one operation was completed successfully.
  776. */
  777. static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops,
  778. int nsops, struct wake_q_head *wake_q)
  779. {
  780. int i;
  781. int semop_completed = 0;
  782. int got_zero = 0;
  783. /* first: the per-semaphore queues, if known */
  784. if (sops) {
  785. for (i = 0; i < nsops; i++) {
  786. int num = sops[i].sem_num;
  787. if (sma->sems[num].semval == 0) {
  788. got_zero = 1;
  789. semop_completed |= wake_const_ops(sma, num, wake_q);
  790. }
  791. }
  792. } else {
  793. /*
  794. * No sops means modified semaphores not known.
  795. * Assume all were changed.
  796. */
  797. for (i = 0; i < sma->sem_nsems; i++) {
  798. if (sma->sems[i].semval == 0) {
  799. got_zero = 1;
  800. semop_completed |= wake_const_ops(sma, i, wake_q);
  801. }
  802. }
  803. }
  804. /*
  805. * If one of the modified semaphores got 0,
  806. * then check the global queue, too.
  807. */
  808. if (got_zero)
  809. semop_completed |= wake_const_ops(sma, -1, wake_q);
  810. return semop_completed;
  811. }
  812. /**
  813. * update_queue - look for tasks that can be completed.
  814. * @sma: semaphore array.
  815. * @semnum: semaphore that was modified.
  816. * @wake_q: lockless wake-queue head.
  817. *
  818. * update_queue must be called after a semaphore in a semaphore array
  819. * was modified. If multiple semaphores were modified, update_queue must
  820. * be called with semnum = -1, as well as with the number of each modified
  821. * semaphore.
  822. * The tasks that must be woken up are added to @wake_q. The return code
  823. * is stored in q->pid.
  824. * The function internally checks if const operations can now succeed.
  825. *
  826. * The function return 1 if at least one semop was completed successfully.
  827. */
  828. static int update_queue(struct sem_array *sma, int semnum, struct wake_q_head *wake_q)
  829. {
  830. struct sem_queue *q, *tmp;
  831. struct list_head *pending_list;
  832. int semop_completed = 0;
  833. if (semnum == -1)
  834. pending_list = &sma->pending_alter;
  835. else
  836. pending_list = &sma->sems[semnum].pending_alter;
  837. again:
  838. list_for_each_entry_safe(q, tmp, pending_list, list) {
  839. int error, restart;
  840. /* If we are scanning the single sop, per-semaphore list of
  841. * one semaphore and that semaphore is 0, then it is not
  842. * necessary to scan further: simple increments
  843. * that affect only one entry succeed immediately and cannot
  844. * be in the per semaphore pending queue, and decrements
  845. * cannot be successful if the value is already 0.
  846. */
  847. if (semnum != -1 && sma->sems[semnum].semval == 0)
  848. break;
  849. error = perform_atomic_semop(sma, q);
  850. /* Does q->sleeper still need to sleep? */
  851. if (error > 0)
  852. continue;
  853. unlink_queue(sma, q);
  854. if (error) {
  855. restart = 0;
  856. } else {
  857. semop_completed = 1;
  858. do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q);
  859. restart = check_restart(sma, q);
  860. }
  861. wake_up_sem_queue_prepare(q, error, wake_q);
  862. if (restart)
  863. goto again;
  864. }
  865. return semop_completed;
  866. }
  867. /**
  868. * set_semotime - set sem_otime
  869. * @sma: semaphore array
  870. * @sops: operations that modified the array, may be NULL
  871. *
  872. * sem_otime is replicated to avoid cache line trashing.
  873. * This function sets one instance to the current time.
  874. */
  875. static void set_semotime(struct sem_array *sma, struct sembuf *sops)
  876. {
  877. if (sops == NULL) {
  878. sma->sems[0].sem_otime = ktime_get_real_seconds();
  879. } else {
  880. sma->sems[sops[0].sem_num].sem_otime =
  881. ktime_get_real_seconds();
  882. }
  883. }
  884. /**
  885. * do_smart_update - optimized update_queue
  886. * @sma: semaphore array
  887. * @sops: operations that were performed
  888. * @nsops: number of operations
  889. * @otime: force setting otime
  890. * @wake_q: lockless wake-queue head
  891. *
  892. * do_smart_update() does the required calls to update_queue and wakeup_zero,
  893. * based on the actual changes that were performed on the semaphore array.
  894. * Note that the function does not do the actual wake-up: the caller is
  895. * responsible for calling wake_up_q().
  896. * It is safe to perform this call after dropping all locks.
  897. */
  898. static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops,
  899. int otime, struct wake_q_head *wake_q)
  900. {
  901. int i;
  902. otime |= do_smart_wakeup_zero(sma, sops, nsops, wake_q);
  903. if (!list_empty(&sma->pending_alter)) {
  904. /* semaphore array uses the global queue - just process it. */
  905. otime |= update_queue(sma, -1, wake_q);
  906. } else {
  907. if (!sops) {
  908. /*
  909. * No sops, thus the modified semaphores are not
  910. * known. Check all.
  911. */
  912. for (i = 0; i < sma->sem_nsems; i++)
  913. otime |= update_queue(sma, i, wake_q);
  914. } else {
  915. /*
  916. * Check the semaphores that were increased:
  917. * - No complex ops, thus all sleeping ops are
  918. * decrease.
  919. * - if we decreased the value, then any sleeping
  920. * semaphore ops won't be able to run: If the
  921. * previous value was too small, then the new
  922. * value will be too small, too.
  923. */
  924. for (i = 0; i < nsops; i++) {
  925. if (sops[i].sem_op > 0) {
  926. otime |= update_queue(sma,
  927. sops[i].sem_num, wake_q);
  928. }
  929. }
  930. }
  931. }
  932. if (otime)
  933. set_semotime(sma, sops);
  934. }
  935. /*
  936. * check_qop: Test if a queued operation sleeps on the semaphore semnum
  937. */
  938. static int check_qop(struct sem_array *sma, int semnum, struct sem_queue *q,
  939. bool count_zero)
  940. {
  941. struct sembuf *sop = q->blocking;
  942. /*
  943. * Linux always (since 0.99.10) reported a task as sleeping on all
  944. * semaphores. This violates SUS, therefore it was changed to the
  945. * standard compliant behavior.
  946. * Give the administrators a chance to notice that an application
  947. * might misbehave because it relies on the Linux behavior.
  948. */
  949. pr_info_once("semctl(GETNCNT/GETZCNT) is since 3.16 Single Unix Specification compliant.\n"
  950. "The task %s (%d) triggered the difference, watch for misbehavior.\n",
  951. current->comm, task_pid_nr(current));
  952. if (sop->sem_num != semnum)
  953. return 0;
  954. if (count_zero && sop->sem_op == 0)
  955. return 1;
  956. if (!count_zero && sop->sem_op < 0)
  957. return 1;
  958. return 0;
  959. }
  960. /* The following counts are associated to each semaphore:
  961. * semncnt number of tasks waiting on semval being nonzero
  962. * semzcnt number of tasks waiting on semval being zero
  963. *
  964. * Per definition, a task waits only on the semaphore of the first semop
  965. * that cannot proceed, even if additional operation would block, too.
  966. */
  967. static int count_semcnt(struct sem_array *sma, ushort semnum,
  968. bool count_zero)
  969. {
  970. struct list_head *l;
  971. struct sem_queue *q;
  972. int semcnt;
  973. semcnt = 0;
  974. /* First: check the simple operations. They are easy to evaluate */
  975. if (count_zero)
  976. l = &sma->sems[semnum].pending_const;
  977. else
  978. l = &sma->sems[semnum].pending_alter;
  979. list_for_each_entry(q, l, list) {
  980. /* all task on a per-semaphore list sleep on exactly
  981. * that semaphore
  982. */
  983. semcnt++;
  984. }
  985. /* Then: check the complex operations. */
  986. list_for_each_entry(q, &sma->pending_alter, list) {
  987. semcnt += check_qop(sma, semnum, q, count_zero);
  988. }
  989. if (count_zero) {
  990. list_for_each_entry(q, &sma->pending_const, list) {
  991. semcnt += check_qop(sma, semnum, q, count_zero);
  992. }
  993. }
  994. return semcnt;
  995. }
  996. /* Free a semaphore set. freeary() is called with sem_ids.rwsem locked
  997. * as a writer and the spinlock for this semaphore set hold. sem_ids.rwsem
  998. * remains locked on exit.
  999. */
  1000. static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
  1001. {
  1002. struct sem_undo *un, *tu;
  1003. struct sem_queue *q, *tq;
  1004. struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
  1005. int i;
  1006. DEFINE_WAKE_Q(wake_q);
  1007. /* Free the existing undo structures for this semaphore set. */
  1008. ipc_assert_locked_object(&sma->sem_perm);
  1009. list_for_each_entry_safe(un, tu, &sma->list_id, list_id) {
  1010. list_del(&un->list_id);
  1011. spin_lock(&un->ulp->lock);
  1012. un->semid = -1;
  1013. list_del_rcu(&un->list_proc);
  1014. spin_unlock(&un->ulp->lock);
  1015. kvfree_rcu(un, rcu);
  1016. }
  1017. /* Wake up all pending processes and let them fail with EIDRM. */
  1018. list_for_each_entry_safe(q, tq, &sma->pending_const, list) {
  1019. unlink_queue(sma, q);
  1020. wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
  1021. }
  1022. list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {
  1023. unlink_queue(sma, q);
  1024. wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
  1025. }
  1026. for (i = 0; i < sma->sem_nsems; i++) {
  1027. struct sem *sem = &sma->sems[i];
  1028. list_for_each_entry_safe(q, tq, &sem->pending_const, list) {
  1029. unlink_queue(sma, q);
  1030. wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
  1031. }
  1032. list_for_each_entry_safe(q, tq, &sem->pending_alter, list) {
  1033. unlink_queue(sma, q);
  1034. wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
  1035. }
  1036. ipc_update_pid(&sem->sempid, NULL);
  1037. }
  1038. /* Remove the semaphore set from the IDR */
  1039. sem_rmid(ns, sma);
  1040. sem_unlock(sma, -1);
  1041. rcu_read_unlock();
  1042. wake_up_q(&wake_q);
  1043. ns->used_sems -= sma->sem_nsems;
  1044. ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
  1045. }
  1046. static unsigned long copy_semid_to_user(void __user *buf, struct semid64_ds *in, int version)
  1047. {
  1048. switch (version) {
  1049. case IPC_64:
  1050. return copy_to_user(buf, in, sizeof(*in));
  1051. case IPC_OLD:
  1052. {
  1053. struct semid_ds out;
  1054. memset(&out, 0, sizeof(out));
  1055. ipc64_perm_to_ipc_perm(&in->sem_perm, &out.sem_perm);
  1056. out.sem_otime = in->sem_otime;
  1057. out.sem_ctime = in->sem_ctime;
  1058. out.sem_nsems = in->sem_nsems;
  1059. return copy_to_user(buf, &out, sizeof(out));
  1060. }
  1061. default:
  1062. return -EINVAL;
  1063. }
  1064. }
  1065. static time64_t get_semotime(struct sem_array *sma)
  1066. {
  1067. int i;
  1068. time64_t res;
  1069. res = sma->sems[0].sem_otime;
  1070. for (i = 1; i < sma->sem_nsems; i++) {
  1071. time64_t to = sma->sems[i].sem_otime;
  1072. if (to > res)
  1073. res = to;
  1074. }
  1075. return res;
  1076. }
  1077. static int semctl_stat(struct ipc_namespace *ns, int semid,
  1078. int cmd, struct semid64_ds *semid64)
  1079. {
  1080. struct sem_array *sma;
  1081. time64_t semotime;
  1082. int err;
  1083. memset(semid64, 0, sizeof(*semid64));
  1084. rcu_read_lock();
  1085. if (cmd == SEM_STAT || cmd == SEM_STAT_ANY) {
  1086. sma = sem_obtain_object(ns, semid);
  1087. if (IS_ERR(sma)) {
  1088. err = PTR_ERR(sma);
  1089. goto out_unlock;
  1090. }
  1091. } else { /* IPC_STAT */
  1092. sma = sem_obtain_object_check(ns, semid);
  1093. if (IS_ERR(sma)) {
  1094. err = PTR_ERR(sma);
  1095. goto out_unlock;
  1096. }
  1097. }
  1098. /* see comment for SHM_STAT_ANY */
  1099. if (cmd == SEM_STAT_ANY)
  1100. audit_ipc_obj(&sma->sem_perm);
  1101. else {
  1102. err = -EACCES;
  1103. if (ipcperms(ns, &sma->sem_perm, S_IRUGO))
  1104. goto out_unlock;
  1105. }
  1106. err = security_sem_semctl(&sma->sem_perm, cmd);
  1107. if (err)
  1108. goto out_unlock;
  1109. ipc_lock_object(&sma->sem_perm);
  1110. if (!ipc_valid_object(&sma->sem_perm)) {
  1111. ipc_unlock_object(&sma->sem_perm);
  1112. err = -EIDRM;
  1113. goto out_unlock;
  1114. }
  1115. kernel_to_ipc64_perm(&sma->sem_perm, &semid64->sem_perm);
  1116. semotime = get_semotime(sma);
  1117. semid64->sem_otime = semotime;
  1118. semid64->sem_ctime = sma->sem_ctime;
  1119. #ifndef CONFIG_64BIT
  1120. semid64->sem_otime_high = semotime >> 32;
  1121. semid64->sem_ctime_high = sma->sem_ctime >> 32;
  1122. #endif
  1123. semid64->sem_nsems = sma->sem_nsems;
  1124. if (cmd == IPC_STAT) {
  1125. /*
  1126. * As defined in SUS:
  1127. * Return 0 on success
  1128. */
  1129. err = 0;
  1130. } else {
  1131. /*
  1132. * SEM_STAT and SEM_STAT_ANY (both Linux specific)
  1133. * Return the full id, including the sequence number
  1134. */
  1135. err = sma->sem_perm.id;
  1136. }
  1137. ipc_unlock_object(&sma->sem_perm);
  1138. out_unlock:
  1139. rcu_read_unlock();
  1140. return err;
  1141. }
  1142. static int semctl_info(struct ipc_namespace *ns, int semid,
  1143. int cmd, void __user *p)
  1144. {
  1145. struct seminfo seminfo;
  1146. int max_idx;
  1147. int err;
  1148. err = security_sem_semctl(NULL, cmd);
  1149. if (err)
  1150. return err;
  1151. memset(&seminfo, 0, sizeof(seminfo));
  1152. seminfo.semmni = ns->sc_semmni;
  1153. seminfo.semmns = ns->sc_semmns;
  1154. seminfo.semmsl = ns->sc_semmsl;
  1155. seminfo.semopm = ns->sc_semopm;
  1156. seminfo.semvmx = SEMVMX;
  1157. seminfo.semmnu = SEMMNU;
  1158. seminfo.semmap = SEMMAP;
  1159. seminfo.semume = SEMUME;
  1160. down_read(&sem_ids(ns).rwsem);
  1161. if (cmd == SEM_INFO) {
  1162. seminfo.semusz = sem_ids(ns).in_use;
  1163. seminfo.semaem = ns->used_sems;
  1164. } else {
  1165. seminfo.semusz = SEMUSZ;
  1166. seminfo.semaem = SEMAEM;
  1167. }
  1168. max_idx = ipc_get_maxidx(&sem_ids(ns));
  1169. up_read(&sem_ids(ns).rwsem);
  1170. if (copy_to_user(p, &seminfo, sizeof(struct seminfo)))
  1171. return -EFAULT;
  1172. return (max_idx < 0) ? 0 : max_idx;
  1173. }
  1174. static int semctl_setval(struct ipc_namespace *ns, int semid, int semnum,
  1175. int val)
  1176. {
  1177. struct sem_undo *un;
  1178. struct sem_array *sma;
  1179. struct sem *curr;
  1180. int err;
  1181. DEFINE_WAKE_Q(wake_q);
  1182. if (val > SEMVMX || val < 0)
  1183. return -ERANGE;
  1184. rcu_read_lock();
  1185. sma = sem_obtain_object_check(ns, semid);
  1186. if (IS_ERR(sma)) {
  1187. rcu_read_unlock();
  1188. return PTR_ERR(sma);
  1189. }
  1190. if (semnum < 0 || semnum >= sma->sem_nsems) {
  1191. rcu_read_unlock();
  1192. return -EINVAL;
  1193. }
  1194. if (ipcperms(ns, &sma->sem_perm, S_IWUGO)) {
  1195. rcu_read_unlock();
  1196. return -EACCES;
  1197. }
  1198. err = security_sem_semctl(&sma->sem_perm, SETVAL);
  1199. if (err) {
  1200. rcu_read_unlock();
  1201. return -EACCES;
  1202. }
  1203. sem_lock(sma, NULL, -1);
  1204. if (!ipc_valid_object(&sma->sem_perm)) {
  1205. sem_unlock(sma, -1);
  1206. rcu_read_unlock();
  1207. return -EIDRM;
  1208. }
  1209. semnum = array_index_nospec(semnum, sma->sem_nsems);
  1210. curr = &sma->sems[semnum];
  1211. ipc_assert_locked_object(&sma->sem_perm);
  1212. list_for_each_entry(un, &sma->list_id, list_id)
  1213. un->semadj[semnum] = 0;
  1214. curr->semval = val;
  1215. ipc_update_pid(&curr->sempid, task_tgid(current));
  1216. sma->sem_ctime = ktime_get_real_seconds();
  1217. /* maybe some queued-up processes were waiting for this */
  1218. do_smart_update(sma, NULL, 0, 0, &wake_q);
  1219. sem_unlock(sma, -1);
  1220. rcu_read_unlock();
  1221. wake_up_q(&wake_q);
  1222. return 0;
  1223. }
  1224. static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
  1225. int cmd, void __user *p)
  1226. {
  1227. struct sem_array *sma;
  1228. struct sem *curr;
  1229. int err, nsems;
  1230. ushort fast_sem_io[SEMMSL_FAST];
  1231. ushort *sem_io = fast_sem_io;
  1232. DEFINE_WAKE_Q(wake_q);
  1233. rcu_read_lock();
  1234. sma = sem_obtain_object_check(ns, semid);
  1235. if (IS_ERR(sma)) {
  1236. rcu_read_unlock();
  1237. return PTR_ERR(sma);
  1238. }
  1239. nsems = sma->sem_nsems;
  1240. err = -EACCES;
  1241. if (ipcperms(ns, &sma->sem_perm, cmd == SETALL ? S_IWUGO : S_IRUGO))
  1242. goto out_rcu_wakeup;
  1243. err = security_sem_semctl(&sma->sem_perm, cmd);
  1244. if (err)
  1245. goto out_rcu_wakeup;
  1246. switch (cmd) {
  1247. case GETALL:
  1248. {
  1249. ushort __user *array = p;
  1250. int i;
  1251. sem_lock(sma, NULL, -1);
  1252. if (!ipc_valid_object(&sma->sem_perm)) {
  1253. err = -EIDRM;
  1254. goto out_unlock;
  1255. }
  1256. if (nsems > SEMMSL_FAST) {
  1257. if (!ipc_rcu_getref(&sma->sem_perm)) {
  1258. err = -EIDRM;
  1259. goto out_unlock;
  1260. }
  1261. sem_unlock(sma, -1);
  1262. rcu_read_unlock();
  1263. sem_io = kvmalloc_array(nsems, sizeof(ushort),
  1264. GFP_KERNEL);
  1265. if (sem_io == NULL) {
  1266. ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
  1267. return -ENOMEM;
  1268. }
  1269. rcu_read_lock();
  1270. sem_lock_and_putref(sma);
  1271. if (!ipc_valid_object(&sma->sem_perm)) {
  1272. err = -EIDRM;
  1273. goto out_unlock;
  1274. }
  1275. }
  1276. for (i = 0; i < sma->sem_nsems; i++)
  1277. sem_io[i] = sma->sems[i].semval;
  1278. sem_unlock(sma, -1);
  1279. rcu_read_unlock();
  1280. err = 0;
  1281. if (copy_to_user(array, sem_io, nsems*sizeof(ushort)))
  1282. err = -EFAULT;
  1283. goto out_free;
  1284. }
  1285. case SETALL:
  1286. {
  1287. int i;
  1288. struct sem_undo *un;
  1289. if (!ipc_rcu_getref(&sma->sem_perm)) {
  1290. err = -EIDRM;
  1291. goto out_rcu_wakeup;
  1292. }
  1293. rcu_read_unlock();
  1294. if (nsems > SEMMSL_FAST) {
  1295. sem_io = kvmalloc_array(nsems, sizeof(ushort),
  1296. GFP_KERNEL);
  1297. if (sem_io == NULL) {
  1298. ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
  1299. return -ENOMEM;
  1300. }
  1301. }
  1302. if (copy_from_user(sem_io, p, nsems*sizeof(ushort))) {
  1303. ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
  1304. err = -EFAULT;
  1305. goto out_free;
  1306. }
  1307. for (i = 0; i < nsems; i++) {
  1308. if (sem_io[i] > SEMVMX) {
  1309. ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
  1310. err = -ERANGE;
  1311. goto out_free;
  1312. }
  1313. }
  1314. rcu_read_lock();
  1315. sem_lock_and_putref(sma);
  1316. if (!ipc_valid_object(&sma->sem_perm)) {
  1317. err = -EIDRM;
  1318. goto out_unlock;
  1319. }
  1320. for (i = 0; i < nsems; i++) {
  1321. sma->sems[i].semval = sem_io[i];
  1322. ipc_update_pid(&sma->sems[i].sempid, task_tgid(current));
  1323. }
  1324. ipc_assert_locked_object(&sma->sem_perm);
  1325. list_for_each_entry(un, &sma->list_id, list_id) {
  1326. for (i = 0; i < nsems; i++)
  1327. un->semadj[i] = 0;
  1328. }
  1329. sma->sem_ctime = ktime_get_real_seconds();
  1330. /* maybe some queued-up processes were waiting for this */
  1331. do_smart_update(sma, NULL, 0, 0, &wake_q);
  1332. err = 0;
  1333. goto out_unlock;
  1334. }
  1335. /* GETVAL, GETPID, GETNCTN, GETZCNT: fall-through */
  1336. }
  1337. err = -EINVAL;
  1338. if (semnum < 0 || semnum >= nsems)
  1339. goto out_rcu_wakeup;
  1340. sem_lock(sma, NULL, -1);
  1341. if (!ipc_valid_object(&sma->sem_perm)) {
  1342. err = -EIDRM;
  1343. goto out_unlock;
  1344. }
  1345. semnum = array_index_nospec(semnum, nsems);
  1346. curr = &sma->sems[semnum];
  1347. switch (cmd) {
  1348. case GETVAL:
  1349. err = curr->semval;
  1350. goto out_unlock;
  1351. case GETPID:
  1352. err = pid_vnr(curr->sempid);
  1353. goto out_unlock;
  1354. case GETNCNT:
  1355. err = count_semcnt(sma, semnum, 0);
  1356. goto out_unlock;
  1357. case GETZCNT:
  1358. err = count_semcnt(sma, semnum, 1);
  1359. goto out_unlock;
  1360. }
  1361. out_unlock:
  1362. sem_unlock(sma, -1);
  1363. out_rcu_wakeup:
  1364. rcu_read_unlock();
  1365. wake_up_q(&wake_q);
  1366. out_free:
  1367. if (sem_io != fast_sem_io)
  1368. kvfree(sem_io);
  1369. return err;
  1370. }
  1371. static inline unsigned long
  1372. copy_semid_from_user(struct semid64_ds *out, void __user *buf, int version)
  1373. {
  1374. switch (version) {
  1375. case IPC_64:
  1376. if (copy_from_user(out, buf, sizeof(*out)))
  1377. return -EFAULT;
  1378. return 0;
  1379. case IPC_OLD:
  1380. {
  1381. struct semid_ds tbuf_old;
  1382. if (copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
  1383. return -EFAULT;
  1384. out->sem_perm.uid = tbuf_old.sem_perm.uid;
  1385. out->sem_perm.gid = tbuf_old.sem_perm.gid;
  1386. out->sem_perm.mode = tbuf_old.sem_perm.mode;
  1387. return 0;
  1388. }
  1389. default:
  1390. return -EINVAL;
  1391. }
  1392. }
  1393. /*
  1394. * This function handles some semctl commands which require the rwsem
  1395. * to be held in write mode.
  1396. * NOTE: no locks must be held, the rwsem is taken inside this function.
  1397. */
  1398. static int semctl_down(struct ipc_namespace *ns, int semid,
  1399. int cmd, struct semid64_ds *semid64)
  1400. {
  1401. struct sem_array *sma;
  1402. int err;
  1403. struct kern_ipc_perm *ipcp;
  1404. down_write(&sem_ids(ns).rwsem);
  1405. rcu_read_lock();
  1406. ipcp = ipcctl_obtain_check(ns, &sem_ids(ns), semid, cmd,
  1407. &semid64->sem_perm, 0);
  1408. if (IS_ERR(ipcp)) {
  1409. err = PTR_ERR(ipcp);
  1410. goto out_unlock1;
  1411. }
  1412. sma = container_of(ipcp, struct sem_array, sem_perm);
  1413. err = security_sem_semctl(&sma->sem_perm, cmd);
  1414. if (err)
  1415. goto out_unlock1;
  1416. switch (cmd) {
  1417. case IPC_RMID:
  1418. sem_lock(sma, NULL, -1);
  1419. /* freeary unlocks the ipc object and rcu */
  1420. freeary(ns, ipcp);
  1421. goto out_up;
  1422. case IPC_SET:
  1423. sem_lock(sma, NULL, -1);
  1424. err = ipc_update_perm(&semid64->sem_perm, ipcp);
  1425. if (err)
  1426. goto out_unlock0;
  1427. sma->sem_ctime = ktime_get_real_seconds();
  1428. break;
  1429. default:
  1430. err = -EINVAL;
  1431. goto out_unlock1;
  1432. }
  1433. out_unlock0:
  1434. sem_unlock(sma, -1);
  1435. out_unlock1:
  1436. rcu_read_unlock();
  1437. out_up:
  1438. up_write(&sem_ids(ns).rwsem);
  1439. return err;
  1440. }
  1441. static long ksys_semctl(int semid, int semnum, int cmd, unsigned long arg, int version)
  1442. {
  1443. struct ipc_namespace *ns;
  1444. void __user *p = (void __user *)arg;
  1445. struct semid64_ds semid64;
  1446. int err;
  1447. if (semid < 0)
  1448. return -EINVAL;
  1449. ns = current->nsproxy->ipc_ns;
  1450. switch (cmd) {
  1451. case IPC_INFO:
  1452. case SEM_INFO:
  1453. return semctl_info(ns, semid, cmd, p);
  1454. case IPC_STAT:
  1455. case SEM_STAT:
  1456. case SEM_STAT_ANY:
  1457. err = semctl_stat(ns, semid, cmd, &semid64);
  1458. if (err < 0)
  1459. return err;
  1460. if (copy_semid_to_user(p, &semid64, version))
  1461. err = -EFAULT;
  1462. return err;
  1463. case GETALL:
  1464. case GETVAL:
  1465. case GETPID:
  1466. case GETNCNT:
  1467. case GETZCNT:
  1468. case SETALL:
  1469. return semctl_main(ns, semid, semnum, cmd, p);
  1470. case SETVAL: {
  1471. int val;
  1472. #if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN)
  1473. /* big-endian 64bit */
  1474. val = arg >> 32;
  1475. #else
  1476. /* 32bit or little-endian 64bit */
  1477. val = arg;
  1478. #endif
  1479. return semctl_setval(ns, semid, semnum, val);
  1480. }
  1481. case IPC_SET:
  1482. if (copy_semid_from_user(&semid64, p, version))
  1483. return -EFAULT;
  1484. fallthrough;
  1485. case IPC_RMID:
  1486. return semctl_down(ns, semid, cmd, &semid64);
  1487. default:
  1488. return -EINVAL;
  1489. }
  1490. }
  1491. SYSCALL_DEFINE4(semctl, int, semid, int, semnum, int, cmd, unsigned long, arg)
  1492. {
  1493. return ksys_semctl(semid, semnum, cmd, arg, IPC_64);
  1494. }
  1495. #ifdef CONFIG_ARCH_WANT_IPC_PARSE_VERSION
  1496. long ksys_old_semctl(int semid, int semnum, int cmd, unsigned long arg)
  1497. {
  1498. int version = ipc_parse_version(&cmd);
  1499. return ksys_semctl(semid, semnum, cmd, arg, version);
  1500. }
  1501. SYSCALL_DEFINE4(old_semctl, int, semid, int, semnum, int, cmd, unsigned long, arg)
  1502. {
  1503. return ksys_old_semctl(semid, semnum, cmd, arg);
  1504. }
  1505. #endif
  1506. #ifdef CONFIG_COMPAT
  1507. struct compat_semid_ds {
  1508. struct compat_ipc_perm sem_perm;
  1509. old_time32_t sem_otime;
  1510. old_time32_t sem_ctime;
  1511. compat_uptr_t sem_base;
  1512. compat_uptr_t sem_pending;
  1513. compat_uptr_t sem_pending_last;
  1514. compat_uptr_t undo;
  1515. unsigned short sem_nsems;
  1516. };
  1517. static int copy_compat_semid_from_user(struct semid64_ds *out, void __user *buf,
  1518. int version)
  1519. {
  1520. memset(out, 0, sizeof(*out));
  1521. if (version == IPC_64) {
  1522. struct compat_semid64_ds __user *p = buf;
  1523. return get_compat_ipc64_perm(&out->sem_perm, &p->sem_perm);
  1524. } else {
  1525. struct compat_semid_ds __user *p = buf;
  1526. return get_compat_ipc_perm(&out->sem_perm, &p->sem_perm);
  1527. }
  1528. }
  1529. static int copy_compat_semid_to_user(void __user *buf, struct semid64_ds *in,
  1530. int version)
  1531. {
  1532. if (version == IPC_64) {
  1533. struct compat_semid64_ds v;
  1534. memset(&v, 0, sizeof(v));
  1535. to_compat_ipc64_perm(&v.sem_perm, &in->sem_perm);
  1536. v.sem_otime = lower_32_bits(in->sem_otime);
  1537. v.sem_otime_high = upper_32_bits(in->sem_otime);
  1538. v.sem_ctime = lower_32_bits(in->sem_ctime);
  1539. v.sem_ctime_high = upper_32_bits(in->sem_ctime);
  1540. v.sem_nsems = in->sem_nsems;
  1541. return copy_to_user(buf, &v, sizeof(v));
  1542. } else {
  1543. struct compat_semid_ds v;
  1544. memset(&v, 0, sizeof(v));
  1545. to_compat_ipc_perm(&v.sem_perm, &in->sem_perm);
  1546. v.sem_otime = in->sem_otime;
  1547. v.sem_ctime = in->sem_ctime;
  1548. v.sem_nsems = in->sem_nsems;
  1549. return copy_to_user(buf, &v, sizeof(v));
  1550. }
  1551. }
  1552. static long compat_ksys_semctl(int semid, int semnum, int cmd, int arg, int version)
  1553. {
  1554. void __user *p = compat_ptr(arg);
  1555. struct ipc_namespace *ns;
  1556. struct semid64_ds semid64;
  1557. int err;
  1558. ns = current->nsproxy->ipc_ns;
  1559. if (semid < 0)
  1560. return -EINVAL;
  1561. switch (cmd & (~IPC_64)) {
  1562. case IPC_INFO:
  1563. case SEM_INFO:
  1564. return semctl_info(ns, semid, cmd, p);
  1565. case IPC_STAT:
  1566. case SEM_STAT:
  1567. case SEM_STAT_ANY:
  1568. err = semctl_stat(ns, semid, cmd, &semid64);
  1569. if (err < 0)
  1570. return err;
  1571. if (copy_compat_semid_to_user(p, &semid64, version))
  1572. err = -EFAULT;
  1573. return err;
  1574. case GETVAL:
  1575. case GETPID:
  1576. case GETNCNT:
  1577. case GETZCNT:
  1578. case GETALL:
  1579. case SETALL:
  1580. return semctl_main(ns, semid, semnum, cmd, p);
  1581. case SETVAL:
  1582. return semctl_setval(ns, semid, semnum, arg);
  1583. case IPC_SET:
  1584. if (copy_compat_semid_from_user(&semid64, p, version))
  1585. return -EFAULT;
  1586. fallthrough;
  1587. case IPC_RMID:
  1588. return semctl_down(ns, semid, cmd, &semid64);
  1589. default:
  1590. return -EINVAL;
  1591. }
  1592. }
  1593. COMPAT_SYSCALL_DEFINE4(semctl, int, semid, int, semnum, int, cmd, int, arg)
  1594. {
  1595. return compat_ksys_semctl(semid, semnum, cmd, arg, IPC_64);
  1596. }
  1597. #ifdef CONFIG_ARCH_WANT_COMPAT_IPC_PARSE_VERSION
  1598. long compat_ksys_old_semctl(int semid, int semnum, int cmd, int arg)
  1599. {
  1600. int version = compat_ipc_parse_version(&cmd);
  1601. return compat_ksys_semctl(semid, semnum, cmd, arg, version);
  1602. }
  1603. COMPAT_SYSCALL_DEFINE4(old_semctl, int, semid, int, semnum, int, cmd, int, arg)
  1604. {
  1605. return compat_ksys_old_semctl(semid, semnum, cmd, arg);
  1606. }
  1607. #endif
  1608. #endif
  1609. /* If the task doesn't already have a undo_list, then allocate one
  1610. * here. We guarantee there is only one thread using this undo list,
  1611. * and current is THE ONE
  1612. *
  1613. * If this allocation and assignment succeeds, but later
  1614. * portions of this code fail, there is no need to free the sem_undo_list.
  1615. * Just let it stay associated with the task, and it'll be freed later
  1616. * at exit time.
  1617. *
  1618. * This can block, so callers must hold no locks.
  1619. */
  1620. static inline int get_undo_list(struct sem_undo_list **undo_listp)
  1621. {
  1622. struct sem_undo_list *undo_list;
  1623. undo_list = current->sysvsem.undo_list;
  1624. if (!undo_list) {
  1625. undo_list = kzalloc(sizeof(*undo_list), GFP_KERNEL_ACCOUNT);
  1626. if (undo_list == NULL)
  1627. return -ENOMEM;
  1628. spin_lock_init(&undo_list->lock);
  1629. refcount_set(&undo_list->refcnt, 1);
  1630. INIT_LIST_HEAD(&undo_list->list_proc);
  1631. current->sysvsem.undo_list = undo_list;
  1632. }
  1633. *undo_listp = undo_list;
  1634. return 0;
  1635. }
  1636. static struct sem_undo *__lookup_undo(struct sem_undo_list *ulp, int semid)
  1637. {
  1638. struct sem_undo *un;
  1639. list_for_each_entry_rcu(un, &ulp->list_proc, list_proc,
  1640. spin_is_locked(&ulp->lock)) {
  1641. if (un->semid == semid)
  1642. return un;
  1643. }
  1644. return NULL;
  1645. }
  1646. static struct sem_undo *lookup_undo(struct sem_undo_list *ulp, int semid)
  1647. {
  1648. struct sem_undo *un;
  1649. assert_spin_locked(&ulp->lock);
  1650. un = __lookup_undo(ulp, semid);
  1651. if (un) {
  1652. list_del_rcu(&un->list_proc);
  1653. list_add_rcu(&un->list_proc, &ulp->list_proc);
  1654. }
  1655. return un;
  1656. }
  1657. /**
  1658. * find_alloc_undo - lookup (and if not present create) undo array
  1659. * @ns: namespace
  1660. * @semid: semaphore array id
  1661. *
  1662. * The function looks up (and if not present creates) the undo structure.
  1663. * The size of the undo structure depends on the size of the semaphore
  1664. * array, thus the alloc path is not that straightforward.
  1665. * Lifetime-rules: sem_undo is rcu-protected, on success, the function
  1666. * performs a rcu_read_lock().
  1667. */
  1668. static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
  1669. {
  1670. struct sem_array *sma;
  1671. struct sem_undo_list *ulp;
  1672. struct sem_undo *un, *new;
  1673. int nsems, error;
  1674. error = get_undo_list(&ulp);
  1675. if (error)
  1676. return ERR_PTR(error);
  1677. rcu_read_lock();
  1678. spin_lock(&ulp->lock);
  1679. un = lookup_undo(ulp, semid);
  1680. spin_unlock(&ulp->lock);
  1681. if (likely(un != NULL))
  1682. goto out;
  1683. /* no undo structure around - allocate one. */
  1684. /* step 1: figure out the size of the semaphore array */
  1685. sma = sem_obtain_object_check(ns, semid);
  1686. if (IS_ERR(sma)) {
  1687. rcu_read_unlock();
  1688. return ERR_CAST(sma);
  1689. }
  1690. nsems = sma->sem_nsems;
  1691. if (!ipc_rcu_getref(&sma->sem_perm)) {
  1692. rcu_read_unlock();
  1693. un = ERR_PTR(-EIDRM);
  1694. goto out;
  1695. }
  1696. rcu_read_unlock();
  1697. /* step 2: allocate new undo structure */
  1698. new = kvzalloc(sizeof(struct sem_undo) + sizeof(short)*nsems,
  1699. GFP_KERNEL_ACCOUNT);
  1700. if (!new) {
  1701. ipc_rcu_putref(&sma->sem_perm, sem_rcu_free);
  1702. return ERR_PTR(-ENOMEM);
  1703. }
  1704. /* step 3: Acquire the lock on semaphore array */
  1705. rcu_read_lock();
  1706. sem_lock_and_putref(sma);
  1707. if (!ipc_valid_object(&sma->sem_perm)) {
  1708. sem_unlock(sma, -1);
  1709. rcu_read_unlock();
  1710. kvfree(new);
  1711. un = ERR_PTR(-EIDRM);
  1712. goto out;
  1713. }
  1714. spin_lock(&ulp->lock);
  1715. /*
  1716. * step 4: check for races: did someone else allocate the undo struct?
  1717. */
  1718. un = lookup_undo(ulp, semid);
  1719. if (un) {
  1720. spin_unlock(&ulp->lock);
  1721. kvfree(new);
  1722. goto success;
  1723. }
  1724. /* step 5: initialize & link new undo structure */
  1725. new->semadj = (short *) &new[1];
  1726. new->ulp = ulp;
  1727. new->semid = semid;
  1728. assert_spin_locked(&ulp->lock);
  1729. list_add_rcu(&new->list_proc, &ulp->list_proc);
  1730. ipc_assert_locked_object(&sma->sem_perm);
  1731. list_add(&new->list_id, &sma->list_id);
  1732. un = new;
  1733. spin_unlock(&ulp->lock);
  1734. success:
  1735. sem_unlock(sma, -1);
  1736. out:
  1737. return un;
  1738. }
  1739. long __do_semtimedop(int semid, struct sembuf *sops,
  1740. unsigned nsops, const struct timespec64 *timeout,
  1741. struct ipc_namespace *ns)
  1742. {
  1743. int error = -EINVAL;
  1744. struct sem_array *sma;
  1745. struct sembuf *sop;
  1746. struct sem_undo *un;
  1747. int max, locknum;
  1748. bool undos = false, alter = false, dupsop = false;
  1749. struct sem_queue queue;
  1750. unsigned long dup = 0;
  1751. ktime_t expires, *exp = NULL;
  1752. bool timed_out = false;
  1753. if (nsops < 1 || semid < 0)
  1754. return -EINVAL;
  1755. if (nsops > ns->sc_semopm)
  1756. return -E2BIG;
  1757. if (timeout) {
  1758. if (!timespec64_valid(timeout))
  1759. return -EINVAL;
  1760. expires = ktime_add_safe(ktime_get(),
  1761. timespec64_to_ktime(*timeout));
  1762. exp = &expires;
  1763. }
  1764. max = 0;
  1765. for (sop = sops; sop < sops + nsops; sop++) {
  1766. unsigned long mask = 1ULL << ((sop->sem_num) % BITS_PER_LONG);
  1767. if (sop->sem_num >= max)
  1768. max = sop->sem_num;
  1769. if (sop->sem_flg & SEM_UNDO)
  1770. undos = true;
  1771. if (dup & mask) {
  1772. /*
  1773. * There was a previous alter access that appears
  1774. * to have accessed the same semaphore, thus use
  1775. * the dupsop logic. "appears", because the detection
  1776. * can only check % BITS_PER_LONG.
  1777. */
  1778. dupsop = true;
  1779. }
  1780. if (sop->sem_op != 0) {
  1781. alter = true;
  1782. dup |= mask;
  1783. }
  1784. }
  1785. if (undos) {
  1786. /* On success, find_alloc_undo takes the rcu_read_lock */
  1787. un = find_alloc_undo(ns, semid);
  1788. if (IS_ERR(un)) {
  1789. error = PTR_ERR(un);
  1790. goto out;
  1791. }
  1792. } else {
  1793. un = NULL;
  1794. rcu_read_lock();
  1795. }
  1796. sma = sem_obtain_object_check(ns, semid);
  1797. if (IS_ERR(sma)) {
  1798. rcu_read_unlock();
  1799. error = PTR_ERR(sma);
  1800. goto out;
  1801. }
  1802. error = -EFBIG;
  1803. if (max >= sma->sem_nsems) {
  1804. rcu_read_unlock();
  1805. goto out;
  1806. }
  1807. error = -EACCES;
  1808. if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO)) {
  1809. rcu_read_unlock();
  1810. goto out;
  1811. }
  1812. error = security_sem_semop(&sma->sem_perm, sops, nsops, alter);
  1813. if (error) {
  1814. rcu_read_unlock();
  1815. goto out;
  1816. }
  1817. error = -EIDRM;
  1818. locknum = sem_lock(sma, sops, nsops);
  1819. /*
  1820. * We eventually might perform the following check in a lockless
  1821. * fashion, considering ipc_valid_object() locking constraints.
  1822. * If nsops == 1 and there is no contention for sem_perm.lock, then
  1823. * only a per-semaphore lock is held and it's OK to proceed with the
  1824. * check below. More details on the fine grained locking scheme
  1825. * entangled here and why it's RMID race safe on comments at sem_lock()
  1826. */
  1827. if (!ipc_valid_object(&sma->sem_perm))
  1828. goto out_unlock;
  1829. /*
  1830. * semid identifiers are not unique - find_alloc_undo may have
  1831. * allocated an undo structure, it was invalidated by an RMID
  1832. * and now a new array with received the same id. Check and fail.
  1833. * This case can be detected checking un->semid. The existence of
  1834. * "un" itself is guaranteed by rcu.
  1835. */
  1836. if (un && un->semid == -1)
  1837. goto out_unlock;
  1838. queue.sops = sops;
  1839. queue.nsops = nsops;
  1840. queue.undo = un;
  1841. queue.pid = task_tgid(current);
  1842. queue.alter = alter;
  1843. queue.dupsop = dupsop;
  1844. error = perform_atomic_semop(sma, &queue);
  1845. if (error == 0) { /* non-blocking successful path */
  1846. DEFINE_WAKE_Q(wake_q);
  1847. /*
  1848. * If the operation was successful, then do
  1849. * the required updates.
  1850. */
  1851. if (alter)
  1852. do_smart_update(sma, sops, nsops, 1, &wake_q);
  1853. else
  1854. set_semotime(sma, sops);
  1855. sem_unlock(sma, locknum);
  1856. rcu_read_unlock();
  1857. wake_up_q(&wake_q);
  1858. goto out;
  1859. }
  1860. if (error < 0) /* non-blocking error path */
  1861. goto out_unlock;
  1862. /*
  1863. * We need to sleep on this operation, so we put the current
  1864. * task into the pending queue and go to sleep.
  1865. */
  1866. if (nsops == 1) {
  1867. struct sem *curr;
  1868. int idx = array_index_nospec(sops->sem_num, sma->sem_nsems);
  1869. curr = &sma->sems[idx];
  1870. if (alter) {
  1871. if (sma->complex_count) {
  1872. list_add_tail(&queue.list,
  1873. &sma->pending_alter);
  1874. } else {
  1875. list_add_tail(&queue.list,
  1876. &curr->pending_alter);
  1877. }
  1878. } else {
  1879. list_add_tail(&queue.list, &curr->pending_const);
  1880. }
  1881. } else {
  1882. if (!sma->complex_count)
  1883. merge_queues(sma);
  1884. if (alter)
  1885. list_add_tail(&queue.list, &sma->pending_alter);
  1886. else
  1887. list_add_tail(&queue.list, &sma->pending_const);
  1888. sma->complex_count++;
  1889. }
  1890. do {
  1891. /* memory ordering ensured by the lock in sem_lock() */
  1892. WRITE_ONCE(queue.status, -EINTR);
  1893. queue.sleeper = current;
  1894. /* memory ordering is ensured by the lock in sem_lock() */
  1895. __set_current_state(TASK_INTERRUPTIBLE);
  1896. sem_unlock(sma, locknum);
  1897. rcu_read_unlock();
  1898. timed_out = !schedule_hrtimeout_range(exp,
  1899. current->timer_slack_ns, HRTIMER_MODE_ABS);
  1900. /*
  1901. * fastpath: the semop has completed, either successfully or
  1902. * not, from the syscall pov, is quite irrelevant to us at this
  1903. * point; we're done.
  1904. *
  1905. * We _do_ care, nonetheless, about being awoken by a signal or
  1906. * spuriously. The queue.status is checked again in the
  1907. * slowpath (aka after taking sem_lock), such that we can detect
  1908. * scenarios where we were awakened externally, during the
  1909. * window between wake_q_add() and wake_up_q().
  1910. */
  1911. rcu_read_lock();
  1912. error = READ_ONCE(queue.status);
  1913. if (error != -EINTR) {
  1914. /* see SEM_BARRIER_2 for purpose/pairing */
  1915. smp_acquire__after_ctrl_dep();
  1916. rcu_read_unlock();
  1917. goto out;
  1918. }
  1919. locknum = sem_lock(sma, sops, nsops);
  1920. if (!ipc_valid_object(&sma->sem_perm))
  1921. goto out_unlock;
  1922. /*
  1923. * No necessity for any barrier: We are protect by sem_lock()
  1924. */
  1925. error = READ_ONCE(queue.status);
  1926. /*
  1927. * If queue.status != -EINTR we are woken up by another process.
  1928. * Leave without unlink_queue(), but with sem_unlock().
  1929. */
  1930. if (error != -EINTR)
  1931. goto out_unlock;
  1932. /*
  1933. * If an interrupt occurred we have to clean up the queue.
  1934. */
  1935. if (timed_out)
  1936. error = -EAGAIN;
  1937. } while (error == -EINTR && !signal_pending(current)); /* spurious */
  1938. unlink_queue(sma, &queue);
  1939. out_unlock:
  1940. sem_unlock(sma, locknum);
  1941. rcu_read_unlock();
  1942. out:
  1943. return error;
  1944. }
  1945. static long do_semtimedop(int semid, struct sembuf __user *tsops,
  1946. unsigned nsops, const struct timespec64 *timeout)
  1947. {
  1948. struct sembuf fast_sops[SEMOPM_FAST];
  1949. struct sembuf *sops = fast_sops;
  1950. struct ipc_namespace *ns;
  1951. int ret;
  1952. ns = current->nsproxy->ipc_ns;
  1953. if (nsops > ns->sc_semopm)
  1954. return -E2BIG;
  1955. if (nsops < 1)
  1956. return -EINVAL;
  1957. if (nsops > SEMOPM_FAST) {
  1958. sops = kvmalloc_array(nsops, sizeof(*sops), GFP_KERNEL);
  1959. if (sops == NULL)
  1960. return -ENOMEM;
  1961. }
  1962. if (copy_from_user(sops, tsops, nsops * sizeof(*tsops))) {
  1963. ret = -EFAULT;
  1964. goto out_free;
  1965. }
  1966. ret = __do_semtimedop(semid, sops, nsops, timeout, ns);
  1967. out_free:
  1968. if (sops != fast_sops)
  1969. kvfree(sops);
  1970. return ret;
  1971. }
  1972. long ksys_semtimedop(int semid, struct sembuf __user *tsops,
  1973. unsigned int nsops, const struct __kernel_timespec __user *timeout)
  1974. {
  1975. if (timeout) {
  1976. struct timespec64 ts;
  1977. if (get_timespec64(&ts, timeout))
  1978. return -EFAULT;
  1979. return do_semtimedop(semid, tsops, nsops, &ts);
  1980. }
  1981. return do_semtimedop(semid, tsops, nsops, NULL);
  1982. }
  1983. SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
  1984. unsigned int, nsops, const struct __kernel_timespec __user *, timeout)
  1985. {
  1986. return ksys_semtimedop(semid, tsops, nsops, timeout);
  1987. }
  1988. #ifdef CONFIG_COMPAT_32BIT_TIME
  1989. long compat_ksys_semtimedop(int semid, struct sembuf __user *tsems,
  1990. unsigned int nsops,
  1991. const struct old_timespec32 __user *timeout)
  1992. {
  1993. if (timeout) {
  1994. struct timespec64 ts;
  1995. if (get_old_timespec32(&ts, timeout))
  1996. return -EFAULT;
  1997. return do_semtimedop(semid, tsems, nsops, &ts);
  1998. }
  1999. return do_semtimedop(semid, tsems, nsops, NULL);
  2000. }
  2001. SYSCALL_DEFINE4(semtimedop_time32, int, semid, struct sembuf __user *, tsems,
  2002. unsigned int, nsops,
  2003. const struct old_timespec32 __user *, timeout)
  2004. {
  2005. return compat_ksys_semtimedop(semid, tsems, nsops, timeout);
  2006. }
  2007. #endif
  2008. SYSCALL_DEFINE3(semop, int, semid, struct sembuf __user *, tsops,
  2009. unsigned, nsops)
  2010. {
  2011. return do_semtimedop(semid, tsops, nsops, NULL);
  2012. }
  2013. /* If CLONE_SYSVSEM is set, establish sharing of SEM_UNDO state between
  2014. * parent and child tasks.
  2015. */
  2016. int copy_semundo(unsigned long clone_flags, struct task_struct *tsk)
  2017. {
  2018. struct sem_undo_list *undo_list;
  2019. int error;
  2020. if (clone_flags & CLONE_SYSVSEM) {
  2021. error = get_undo_list(&undo_list);
  2022. if (error)
  2023. return error;
  2024. refcount_inc(&undo_list->refcnt);
  2025. tsk->sysvsem.undo_list = undo_list;
  2026. } else
  2027. tsk->sysvsem.undo_list = NULL;
  2028. return 0;
  2029. }
  2030. /*
  2031. * add semadj values to semaphores, free undo structures.
  2032. * undo structures are not freed when semaphore arrays are destroyed
  2033. * so some of them may be out of date.
  2034. * IMPLEMENTATION NOTE: There is some confusion over whether the
  2035. * set of adjustments that needs to be done should be done in an atomic
  2036. * manner or not. That is, if we are attempting to decrement the semval
  2037. * should we queue up and wait until we can do so legally?
  2038. * The original implementation attempted to do this (queue and wait).
  2039. * The current implementation does not do so. The POSIX standard
  2040. * and SVID should be consulted to determine what behavior is mandated.
  2041. */
  2042. void exit_sem(struct task_struct *tsk)
  2043. {
  2044. struct sem_undo_list *ulp;
  2045. ulp = tsk->sysvsem.undo_list;
  2046. if (!ulp)
  2047. return;
  2048. tsk->sysvsem.undo_list = NULL;
  2049. if (!refcount_dec_and_test(&ulp->refcnt))
  2050. return;
  2051. for (;;) {
  2052. struct sem_array *sma;
  2053. struct sem_undo *un;
  2054. int semid, i;
  2055. DEFINE_WAKE_Q(wake_q);
  2056. cond_resched();
  2057. rcu_read_lock();
  2058. un = list_entry_rcu(ulp->list_proc.next,
  2059. struct sem_undo, list_proc);
  2060. if (&un->list_proc == &ulp->list_proc) {
  2061. /*
  2062. * We must wait for freeary() before freeing this ulp,
  2063. * in case we raced with last sem_undo. There is a small
  2064. * possibility where we exit while freeary() didn't
  2065. * finish unlocking sem_undo_list.
  2066. */
  2067. spin_lock(&ulp->lock);
  2068. spin_unlock(&ulp->lock);
  2069. rcu_read_unlock();
  2070. break;
  2071. }
  2072. spin_lock(&ulp->lock);
  2073. semid = un->semid;
  2074. spin_unlock(&ulp->lock);
  2075. /* exit_sem raced with IPC_RMID, nothing to do */
  2076. if (semid == -1) {
  2077. rcu_read_unlock();
  2078. continue;
  2079. }
  2080. sma = sem_obtain_object_check(tsk->nsproxy->ipc_ns, semid);
  2081. /* exit_sem raced with IPC_RMID, nothing to do */
  2082. if (IS_ERR(sma)) {
  2083. rcu_read_unlock();
  2084. continue;
  2085. }
  2086. sem_lock(sma, NULL, -1);
  2087. /* exit_sem raced with IPC_RMID, nothing to do */
  2088. if (!ipc_valid_object(&sma->sem_perm)) {
  2089. sem_unlock(sma, -1);
  2090. rcu_read_unlock();
  2091. continue;
  2092. }
  2093. un = __lookup_undo(ulp, semid);
  2094. if (un == NULL) {
  2095. /* exit_sem raced with IPC_RMID+semget() that created
  2096. * exactly the same semid. Nothing to do.
  2097. */
  2098. sem_unlock(sma, -1);
  2099. rcu_read_unlock();
  2100. continue;
  2101. }
  2102. /* remove un from the linked lists */
  2103. ipc_assert_locked_object(&sma->sem_perm);
  2104. list_del(&un->list_id);
  2105. spin_lock(&ulp->lock);
  2106. list_del_rcu(&un->list_proc);
  2107. spin_unlock(&ulp->lock);
  2108. /* perform adjustments registered in un */
  2109. for (i = 0; i < sma->sem_nsems; i++) {
  2110. struct sem *semaphore = &sma->sems[i];
  2111. if (un->semadj[i]) {
  2112. semaphore->semval += un->semadj[i];
  2113. /*
  2114. * Range checks of the new semaphore value,
  2115. * not defined by sus:
  2116. * - Some unices ignore the undo entirely
  2117. * (e.g. HP UX 11i 11.22, Tru64 V5.1)
  2118. * - some cap the value (e.g. FreeBSD caps
  2119. * at 0, but doesn't enforce SEMVMX)
  2120. *
  2121. * Linux caps the semaphore value, both at 0
  2122. * and at SEMVMX.
  2123. *
  2124. * Manfred <[email protected]>
  2125. */
  2126. if (semaphore->semval < 0)
  2127. semaphore->semval = 0;
  2128. if (semaphore->semval > SEMVMX)
  2129. semaphore->semval = SEMVMX;
  2130. ipc_update_pid(&semaphore->sempid, task_tgid(current));
  2131. }
  2132. }
  2133. /* maybe some queued-up processes were waiting for this */
  2134. do_smart_update(sma, NULL, 0, 1, &wake_q);
  2135. sem_unlock(sma, -1);
  2136. rcu_read_unlock();
  2137. wake_up_q(&wake_q);
  2138. kvfree_rcu(un, rcu);
  2139. }
  2140. kfree(ulp);
  2141. }
  2142. #ifdef CONFIG_PROC_FS
  2143. static int sysvipc_sem_proc_show(struct seq_file *s, void *it)
  2144. {
  2145. struct user_namespace *user_ns = seq_user_ns(s);
  2146. struct kern_ipc_perm *ipcp = it;
  2147. struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
  2148. time64_t sem_otime;
  2149. /*
  2150. * The proc interface isn't aware of sem_lock(), it calls
  2151. * ipc_lock_object(), i.e. spin_lock(&sma->sem_perm.lock).
  2152. * (in sysvipc_find_ipc)
  2153. * In order to stay compatible with sem_lock(), we must
  2154. * enter / leave complex_mode.
  2155. */
  2156. complexmode_enter(sma);
  2157. sem_otime = get_semotime(sma);
  2158. seq_printf(s,
  2159. "%10d %10d %4o %10u %5u %5u %5u %5u %10llu %10llu\n",
  2160. sma->sem_perm.key,
  2161. sma->sem_perm.id,
  2162. sma->sem_perm.mode,
  2163. sma->sem_nsems,
  2164. from_kuid_munged(user_ns, sma->sem_perm.uid),
  2165. from_kgid_munged(user_ns, sma->sem_perm.gid),
  2166. from_kuid_munged(user_ns, sma->sem_perm.cuid),
  2167. from_kgid_munged(user_ns, sma->sem_perm.cgid),
  2168. sem_otime,
  2169. sma->sem_ctime);
  2170. complexmode_tryleave(sma);
  2171. return 0;
  2172. }
  2173. #endif