From 590a98d5d108653cb4ebdf61ec73b2d31a1f7b81 Mon Sep 17 00:00:00 2001 From: Hangyu Hua Date: Sat, 1 Jan 2022 01:21:38 +0800 Subject: [PATCH 001/156] UPSTREAM: usb: gadget: clear related members when goto fail commit 501e38a5531efbd77d5c73c0ba838a889bfc1d74 upstream. dev->config and dev->hs_config and dev->dev need to be cleaned if dev_config fails to avoid UAF. Bug: 220261709 Acked-by: Alan Stern Signed-off-by: Hangyu Hua Link: https://lore.kernel.org/r/20211231172138.7993-3-hbh25y@gmail.com Signed-off-by: Greg Kroah-Hartman Signed-off-by: Lee Jones Change-Id: I149a16bc8db7262c3ab9c2f72a0f10c6caebee83 --- drivers/usb/gadget/legacy/inode.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c index 217d2b66fa51..523d05e5dca5 100644 --- a/drivers/usb/gadget/legacy/inode.c +++ b/drivers/usb/gadget/legacy/inode.c @@ -1876,8 +1876,8 @@ dev_config (struct file *fd, const char __user *buf, size_t len, loff_t *ptr) value = usb_gadget_probe_driver(&gadgetfs_driver); if (value != 0) { - kfree (dev->buf); - dev->buf = NULL; + spin_lock_irq(&dev->lock); + goto fail; } else { /* at this point "good" hardware has for the first time * let the USB the host see us. alternatively, if users @@ -1894,6 +1894,9 @@ dev_config (struct file *fd, const char __user *buf, size_t len, loff_t *ptr) return value; fail: + dev->config = NULL; + dev->hs_config = NULL; + dev->dev = NULL; spin_unlock_irq (&dev->lock); pr_debug ("%s: %s fail %zd, %p\n", shortname, __func__, value, dev); kfree (dev->buf); From ff0000fe82f45cee26c83abb40b01e43c52739e9 Mon Sep 17 00:00:00 2001 From: Hangyu Hua Date: Sat, 1 Jan 2022 01:21:37 +0800 Subject: [PATCH 002/156] UPSTREAM: usb: gadget: don't release an existing dev->buf commit 89f3594d0de58e8a57d92d497dea9fee3d4b9cda upstream. dev->buf does not need to be released if it already exists before executing dev_config. Bug: 220261709 Acked-by: Alan Stern Signed-off-by: Hangyu Hua Link: https://lore.kernel.org/r/20211231172138.7993-2-hbh25y@gmail.com Signed-off-by: Greg Kroah-Hartman Signed-off-by: Lee Jones Change-Id: Id53d6770fbae0a7fcf0fa136157c0ab34fb5da64 --- drivers/usb/gadget/legacy/inode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c index 523d05e5dca5..454860d52ce7 100644 --- a/drivers/usb/gadget/legacy/inode.c +++ b/drivers/usb/gadget/legacy/inode.c @@ -1828,8 +1828,9 @@ dev_config (struct file *fd, const char __user *buf, size_t len, loff_t *ptr) spin_lock_irq (&dev->lock); value = -EINVAL; if (dev->buf) { + spin_unlock_irq(&dev->lock); kfree(kbuf); - goto fail; + return value; } dev->buf = kbuf; From 87c1f135bf8c5ba15f5258ee139518a502bd7507 Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Thu, 3 Mar 2022 15:55:10 +0100 Subject: [PATCH 003/156] UPSTREAM: Revert "xfrm: state and policy should fail if XFRMA_IF_ID 0" commit a3d9001b4e287fc043e5539d03d71a32ab114bcb upstream. This reverts commit 68ac0f3810e76a853b5f7b90601a05c3048b8b54 because ID 0 was meant to be used for configuring the policy/state without matching for a specific interface (e.g., Cilium is affected, see https://github.com/cilium/cilium/pull/18789 and https://github.com/cilium/cilium/pull/19019). Bug: 220227367 Signed-off-by: Kai Lueke Signed-off-by: Steffen Klassert Signed-off-by: Greg Kroah-Hartman (cherry picked from commit bdf0316982f00010d6e56f1379a51cd0568d51cd) Signed-off-by: Greg Kroah-Hartman Change-Id: I0d6c17e14d562cef2a4d76b7c4299cf0dd3641d1 --- net/xfrm/xfrm_user.c | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index d1b6beeb1399..ff1728234968 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -621,13 +621,8 @@ static struct xfrm_state *xfrm_state_construct(struct net *net, xfrm_smark_init(attrs, &x->props.smark); - if (attrs[XFRMA_IF_ID]) { + if (attrs[XFRMA_IF_ID]) x->if_id = nla_get_u32(attrs[XFRMA_IF_ID]); - if (!x->if_id) { - err = -EINVAL; - goto error; - } - } err = __xfrm_init_state(x, false, attrs[XFRMA_OFFLOAD_DEV]); if (err) @@ -1358,13 +1353,8 @@ static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh, mark = xfrm_mark_get(attrs, &m); - if (attrs[XFRMA_IF_ID]) { + if (attrs[XFRMA_IF_ID]) if_id = nla_get_u32(attrs[XFRMA_IF_ID]); - if (!if_id) { - err = -EINVAL; - goto out_noput; - } - } if (p->info.seq) { x = xfrm_find_acq_byseq(net, mark, p->info.seq); @@ -1677,13 +1667,8 @@ static struct xfrm_policy *xfrm_policy_construct(struct net *net, struct xfrm_us xfrm_mark_get(attrs, &xp->mark); - if (attrs[XFRMA_IF_ID]) { + if (attrs[XFRMA_IF_ID]) xp->if_id = nla_get_u32(attrs[XFRMA_IF_ID]); - if (!xp->if_id) { - err = -EINVAL; - goto error; - } - } return xp; error: From 6282531a84bc383f696278ef90fb3d3e1593a93b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 11 Apr 2021 01:46:40 +0100 Subject: [PATCH 004/156] UPSTREAM: io_uring: return back safer resurrect commit f70865db5ff35f5ed0c7e9ef63e7cca3d4947f04 upstream. Revert of revert of "io_uring: wait potential ->release() on resurrect", which adds a helper for resurrect not racing completion reinit, as was removed because of a strange bug with no clear root or link to the patch. Was improved, instead of rcu_synchronize(), just wait_for_completion() because we're at 0 refs and it will happen very shortly. Specifically use non-interruptible version to ignore all pending signals that may have ended prior interruptible wait. This reverts commit cb5e1b81304e089ee3ca948db4d29f71902eb575. Bug: 222091980 Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/7a080c20f686d026efade810b116b72f88abaff9.1618101759.git.asml.silence@gmail.com Signed-off-by: Jens Axboe Cc: Lee Jones Signed-off-by: Greg Kroah-Hartman Signed-off-by: Lee Jones Change-Id: If9b7b96b7cdc777640b1b5aa20d148bdad54954a --- fs/io_uring.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 104dff9c7131..b258ba20571d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1009,6 +1009,18 @@ static inline bool __io_match_files(struct io_kiocb *req, req->work.identity->files == files; } +static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl) +{ + bool got = percpu_ref_tryget(ref); + + /* already at zero, wait for ->release() */ + if (!got) + wait_for_completion(compl); + percpu_ref_resurrect(ref); + if (got) + percpu_ref_put(ref); +} + static bool io_match_task(struct io_kiocb *head, struct task_struct *task, struct files_struct *files) @@ -9756,12 +9768,11 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, if (ret < 0) break; } while (1); - mutex_lock(&ctx->uring_lock); if (ret) { - percpu_ref_resurrect(&ctx->refs); - goto out_quiesce; + io_refs_resurrect(&ctx->refs, &ctx->ref_comp); + return ret; } } @@ -9854,7 +9865,6 @@ out: if (io_register_op_must_quiesce(opcode)) { /* bring the ctx back to life */ percpu_ref_reinit(&ctx->refs); -out_quiesce: reinit_completion(&ctx->ref_comp); } return ret; From e7f39d0aa294f998ddd55c5becb5e9601912da0d Mon Sep 17 00:00:00 2001 From: Oliver Neukum Date: Thu, 17 Feb 2022 14:10:44 +0100 Subject: [PATCH 005/156] UPSTREAM: sr9700: sanity check for packet length [ Upstream commit e9da0b56fe27206b49f39805f7dcda8a89379062 ] A malicious device can leak heap data to user space providing bogus frame lengths. Introduce a sanity check. Bug: 225469258 Signed-off-by: Oliver Neukum Reviewed-by: Grant Grundler Signed-off-by: David S. Miller Signed-off-by: Lee Jones Change-Id: I2f17bd57b5c4228d0420f716527316a878a34efd --- drivers/net/usb/sr9700.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/usb/sr9700.c b/drivers/net/usb/sr9700.c index e04c8054c2cf..fce6713e970b 100644 --- a/drivers/net/usb/sr9700.c +++ b/drivers/net/usb/sr9700.c @@ -410,7 +410,7 @@ static int sr9700_rx_fixup(struct usbnet *dev, struct sk_buff *skb) /* ignore the CRC length */ len = (skb->data[1] | (skb->data[2] << 8)) - 4; - if (len > ETH_FRAME_LEN) + if (len > ETH_FRAME_LEN || len > skb->len) return 0; /* the last packet of current skb */ From 74720dae8b02bd45816bea52ce7ba3144b469389 Mon Sep 17 00:00:00 2001 From: Aran Dalton Date: Mon, 7 Mar 2022 17:34:34 +0800 Subject: [PATCH 006/156] ANDROID: usb: gadget: f_accessory: add compat_ioctl support On Android 32-bit system, the following Cts Verifier testcase failed: manualTests#com.android.cts.verifier.usb.accessory.UsbAccessoryTestActivity The reason is that compat_ioctl() needs to be called. So let's add compat_ioctl() for 32-bit applications to solve this issue. Bug: 223101878 Change-Id: I6e1f797d919494d293184411041955c33ad08aef Signed-off-by: Aran Dalton (cherry picked from commit 77bf53b4861491ad49414d8fceefda2602274519) --- drivers/usb/gadget/function/f_accessory.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/usb/gadget/function/f_accessory.c b/drivers/usb/gadget/function/f_accessory.c index 66a603f3327d..3510f6d39f0c 100644 --- a/drivers/usb/gadget/function/f_accessory.c +++ b/drivers/usb/gadget/function/f_accessory.c @@ -931,6 +931,7 @@ static const struct file_operations acc_fops = { .read = acc_read, .write = acc_write, .unlocked_ioctl = acc_ioctl, + .compat_ioctl = acc_ioctl, .open = acc_open, .release = acc_release, }; From 71d560e017415ead361cfc10cc50ef36a0545cc2 Mon Sep 17 00:00:00 2001 From: Peifeng Li Date: Mon, 28 Feb 2022 15:25:30 +0800 Subject: [PATCH 007/156] ANDROID: vendor_hooks: export shrink_slab Export shrink_slab to module for do shrink-memory action. Bug: 221768451 Signed-off-by: Peifeng Li Change-Id: I5abe9ad419d64999b714d879c228625a243e90d1 --- mm/vmscan.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 46d8e65071d8..2f496b3f1e5f 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -670,7 +670,7 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, * * Returns the number of reclaimed slab objects. */ -static unsigned long shrink_slab(gfp_t gfp_mask, int nid, +unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg, int priority) { @@ -722,6 +722,7 @@ out: cond_resched(); return freed; } +EXPORT_SYMBOL_GPL(shrink_slab); void drop_slab_node(int nid) { From d9845e9e5c9a6ec115da79bc0b6711ef525b49b6 Mon Sep 17 00:00:00 2001 From: Liujie Xie Date: Wed, 23 Mar 2022 17:56:36 +0800 Subject: [PATCH 008/156] ANDROID: export walk_page_range and swp_swap_info Export walk_page_range and swp_swap_info for reading swap from backing device to zram. Bug: 225273514 Signed-off-by: Liujie Xie Change-Id: If888cfc2823d8003b62bdb177740643696cf6f7e --- mm/pagewalk.c | 1 + mm/swapfile.c | 1 + 2 files changed, 2 insertions(+) diff --git a/mm/pagewalk.c b/mm/pagewalk.c index e81640d9f177..5d5d1c47894e 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -430,6 +430,7 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, } while (start = next, start < end); return err; } +EXPORT_SYMBOL_GPL(walk_page_range); /* * Similar to walk_page_range() but can walk any page tables even if they are diff --git a/mm/swapfile.c b/mm/swapfile.c index 5f2e709f2d39..646145953eeb 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -3560,6 +3560,7 @@ struct swap_info_struct *swp_swap_info(swp_entry_t entry) { return swap_type_to_swap_info(swp_type(entry)); } +EXPORT_SYMBOL_GPL(swp_swap_info); struct swap_info_struct *page_swap_info(struct page *page) { From ec48b1892eb5425c7cb9136f19b00ac14da7f0c2 Mon Sep 17 00:00:00 2001 From: Yunfei Wang Date: Thu, 10 Mar 2022 14:36:48 +0800 Subject: [PATCH 009/156] FROMGIT: iommu/iova: Improve 32-bit free space estimate For various reasons based on the allocator behaviour and typical use-cases at the time, when the max32_alloc_size optimisation was introduced it seemed reasonable to couple the reset of the tracked size to the update of cached32_node upon freeing a relevant IOVA. However, since subsequent optimisations focused on helping genuine 32-bit devices make best use of even more limited address spaces, it is now a lot more likely for cached32_node to be anywhere in a "full" 32-bit address space, and as such more likely for space to become available from IOVAs below that node being freed. At this point, the short-cut in __cached_rbnode_delete_update() really doesn't hold up any more, and we need to fix the logic to reliably provide the expected behaviour. We still want cached32_node to only move upwards, but we should reset the allocation size if *any* 32-bit space has become available. Reported-by: Yunfei Wang Signed-off-by: Robin Murphy Reviewed-by: Miles Chen Link: https://lore.kernel.org/r/033815732d83ca73b13c11485ac39336f15c3b40.1646318408.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel Bug: 223712131 (cherry picked from commit 5b61343b50590fb04a3f6be2cdc4868091757262 https://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git core) Signed-off-by: Yunfei Wang Change-Id: I5026411dd022c6ddea5c0e4da6e69c7b14162c3f --- drivers/iommu/iova.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 7e3f4d6c5e4c..53994947ecf7 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -139,10 +139,11 @@ __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) cached_iova = rb_entry(iovad->cached32_node, struct iova, node); if (free == cached_iova || (free->pfn_hi < iovad->dma_32bit_pfn && - free->pfn_lo >= cached_iova->pfn_lo)) { + free->pfn_lo >= cached_iova->pfn_lo)) iovad->cached32_node = rb_next(&free->node); + + if (free->pfn_lo < iovad->dma_32bit_pfn) iovad->max32_alloc_size = iovad->dma_32bit_pfn; - } cached_iova = rb_entry(iovad->cached_node, struct iova, node); if (free->pfn_lo >= cached_iova->pfn_lo) From a3f112353cc5dd9d9441958feda718de308d918c Mon Sep 17 00:00:00 2001 From: Liujie Xie Date: Sat, 26 Mar 2022 15:02:01 +0800 Subject: [PATCH 010/156] ANDROID: GKI: Add symbols to symbol list Add walk_page_range() and swp_swap_info() to symbol list. Leaf changes summary: 2 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 2 Added functions Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 0 Added variable 2 Added functions: [A] 'function swap_info_struct* swp_swap_info(swp_entry_t)' [A] 'function int walk_page_range(mm_struct*, unsigned long int, unsigned long int, const mm_walk_ops*, void*)' Bug: 225273514 Signed-off-by: Liujie Xie Change-Id: Ia3e8ba0d0a8fe4ed3953af9baf9028d5f27e76e2 --- android/abi_gki_aarch64.xml | 264 +++++++++++++++++++++++++++++++++- android/abi_gki_aarch64_oplus | 2 + 2 files changed, 265 insertions(+), 1 deletion(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 13e355956e3c..684bc1099172 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -4954,6 +4954,7 @@ + @@ -5778,6 +5779,7 @@ + @@ -8870,6 +8872,14 @@ + + + + + + + + @@ -10877,6 +10887,10 @@ + + + + @@ -11426,6 +11440,14 @@ + + + + + + + + @@ -11469,6 +11491,7 @@ + @@ -12403,6 +12426,13 @@ + + + + + + + @@ -17789,6 +17819,7 @@ + @@ -20236,6 +20267,9 @@ + + + @@ -21324,6 +21358,7 @@ + @@ -27062,6 +27097,13 @@ + + + + + + + @@ -27078,6 +27120,7 @@ + @@ -29604,6 +29647,12 @@ + + + + + + @@ -29919,7 +29968,83 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -30526,6 +30651,13 @@ + + + + + + + @@ -31681,6 +31813,13 @@ + + + + + + + @@ -33752,6 +33891,7 @@ + @@ -36351,6 +36491,7 @@ + @@ -38165,6 +38306,7 @@ + @@ -39424,6 +39566,7 @@ + @@ -48015,6 +48158,11 @@ + + + + + @@ -50914,6 +51062,14 @@ + + + + + + + + @@ -60800,6 +60956,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -66174,6 +66362,7 @@ + @@ -66494,6 +66683,13 @@ + + + + + + + @@ -75668,6 +75864,7 @@ + @@ -84057,6 +84254,12 @@ + + + + + + @@ -85130,6 +85333,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -87024,6 +87250,7 @@ + @@ -100235,6 +100462,7 @@ + @@ -101828,6 +102056,7 @@ + @@ -102893,6 +103122,7 @@ + @@ -105835,6 +106065,17 @@ + + + + + + + + + + + @@ -107843,6 +108084,13 @@ + + + + + + + @@ -109521,6 +109769,7 @@ + @@ -111958,6 +112207,7 @@ + @@ -140217,6 +140467,10 @@ + + + + @@ -144408,6 +144662,14 @@ + + + + + + + + diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index c01dfbccd138..d883e8003257 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -2432,6 +2432,7 @@ __sw_hweight32 __sw_hweight64 __sw_hweight8 + swp_swap_info sync_file_create sync_file_get_fence synchronize_irq @@ -3339,6 +3340,7 @@ wakeup_source_register wakeup_source_remove wakeup_source_unregister + walk_page_range __warn_printk watchdog_init_timeout watchdog_set_restart_priority From 95380146ce09e30fc1adfd6069c4cb651d29cc04 Mon Sep 17 00:00:00 2001 From: Liujie Xie Date: Thu, 24 Mar 2022 10:17:40 +0800 Subject: [PATCH 011/156] ANDROID: vendor_hooks: Add hook in shrink_node_memcgs Add vendor hook in shrink_node_memcgs to adjust whether to skip memory reclamation of memcg. Bug: 226482420 Signed-off-by: Liujie Xie Change-Id: I925856353e63c5a821027de4f8476c833e21b982 --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/vmscan.h | 4 +++- mm/vmscan.c | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 3e26777a54c2..7ccf4c4fbf33 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -393,3 +393,4 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_thread_release); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_has_work_ilocked); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_read_done); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_handle_tlb_conf); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_node_memcgs); diff --git a/include/trace/hooks/vmscan.h b/include/trace/hooks/vmscan.h index 95fb17af29e8..48a00c6cede0 100644 --- a/include/trace/hooks/vmscan.h +++ b/include/trace/hooks/vmscan.h @@ -28,7 +28,9 @@ DECLARE_RESTRICTED_HOOK(android_rvh_set_balance_anon_file_reclaim, DECLARE_HOOK(android_vh_page_referenced_check_bypass, TP_PROTO(struct page *page, unsigned long nr_to_scan, int lru, bool *bypass), TP_ARGS(page, nr_to_scan, lru, bypass)); - +DECLARE_HOOK(android_vh_shrink_node_memcgs, + TP_PROTO(struct mem_cgroup *memcg, bool *skip), + TP_ARGS(memcg, skip)); #endif /* _TRACE_HOOK_VMSCAN_H */ /* This part must be outside protection */ #include diff --git a/mm/vmscan.c b/mm/vmscan.c index 2f496b3f1e5f..89beec62bfa2 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2699,6 +2699,7 @@ static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc) struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); unsigned long reclaimed; unsigned long scanned; + bool skip = false; /* * This loop can become CPU-bound when target memcgs @@ -2708,6 +2709,10 @@ static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc) */ cond_resched(); + trace_android_vh_shrink_node_memcgs(memcg, &skip); + if (skip) + continue; + mem_cgroup_calculate_protection(target_memcg, memcg); if (mem_cgroup_below_min(memcg)) { From bc159fee3d373f067cf9a3aa26fda83ef9f2fcc1 Mon Sep 17 00:00:00 2001 From: Peifeng Li Date: Tue, 29 Mar 2022 18:00:16 +0800 Subject: [PATCH 012/156] ANDROID: GKI: Update symbols to symbol list Update symbols to symbol list externed by oem modules. Leaf changes summary: 1 artifact changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 1 Added function Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 0 Added variable 1 Added function: [A] 'function unsigned long int shrink_slab(gfp_t, int, mem_cgroup*, int)' Bug: 193384408 Signed-off-by: Peifeng Li Change-Id: I60dc617ba6c33c5ef52c89488acb8cc62ed22a61 --- android/abi_gki_aarch64.xml | 18 +++++++++++++----- android/abi_gki_aarch64_oplus | 1 + 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 684bc1099172..0dab11372746 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -4566,6 +4566,7 @@ + @@ -138381,6 +138382,13 @@ + + + + + + + @@ -141174,11 +141182,11 @@ - - - - - + + + + + diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index d883e8003257..a97cc169bb0e 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -2248,6 +2248,7 @@ __sg_page_iter_start sg_scsi_ioctl show_regs + shrink_slab sigprocmask si_mem_available si_meminfo From a2485b8abd57d21ce3b47b7bbcadf0a7bf05b0b0 Mon Sep 17 00:00:00 2001 From: Peifeng Li Date: Tue, 15 Mar 2022 21:24:08 +0800 Subject: [PATCH 013/156] ANDROID: vendor_hooks: Add hooks to for alloc_contig_range Provide a vendor hook to allow drain_all_pages to be skipped during alloc_contig_range in some cases to avoid delays caused by it in cases when the benefits of draining pcp lists are known to be small. Bug: 224732340 Signed-off-by: Peifeng Li Change-Id: I0a82f668cf985ad5344d666c0c6372a7e61c3798 --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/mm.h | 3 +++ mm/page_alloc.c | 6 +++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 7ccf4c4fbf33..44507a74e573 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -293,6 +293,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tune_swappiness); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_slab_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_referenced_check_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_drain_all_pages_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cma_drain_all_pages_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_psi_event); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_psi_group); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpuset_fork); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index cc00dcecfd71..b5db91c72bf5 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -122,6 +122,9 @@ DECLARE_HOOK(android_vh_drain_all_pages_bypass, int migratetype, unsigned long did_some_progress, bool *bypass), TP_ARGS(gfp_mask, order, alloc_flags, migratetype, did_some_progress, bypass)); +DECLARE_HOOK(android_vh_cma_drain_all_pages_bypass, + TP_PROTO(unsigned int migratetype, bool *bypass), + TP_ARGS(migratetype, bypass)); struct device; DECLARE_HOOK(android_vh_subpage_dma_contig_alloc, TP_PROTO(bool *allow_subpage_alloc, struct device *dev, size_t *size), diff --git a/mm/page_alloc.c b/mm/page_alloc.c index a9822a2175c5..4bd60d681200 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -8720,6 +8720,7 @@ int alloc_contig_range(unsigned long start, unsigned long end, unsigned long outer_start, outer_end; unsigned int order; int ret = 0; + bool skip_drain_all_pages = false; struct compact_control cc = { .nr_migratepages = 0, @@ -8765,7 +8766,10 @@ int alloc_contig_range(unsigned long start, unsigned long end, return ret; } - drain_all_pages(cc.zone); + trace_android_vh_cma_drain_all_pages_bypass(migratetype, + &skip_drain_all_pages); + if (skip_drain_all_pages) + drain_all_pages(cc.zone); /* * In case of -EBUSY, we'd like to know which page causes problem. From cb7c1a4c787f882b16f6f8ebb5819203d6900460 Mon Sep 17 00:00:00 2001 From: Peifeng Li Date: Sat, 26 Mar 2022 20:43:40 +0800 Subject: [PATCH 014/156] ANDROID: vendor_hooks: Add hooks to for free_unref_page_commit Provide a vendor hook to skip cma-pages to add in pcplist when free_unref_page_commit. The patch is revelant to skip drain_all_pages in alloc_contig_range, the revelant hooks is android_vh_cma_drain_all_pages_bypass which is to avoid to delay in drain pcppages when drain_all_pages. In most case, pcp->high is small so that free-pages with other mt_types can also fill with pcplist full. Bug: 224732340 Signed-off-by: Peifeng Li Change-Id: Ifdeeed9f8934d87671ec3fa6787a02675b993082 --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/mm.h | 3 +++ mm/page_alloc.c | 6 +++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 44507a74e573..2e2285789229 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -294,6 +294,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_slab_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_referenced_check_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_drain_all_pages_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cma_drain_all_pages_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_pcplist_add_cma_pages_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_psi_event); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_psi_group); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpuset_fork); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index b5db91c72bf5..bde2f3556c2a 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -125,6 +125,9 @@ DECLARE_HOOK(android_vh_drain_all_pages_bypass, DECLARE_HOOK(android_vh_cma_drain_all_pages_bypass, TP_PROTO(unsigned int migratetype, bool *bypass), TP_ARGS(migratetype, bypass)); +DECLARE_HOOK(android_vh_pcplist_add_cma_pages_bypass, + TP_PROTO(int migratetype, bool *bypass), + TP_ARGS(migratetype, bypass)); struct device; DECLARE_HOOK(android_vh_subpage_dma_contig_alloc, TP_PROTO(bool *allow_subpage_alloc, struct device *dev, size_t *size), diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 4bd60d681200..30f52cc94948 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3301,6 +3301,7 @@ static void free_unref_page_commit(struct page *page, unsigned long pfn) struct zone *zone = page_zone(page); struct per_cpu_pages *pcp; int migratetype; + bool pcp_skip_cma_pages = false; migratetype = get_pcppage_migratetype(page); __count_vm_event(PGFREE); @@ -3313,7 +3314,10 @@ static void free_unref_page_commit(struct page *page, unsigned long pfn) * excessively into the page allocator */ if (migratetype >= MIGRATE_PCPTYPES) { - if (unlikely(is_migrate_isolate(migratetype))) { + trace_android_vh_pcplist_add_cma_pages_bypass(migratetype, + &pcp_skip_cma_pages); + if (unlikely(is_migrate_isolate(migratetype)) || + pcp_skip_cma_pages) { free_one_page(zone, page, pfn, 0, migratetype, FPI_NONE); return; From 7a7eadac581c1d66056638713ecaf3c1e472b84a Mon Sep 17 00:00:00 2001 From: Lee Jones Date: Thu, 31 Mar 2022 08:47:05 +0000 Subject: [PATCH 015/156] Revert "ANDROID: dm-bow: Protect Ranges fetched and erased from the RB tree" This reverts commit f3ca80cced4e447653b51be876c026c337d84702. Reason for revert: Needs rework - causes unforeseen deadlock. Bug: 227141277 Change-Id: I46f2b3d34c6a6a0a96d0fcc10086aa0d1687b127 Signed-off-by: Lee Jones --- drivers/md/dm-bow.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/drivers/md/dm-bow.c b/drivers/md/dm-bow.c index e666ca01d220..cfd1fa63ff97 100644 --- a/drivers/md/dm-bow.c +++ b/drivers/md/dm-bow.c @@ -236,7 +236,6 @@ static void set_type(struct bow_context *bc, struct bow_range **br, int type) (*br)->type = type; - mutex_lock(&bc->ranges_lock); if (next->type == type) { if (type == TRIMMED) list_del(&next->trimmed_list); @@ -250,7 +249,6 @@ static void set_type(struct bow_context *bc, struct bow_range **br, int type) rb_erase(&(*br)->node, &bc->ranges); kfree(*br); } - mutex_unlock(&bc->ranges_lock); *br = NULL; } @@ -601,7 +599,6 @@ static void dm_bow_dtr(struct dm_target *ti) struct bow_context *bc = (struct bow_context *) ti->private; struct kobject *kobj; - mutex_lock(&bc->ranges_lock); while (rb_first(&bc->ranges)) { struct bow_range *br = container_of(rb_first(&bc->ranges), struct bow_range, node); @@ -609,8 +606,6 @@ static void dm_bow_dtr(struct dm_target *ti) rb_erase(&br->node, &bc->ranges); kfree(br); } - mutex_unlock(&bc->ranges_lock); - if (bc->workqueue) destroy_workqueue(bc->workqueue); if (bc->bufio) @@ -1186,7 +1181,6 @@ static void dm_bow_tablestatus(struct dm_target *ti, char *result, return; } - mutex_lock(&bc->ranges_lock); for (i = rb_first(&bc->ranges); i; i = rb_next(i)) { struct bow_range *br = container_of(i, struct bow_range, node); @@ -1194,11 +1188,11 @@ static void dm_bow_tablestatus(struct dm_target *ti, char *result, readable_type[br->type], (unsigned long long)br->sector); if (result >= end) - goto unlock; + return; result += scnprintf(result, end - result, "\n"); if (result >= end) - goto unlock; + return; if (br->type == TRIMMED) ++trimmed_range_count; @@ -1220,22 +1214,19 @@ static void dm_bow_tablestatus(struct dm_target *ti, char *result, if (!rb_next(i)) { scnprintf(result, end - result, "\nERROR: Last range not of type TOP"); - goto unlock; + return; } if (br->sector > range_top(br)) { scnprintf(result, end - result, "\nERROR: sectors out of order"); - goto unlock; + return; } } if (trimmed_range_count != trimmed_list_length) scnprintf(result, end - result, "\nERROR: not all trimmed ranges in trimmed list"); - -unlock: - mutex_unlock(&bc->ranges_lock); } static void dm_bow_status(struct dm_target *ti, status_type_t type, From 7b7125914c306d3da23534f0157f8ae496ee4454 Mon Sep 17 00:00:00 2001 From: Liujie Xie Date: Thu, 31 Mar 2022 12:04:31 +0800 Subject: [PATCH 016/156] ANDROID: GKI: Add hook symbol to symbol list Add shrink_node_memcgs() tracepoint symbols to symbol list. Leaf changes summary: 2 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 1 Added function Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 1 Added variable 1 Added function: [A] 'function int __traceiter_android_vh_shrink_node_memcgs(void*, mem_cgroup*, bool*)' 1 Added variable: [A] 'tracepoint __tracepoint_android_vh_shrink_node_memcgs' Bug: 226482420 Signed-off-by: Liujie Xie Change-Id: Iff8a2b95a4e891bce814e9a70520368e28b4b5f0 --- android/abi_gki_aarch64.xml | 123 ++++++++++++++++++---------------- android/abi_gki_aarch64_oplus | 2 + 2 files changed, 68 insertions(+), 57 deletions(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 0dab11372746..74944873ed26 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -526,6 +526,7 @@ + @@ -6139,6 +6140,7 @@ + @@ -63343,15 +63345,15 @@ - + - + - + - + @@ -100576,12 +100578,12 @@ - + - + - + @@ -113784,11 +113786,11 @@ - - - - - + + + + + @@ -114392,9 +114394,9 @@ - - - + + + @@ -114402,9 +114404,9 @@ - - - + + + @@ -116954,6 +116956,12 @@ + + + + + + @@ -117840,11 +117848,12 @@ + - + @@ -118414,9 +118423,9 @@ - - - + + + @@ -118533,9 +118542,9 @@ - - - + + + @@ -127310,14 +127319,14 @@ - - - + + + - - - + + + @@ -127997,8 +128006,8 @@ - - + + @@ -135677,8 +135686,8 @@ - - + + @@ -138389,11 +138398,11 @@ - + - - + + @@ -140114,9 +140123,9 @@ - - - + + + @@ -141182,11 +141191,11 @@ - - - - - + + + + + @@ -145051,19 +145060,19 @@ - - - - - - + + + + + + - - - - - + + + + + diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index a97cc169bb0e..cdb5d44f797a 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -2688,6 +2688,7 @@ __traceiter_android_vh_show_resume_epoch_val __traceiter_android_vh_show_stack_hash __traceiter_android_vh_show_suspend_epoch_val + __traceiter_android_vh_shrink_node_memcgs __traceiter_android_vh_sync_txn_recvd __traceiter_android_vh_syscall_prctl_finished __traceiter_android_vh_timer_calc_index @@ -2892,6 +2893,7 @@ __tracepoint_android_vh_show_resume_epoch_val __tracepoint_android_vh_show_stack_hash __tracepoint_android_vh_show_suspend_epoch_val + __tracepoint_android_vh_shrink_node_memcgs __tracepoint_android_vh_sync_txn_recvd __tracepoint_android_vh_syscall_prctl_finished __tracepoint_android_vh_timer_calc_index From 51513a17753852beb362a7f14fcc60ef40203a18 Mon Sep 17 00:00:00 2001 From: Peifeng Li Date: Thu, 31 Mar 2022 17:55:33 +0800 Subject: [PATCH 017/156] ANDROID: GKI: Update symbols to symbol list Leaf changes summary: 2 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 1 Added function Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 1 Added variable 1 Added function: [A] 'function int __traceiter_android_vh_cma_drain_all_pages_bypass(void*, unsigned int, bool*)' 1 Added variable: [A] 'tracepoint __tracepoint_android_vh_cma_drain_all_pages_bypass' Bug: 193384408 Signed-off-by: Peifeng Li Change-Id: Ib9f95f702e2776a6caded709a88f1e9f2ee154b1 --- android/abi_gki_aarch64.xml | 9 +++++++++ android/abi_gki_aarch64_oplus | 2 ++ 2 files changed, 11 insertions(+) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 74944873ed26..9702f89c75e9 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -414,6 +414,7 @@ + @@ -6014,6 +6015,7 @@ + @@ -116266,6 +116268,12 @@ + + + + + + @@ -117722,6 +117730,7 @@ + diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index cdb5d44f797a..ece922a3f41e 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -2697,6 +2697,7 @@ __traceiter_android_vh_tune_swappiness __traceiter_android_vh_page_referenced_check_bypass __traceiter_android_vh_drain_all_pages_bypass + __traceiter_android_vh_cma_drain_all_pages_bypass __traceiter_android_vh_ufs_compl_command __traceiter_android_vh_ufs_send_command __traceiter_android_vh_ufs_send_tm_command @@ -2902,6 +2903,7 @@ __tracepoint_android_vh_tune_swappiness __tracepoint_android_vh_page_referenced_check_bypass __tracepoint_android_vh_drain_all_pages_bypass + __tracepoint_android_vh_cma_drain_all_pages_bypass __tracepoint_android_vh_ufs_compl_command __tracepoint_android_vh_ufs_send_command __tracepoint_android_vh_ufs_send_tm_command From e2c0e8502e880d10fb972ad3be1769594ab49b57 Mon Sep 17 00:00:00 2001 From: Peifeng Li Date: Thu, 31 Mar 2022 19:27:44 +0800 Subject: [PATCH 018/156] ANDROID: GKI: Update symbols to symbol list Leaf changes summary: 2 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 1 Added function Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 1 Added variable 1 Added function: [A] 'function int __traceiter_android_vh_pcplist_add_cma_pages_bypass(void*, int, bool*)' 1 Added variable: [A] 'tracepoint __tracepoint_android_vh_pcplist_add_cma_pages_bypass' Bug: 193384408 Signed-off-by: Peifeng Li Change-Id: Ic8eeb43ed58120379c7fc16642545d70d220384a --- android/abi_gki_aarch64.xml | 9 +++++++++ android/abi_gki_aarch64_oplus | 2 ++ 2 files changed, 11 insertions(+) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 9702f89c75e9..36efaa810fd5 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -488,6 +488,7 @@ + @@ -6093,6 +6094,7 @@ + @@ -116736,6 +116738,12 @@ + + + + + + @@ -117808,6 +117816,7 @@ + diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index ece922a3f41e..19e78938fd20 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -2698,6 +2698,7 @@ __traceiter_android_vh_page_referenced_check_bypass __traceiter_android_vh_drain_all_pages_bypass __traceiter_android_vh_cma_drain_all_pages_bypass + __traceiter_android_vh_pcplist_add_cma_pages_bypass __traceiter_android_vh_ufs_compl_command __traceiter_android_vh_ufs_send_command __traceiter_android_vh_ufs_send_tm_command @@ -2904,6 +2905,7 @@ __tracepoint_android_vh_page_referenced_check_bypass __tracepoint_android_vh_drain_all_pages_bypass __tracepoint_android_vh_cma_drain_all_pages_bypass + __tracepoint_android_vh_pcplist_add_cma_pages_bypass __tracepoint_android_vh_ufs_compl_command __tracepoint_android_vh_ufs_send_command __tracepoint_android_vh_ufs_send_tm_command From 404df4751a5faf73db224dfe6078040c0ab776e3 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 24 Mar 2022 11:00:05 -0700 Subject: [PATCH 019/156] ANDROID: mm: Fix implicit declaration of function 'isolate_lru_page' When compiled with CONFIG_SHMEM=n, shmem.c does not include internal.h and isolate_lru_page function declaration can't be found. Fix this by making isolate_lru_page usage conditional upon CONFIG_SHMEM inside reclaim_shmem_address_space. Fixes: daeabfe7fa04 ("ANDROID: mm: add reclaim_shmem_address_space() for faster reclaims") Signed-off-by: Suren Baghdasaryan Change-Id: Ia46a57681d26ac103e84ef7caa61a22dbd45cf04 --- mm/shmem.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/shmem.c b/mm/shmem.c index 364703bd305e..8fb9d2ddec70 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -4307,6 +4307,7 @@ EXPORT_SYMBOL_GPL(shmem_mark_page_lazyfree); int reclaim_shmem_address_space(struct address_space *mapping) { +#ifdef CONFIG_SHMEM pgoff_t start = 0; struct page *page; LIST_HEAD(page_list); @@ -4340,5 +4341,8 @@ int reclaim_shmem_address_space(struct address_space *mapping) reclaimed = reclaim_pages_from_list(&page_list); return reclaimed; +#else + return 0; +#endif } EXPORT_SYMBOL_GPL(reclaim_shmem_address_space); From 002528dfb515ad62b1616650042180e09f5bf59e Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Sun, 2 Jan 2022 21:35:58 -0500 Subject: [PATCH 020/156] UPSTREAM: locking/lockdep: Avoid potential access of invalid memory in lock_class commit 61cc4534b6550997c97a03759ab46b29d44c0017 upstream. It was found that reading /proc/lockdep after a lockdep splat may potentially cause an access to freed memory if lockdep_unregister_key() is called after the splat but before access to /proc/lockdep [1]. This is due to the fact that graph_lock() call in lockdep_unregister_key() fails after the clearing of debug_locks by the splat process. After lockdep_unregister_key() is called, the lock_name may be freed but the corresponding lock_class structure still have a reference to it. That invalid memory pointer will then be accessed when /proc/lockdep is read by a user and a use-after-free (UAF) error will be reported if KASAN is enabled. To fix this problem, lockdep_unregister_key() is now modified to always search for a matching key irrespective of the debug_locks state and zap the corresponding lock class if a matching one is found. [1] https://lore.kernel.org/lkml/77f05c15-81b6-bddd-9650-80d5f23fe330@i-love.sakura.ne.jp/ Bug: 225086211 Fixes: 8b39adbee805 ("locking/lockdep: Make lockdep_unregister_key() honor 'debug_locks' again") Reported-by: Tetsuo Handa Signed-off-by: Waiman Long Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Bart Van Assche Link: https://lkml.kernel.org/r/20220103023558.1377055-1-longman@redhat.com Signed-off-by: Cheng Jui Wang Change-Id: I1b03e363142d2b9905f8f263b02d3c7cfdbc515a --- kernel/locking/lockdep.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 1f6a2f1226fa..3ee4fb8d312c 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -6209,7 +6209,13 @@ void lockdep_reset_lock(struct lockdep_map *lock) lockdep_reset_lock_reg(lock); } -/* Unregister a dynamically allocated key. */ +/* + * Unregister a dynamically allocated key. + * + * Unlike lockdep_register_key(), a search is always done to find a matching + * key irrespective of debug_locks to avoid potential invalid access to freed + * memory in lock_class entry. + */ void lockdep_unregister_key(struct lock_class_key *key) { struct hlist_head *hash_head = keyhashentry(key); @@ -6224,10 +6230,8 @@ void lockdep_unregister_key(struct lock_class_key *key) return; raw_local_irq_save(flags); - if (!graph_lock()) - goto out_irq; + lockdep_lock(); - pf = get_pending_free(); hlist_for_each_entry_rcu(k, hash_head, hash_entry) { if (k == key) { hlist_del_rcu(&k->hash_entry); @@ -6235,11 +6239,13 @@ void lockdep_unregister_key(struct lock_class_key *key) break; } } - WARN_ON_ONCE(!found); - __lockdep_free_key_range(pf, key, 1); - call_rcu_zapped(pf); - graph_unlock(); -out_irq: + WARN_ON_ONCE(!found && debug_locks); + if (found) { + pf = get_pending_free(); + __lockdep_free_key_range(pf, key, 1); + call_rcu_zapped(pf); + } + lockdep_unlock(); raw_local_irq_restore(flags); /* Wait until is_dynamic_key() has finished accessing k->hash_entry. */ From ceb6918d1d4a413e79bfcaea46d63721d53af632 Mon Sep 17 00:00:00 2001 From: Saravana Kannan Date: Wed, 30 Mar 2022 14:58:30 -0700 Subject: [PATCH 021/156] ANDROID: vendor_hooks: Reduce pointless modversions CRC churn When vendor hooks are added to a file that previously didn't have any vendor hooks, we end up indirectly including linux/tracepoint.h. This causes some data types that used to be opaque (forward declared) to the code to become visible to the code. Modversions correctly catches this change in visibility, but we don't really care about the data types made visible when linux/tracepoint.h is included. So, hide this from modversions in the central vendor_hooks.h file instead of having to fix this on a case by case basis. Since this is a KMI frozen branch, existing vendor hook headers are left as is to avoid KMI breakage due to CRC churn. To avoid future pointless CRC churn, new vendor hook header files that include vendor_hooks.h should not include linux/tracepoint.h directly. Bug: 227513263 Bug: 226140073 Signed-off-by: Saravana Kannan Change-Id: Ia88e6af11dd94fe475c464eb30a6e5e1e24c938b --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/vendor_hooks.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 2e2285789229..2ae0b6eecc37 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -8,6 +8,7 @@ #define CREATE_TRACE_POINTS #include +#include #include #include #include diff --git a/include/trace/hooks/vendor_hooks.h b/include/trace/hooks/vendor_hooks.h index 87fe9ccdc331..2c2d57b6cf2a 100644 --- a/include/trace/hooks/vendor_hooks.h +++ b/include/trace/hooks/vendor_hooks.h @@ -7,7 +7,9 @@ * will override the DECLARE_RESTRICTED_HOOK and break the second include. */ +#ifndef __GENKSYMS__ #include +#endif #if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_ANDROID_VENDOR_HOOKS) From d8fade2b40aaf421f46381cdba959cd650a87f2a Mon Sep 17 00:00:00 2001 From: wuzhe Date: Sat, 2 Apr 2022 23:18:52 +0800 Subject: [PATCH 022/156] ANDROID: GKI: Update symbols to abi_gki_aarch64_oplus Leaf changes summary: 42 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 42 Added functions Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 0 Added variable 42 Added functions: [A] 'function int __cleancache_get_page(page*)' [A] 'function unsigned long int __page_file_index(page*)' [A] 'function address_space* __page_file_mapping(page*)' [A] 'function int __percpu_counter_init(percpu_counter*, s64, gfp_t, lock_class_key*)' [A] 'function s64 __percpu_counter_sum(percpu_counter*)' [A] 'function void __xa_clear_mark(xarray*, unsigned long int, xa_mark_t)' [A] 'function int add_swap_extent(swap_info_struct*, unsigned long int, unsigned long int, long long unsigned int)' [A] 'function int add_to_page_cache_lru(page*, address_space*, unsigned long int, gfp_t)' [A] 'function long int congestion_wait(int, long int)' [A] 'function bool filemap_allow_speculation()' [A] 'function int filemap_check_errors(address_space*)' [A] 'function vm_fault_t filemap_map_pages(vm_fault*, unsigned long int, unsigned long int)' [A] 'function void generate_random_uuid(unsigned char*)' [A] 'function int kset_register(kset*)' [A] 'function char* match_strdup(const __anonymous_struct__*)' [A] 'function void migrate_page_copy(page*, page*)' [A] 'function int migrate_page_move_mapping(address_space*, page*, page*, int)' [A] 'function void migrate_page_states(page*, page*)' [A] 'function void page_cache_ra_unbounded(readahead_control*, unsigned long int, unsigned long int)' [A] 'function void page_cache_sync_ra(readahead_control*, file_ra_state*, unsigned long int)' [A] 'function const char* page_get_link(dentry*, inode*, delayed_call*)' [A] 'function int page_symlink(inode*, const char*, int)' [A] 'function int pagecache_write_begin(file*, address_space*, loff_t, unsigned int, unsigned int, page**, void**)' [A] 'function int pagecache_write_end(file*, address_space*, loff_t, unsigned int, unsigned int, page*, void*)' [A] 'function void percpu_counter_add_batch(percpu_counter*, long long int, int)' [A] 'function void percpu_counter_destroy(percpu_counter*)' [A] 'function void percpu_counter_set(percpu_counter*, long long int)' [A] 'function unsigned int radix_tree_gang_lookup(const xarray*, void**, unsigned long int, unsigned int)' [A] 'function int radix_tree_preload(unsigned int)' [A] 'function page* read_cache_page_gfp(address_space*, unsigned long int, gfp_t)' [A] 'function void truncate_inode_pages_range(address_space*, loff_t, loff_t)' [A] 'function void truncate_pagecache_range(inode*, loff_t, loff_t)' [A] 'function int utf8_casefold(const unicode_map*, const qstr*, unsigned char*, unsigned long int)' [A] 'function unicode_map* utf8_load(const char*)' [A] 'function int utf8_strncasecmp_folded(const unicode_map*, const qstr*, const qstr*)' [A] 'function void utf8_unload(unicode_map*)' [A] 'function int utf8s_to_utf16s(const unsigned char*, int, utf16_endian, unsigned short int*, int)' [A] 'function void vm_unmap_aliases()' [A] 'function void wait_for_completion_io(completion*)' [A] 'function void wait_for_stable_page(page*)' [A] 'function void wait_on_page_writeback(page*)' [A] 'function bool xa_get_mark(xarray*, unsigned long int, unsigned int)' Bug: 227860734 Signed-off-by: wuzhe Change-Id: Ied8975ea5ca3026f2177249604a6721b8739a6df --- android/abi_gki_aarch64.xml | 11398 +++++++++++++++++--------------- android/abi_gki_aarch64_oplus | 42 + 2 files changed, 5971 insertions(+), 5469 deletions(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 36efaa810fd5..b3258f6a0bcb 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -42,6 +42,7 @@ + @@ -192,9 +193,13 @@ + + + + @@ -654,6 +659,7 @@ + @@ -710,10 +716,12 @@ + + @@ -1125,6 +1133,7 @@ + @@ -2352,10 +2361,13 @@ + + + @@ -2448,6 +2460,7 @@ + @@ -3047,6 +3060,7 @@ + @@ -3164,6 +3178,7 @@ + @@ -3242,6 +3257,9 @@ + + + @@ -3631,14 +3649,20 @@ + + + + + + @@ -3765,6 +3789,9 @@ + + + @@ -4050,12 +4077,14 @@ + + @@ -4096,6 +4125,7 @@ + @@ -5093,7 +5123,9 @@ + + @@ -5502,6 +5534,11 @@ + + + + + @@ -5728,6 +5765,7 @@ + @@ -5762,12 +5800,15 @@ + + + @@ -5803,6 +5844,7 @@ + @@ -7098,6 +7140,7 @@ + @@ -8873,9 +8916,9 @@ - - - + + + @@ -8896,6 +8939,7 @@ + @@ -8977,6 +9021,7 @@ + @@ -11852,7 +11897,6 @@ - @@ -12259,8 +12303,8 @@ - - + + @@ -12901,23 +12945,7 @@ - - - - - - - - - - - - - - - - - + @@ -13565,7 +13593,6 @@ - @@ -15115,6 +15142,12 @@ + + + + + + @@ -15609,9 +15642,6 @@ - - - @@ -15811,29 +15841,7 @@ - - - - - - - - - - - - - - - - - - - - - - - + @@ -18723,6 +18731,7 @@ + @@ -19113,7 +19122,11 @@ - + + + + + @@ -20068,6 +20081,7 @@ + @@ -20265,6 +20279,7 @@ + @@ -20705,6 +20720,14 @@ + + + + + + + + @@ -21055,7 +21078,7 @@ - + @@ -21787,7 +21810,6 @@ - @@ -22528,11 +22550,6 @@ - - - - - @@ -24716,7 +24733,7 @@ - + @@ -24755,6 +24772,7 @@ + @@ -24941,7 +24959,6 @@ - @@ -25246,23 +25263,7 @@ - - - - - - - - - - - - - - - - - + @@ -25925,7 +25926,7 @@ - + @@ -26081,7 +26082,7 @@ - + @@ -26302,6 +26303,7 @@ + @@ -26379,7 +26381,7 @@ - + @@ -26436,7 +26438,6 @@ - @@ -26793,7 +26794,6 @@ - @@ -27432,7 +27432,7 @@ - + @@ -28374,6 +28374,7 @@ + @@ -31237,6 +31238,9 @@ + + + @@ -31350,7 +31354,6 @@ - @@ -32223,9 +32226,9 @@ - - - + + + @@ -33112,6 +33115,7 @@ + @@ -33366,68 +33370,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -33934,9 +33877,10 @@ + - + @@ -34348,7 +34292,6 @@ - @@ -34547,7 +34490,7 @@ - + @@ -36184,6 +36127,10 @@ + + + + @@ -36865,17 +36812,7 @@ - - - - - - - - - - - + @@ -38103,6 +38040,7 @@ + @@ -38893,8 +38831,8 @@ - - + + @@ -39867,6 +39805,17 @@ + + + + + + + + + + + @@ -40147,9 +40096,9 @@ - - - + + + @@ -40656,7 +40605,6 @@ - @@ -41640,6 +41588,7 @@ + @@ -44112,6 +44061,13 @@ + + + + + + + @@ -44141,7 +44097,15 @@ + + + + + + + + @@ -44478,7 +44442,18 @@ - + + + + + + + + + + + + @@ -44574,12 +44549,12 @@ - - - - - - + + + + + + @@ -44684,6 +44659,7 @@ + @@ -45464,21 +45440,21 @@ - + - + - + - + - + - + @@ -45925,6 +45901,7 @@ + @@ -47576,8 +47553,8 @@ - - + + @@ -47923,35 +47900,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -48165,7 +48114,7 @@ - + @@ -48516,14 +48465,7 @@ - - - - - - - - + @@ -49169,6 +49111,11 @@ + + + + + @@ -49580,7 +49527,7 @@ - + @@ -49706,11 +49653,6 @@ - - - - - @@ -50228,7 +50170,56 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -50569,7 +50560,6 @@ - @@ -51517,7 +51507,7 @@ - + @@ -52362,8 +52352,26 @@ + + + + + + + + + + + + + + + + + + @@ -52622,6 +52630,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -52759,6 +52787,7 @@ + @@ -52809,7 +52838,6 @@ - @@ -53420,23 +53448,7 @@ - - - - - - - - - - - - - - - - - + @@ -53798,6 +53810,7 @@ + @@ -54728,9 +54741,9 @@ - - - + + + @@ -55450,7 +55463,6 @@ - @@ -56035,7 +56047,7 @@ - + @@ -56500,7 +56512,7 @@ - + @@ -57371,6 +57383,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -57579,7 +57614,6 @@ - @@ -58716,6 +58750,7 @@ + @@ -59178,6 +59213,11 @@ + + + + + @@ -60402,7 +60442,44 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -61628,7 +61705,65 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -62405,6 +62540,7 @@ + @@ -63045,6 +63181,7 @@ + @@ -63349,17 +63486,7 @@ - - - - - - - - - - - + @@ -64439,7 +64566,7 @@ - + @@ -64933,10 +65060,10 @@ - - - - + + + + @@ -64987,11 +65114,6 @@ - - - - - @@ -65575,7 +65697,47 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -66171,6 +66333,11 @@ + + + + + @@ -67205,6 +67372,11 @@ + + + + + @@ -67666,7 +67838,7 @@ - + @@ -69532,6 +69704,7 @@ + @@ -69728,7 +69901,6 @@ - @@ -70439,7 +70611,6 @@ - @@ -71338,7 +71509,14 @@ - + + + + + + + + @@ -71472,6 +71650,14 @@ + + + + + + + + @@ -72160,10 +72346,9 @@ - - - + + @@ -72282,6 +72467,7 @@ + @@ -72606,6 +72792,7 @@ + @@ -72969,8 +73156,8 @@ - - + + @@ -73042,7 +73229,6 @@ - @@ -73510,7 +73696,38 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -73621,7 +73838,7 @@ - + @@ -74262,6 +74479,7 @@ + @@ -76311,6 +76529,7 @@ + @@ -77718,12 +77937,12 @@ - + - + @@ -78779,12 +78998,12 @@ - - - - - - + + + + + + @@ -80081,7 +80300,6 @@ - @@ -80321,6 +80539,7 @@ + @@ -81046,7 +81265,6 @@ - @@ -81634,7 +81852,6 @@ - @@ -81707,9 +81924,6 @@ - - - @@ -81726,6 +81940,11 @@ + + + + + @@ -81748,6 +81967,7 @@ + @@ -82208,7 +82428,7 @@ - + @@ -82284,6 +82504,14 @@ + + + + + + + + @@ -85391,7 +85619,6 @@ - @@ -86010,7 +86237,7 @@ - + @@ -86691,9 +86918,9 @@ - - - + + + @@ -87162,7 +87389,26 @@ - + + + + + + + + + + + + + + + + + + + + @@ -87187,7 +87433,7 @@ - + @@ -87212,7 +87458,6 @@ - @@ -87251,7 +87496,6 @@ - @@ -88666,7 +88910,7 @@ - + @@ -89313,8 +89557,8 @@ - - + + @@ -90389,6 +90633,7 @@ + @@ -90549,14 +90794,6 @@ - - - - - - - - @@ -90991,20 +91228,7 @@ - - - - - - - - - - - - - - + @@ -91561,7 +91785,7 @@ - + @@ -91736,7 +91960,7 @@ - + @@ -92031,7 +92255,6 @@ - @@ -92172,12 +92395,12 @@ - + - + @@ -92269,6 +92492,7 @@ + @@ -93743,6 +93967,7 @@ + @@ -94255,6 +94480,17 @@ + + + + + + + + + + + @@ -95017,7 +95253,7 @@ - + @@ -95529,7 +95765,7 @@ - + @@ -95575,7 +95811,7 @@ - + @@ -96013,7 +96249,7 @@ - + @@ -96720,6 +96956,9 @@ + + + @@ -97959,7 +98198,6 @@ - @@ -98235,8 +98473,8 @@ - - + + @@ -98345,6 +98583,7 @@ + @@ -98445,8 +98684,8 @@ - - + + @@ -99603,7 +99842,6 @@ - @@ -100469,7 +100707,6 @@ - @@ -100582,14 +100819,7 @@ - - - - - - - - + @@ -102074,6 +102304,9 @@ + + + @@ -102086,9 +102319,6 @@ - - - @@ -102112,7 +102342,7 @@ - + @@ -102253,8 +102483,8 @@ - - + + @@ -102854,6 +103084,7 @@ + @@ -103122,7 +103353,6 @@ - @@ -103657,20 +103887,7 @@ - - - - - - - - - - - - - - + @@ -103729,9 +103946,6 @@ - - - @@ -107176,7 +107390,6 @@ - @@ -107191,6 +107404,20 @@ + + + + + + + + + + + + + + @@ -107525,7 +107752,6 @@ - @@ -111461,6 +111687,11 @@ + + + + + @@ -111787,7 +112018,6 @@ - @@ -113555,6 +113785,7 @@ + @@ -113639,6 +113870,7 @@ + @@ -113762,8 +113994,8 @@ - - + + @@ -113974,6 +114206,10 @@ + + + + @@ -114340,7 +114576,6 @@ - @@ -114891,6 +115126,14 @@ + + + + + + + + @@ -114906,6 +115149,17 @@ + + + + + + + + + + + @@ -118136,6 +118390,12 @@ + + + + + + @@ -118155,9 +118415,9 @@ - - - + + + @@ -118168,33 +118428,33 @@ - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + @@ -118204,27 +118464,27 @@ - - - + + + - - - + + + - - - + + + - - - + + + @@ -118234,8 +118494,8 @@ - - + + @@ -118254,8 +118514,8 @@ - - + + @@ -118271,57 +118531,57 @@ - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + @@ -118332,16 +118592,16 @@ - - + + - - + + @@ -118365,12 +118625,12 @@ - - - - - - + + + + + + @@ -118387,9 +118647,9 @@ - - - + + + @@ -118405,22 +118665,29 @@ - - + + - - - + + + + + + + + + + - - + + - - - + + + @@ -118430,15 +118697,22 @@ - - - + + + + + + + + + + - - - + + + @@ -118451,22 +118725,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -118545,13 +118819,13 @@ - - - - - - - + + + + + + + @@ -118560,29 +118834,29 @@ - - - + + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -118624,11 +118898,11 @@ - - - - - + + + + + @@ -118648,14 +118922,14 @@ - - + + - - - - + + + + @@ -118678,15 +118952,15 @@ - - - - + + + + - - - + + + @@ -118694,11 +118968,11 @@ - - - - - + + + + + @@ -118749,21 +119023,21 @@ - - - - - + + + + + - - - - - - + + + + + + - - + + @@ -118804,18 +119078,18 @@ - - + + - - - + + + - - - + + + @@ -118848,17 +119122,17 @@ - - - - + + + + - - - - - + + + + + @@ -118869,42 +119143,42 @@ - - - - + + + + - - + + - - - + + + - - - + + + - - + + - - - - + + + + - - + + - - + + @@ -118914,15 +119188,15 @@ - - + + - - - - - + + + + + @@ -118935,9 +119209,9 @@ - - - + + + @@ -118946,49 +119220,49 @@ - - - - + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -118998,10 +119272,10 @@ - - - - + + + + @@ -119051,17 +119325,17 @@ - - + + - - + + - - - + + + @@ -119096,10 +119370,10 @@ - - - - + + + + @@ -119131,13 +119405,13 @@ - - - + + + - - + + @@ -119168,8 +119442,8 @@ - - + + @@ -119178,8 +119452,8 @@ - - + + @@ -119201,8 +119475,8 @@ - - + + @@ -119249,8 +119523,8 @@ - - + + @@ -119272,10 +119546,10 @@ - - - - + + + + @@ -119311,9 +119585,9 @@ - - - + + + @@ -119462,16 +119736,16 @@ - - + + - - + + @@ -119520,34 +119794,34 @@ - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - + + + - - - + + + @@ -119568,31 +119842,31 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -119610,9 +119884,9 @@ - - - + + + @@ -119660,67 +119934,67 @@ - - - - + + + + - - - - - + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - + + + + + + + + + + @@ -119736,16 +120010,16 @@ - - + + - - - - - - + + + + + + @@ -119766,22 +120040,22 @@ - - + + - - + + - - + + - - + + @@ -119805,23 +120079,23 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + @@ -119831,8 +120105,8 @@ - - + + @@ -119845,8 +120119,8 @@ - - + + @@ -119916,9 +120190,9 @@ - - - + + + @@ -119962,11 +120236,11 @@ - - - - - + + + + + @@ -119982,19 +120256,19 @@ - - - + + + - - - + + + - - - + + + @@ -120026,9 +120300,9 @@ - - - + + + @@ -120117,14 +120391,14 @@ - - - + + + - - - + + + @@ -120133,47 +120407,47 @@ - - - + + + - - - - + + + + - - + + - - - - - + + + + + - - - - - + + + + + - - + + - - - - + + + + - - + + @@ -120186,26 +120460,26 @@ - - + + - - + + - - - + + + - - - + + + @@ -120214,48 +120488,48 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -120266,8 +120540,8 @@ - - + + @@ -120385,8 +120659,8 @@ - - + + @@ -120494,28 +120768,28 @@ - - - + + + - - + + - - - - - + + + + + @@ -120526,13 +120800,13 @@ - - - - - - - + + + + + + + @@ -120543,11 +120817,11 @@ - - - - - + + + + + @@ -120559,11 +120833,11 @@ - - - - - + + + + + @@ -120582,8 +120856,8 @@ - - + + @@ -120620,10 +120894,10 @@ - - - - + + + + @@ -120661,22 +120935,22 @@ - - + + - - - - + + + + - - + + - - + + @@ -120687,6 +120961,11 @@ + + + + + @@ -120703,22 +120982,22 @@ - - + + - - - - + + + + - - - + + + @@ -120779,8 +121058,8 @@ - - + + @@ -120811,18 +121090,18 @@ - - + + - - - + + + - - - + + + @@ -120834,8 +121113,8 @@ - - + + @@ -120851,9 +121130,9 @@ - - - + + + @@ -120886,12 +121165,12 @@ - - + + - - + + @@ -120902,23 +121181,23 @@ - - + + - - - + + + - - - - + + + + @@ -120933,9 +121212,9 @@ - - - + + + @@ -120954,8 +121233,8 @@ - - + + @@ -120975,58 +121254,58 @@ - - - + + + - - - + + + - - - - + + + + - - - - - + + + + + - - - - - + + + + + - + - + - - - + + + - - - - - + + + + + - - - - - + + + + + @@ -121075,12 +121354,12 @@ - - + + - - + + @@ -121109,10 +121388,10 @@ - - - - + + + + @@ -121167,42 +121446,42 @@ - - - - - - + + + + + + - - - - - - + + + + + + - - + + - - - + + + - - - + + + - + @@ -121245,10 +121524,10 @@ - - - - + + + + @@ -121256,9 +121535,9 @@ - - - + + + @@ -121268,7 +121547,7 @@ - + @@ -121289,8 +121568,8 @@ - - + + @@ -121311,22 +121590,22 @@ - - + + - - - + + + - - - + + + @@ -121376,11 +121655,11 @@ - - - - - + + + + + @@ -121388,44 +121667,44 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - - + + + + @@ -121446,8 +121725,8 @@ - - + + @@ -121472,18 +121751,18 @@ - - - + + + - - - + + + @@ -121500,9 +121779,9 @@ - - - + + + @@ -121513,11 +121792,11 @@ - - - - - + + + + + @@ -121527,11 +121806,11 @@ - - + + - + @@ -121553,27 +121832,27 @@ - - + + - - - + + + - - + + - - - - + + + + @@ -121609,12 +121888,12 @@ - - + + - - + + @@ -121623,7 +121902,7 @@ - + @@ -121654,11 +121933,11 @@ - - - - - + + + + + @@ -121668,25 +121947,25 @@ - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -121723,11 +122002,11 @@ - - - - - + + + + + @@ -121737,11 +122016,11 @@ - - - - - + + + + + @@ -121773,9 +122052,9 @@ - - - + + + @@ -121786,8 +122065,8 @@ - - + + @@ -121797,9 +122076,9 @@ - - - + + + @@ -121808,11 +122087,11 @@ - - - - - + + + + + @@ -121825,16 +122104,16 @@ - - + + - - + + - - + + @@ -121849,8 +122128,8 @@ - - + + @@ -121892,49 +122171,49 @@ - - + + - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -121956,9 +122235,9 @@ - - - + + + @@ -121971,9 +122250,9 @@ - - - + + + @@ -121995,10 +122274,10 @@ - - - - + + + + @@ -122029,15 +122308,15 @@ - - - - + + + + - - - + + + @@ -122057,8 +122336,8 @@ - - + + @@ -122074,8 +122353,8 @@ - - + + @@ -122083,13 +122362,13 @@ - - + + - - - + + + @@ -122100,9 +122379,9 @@ - - - + + + @@ -122119,16 +122398,16 @@ - - + + - - + + @@ -122153,9 +122432,9 @@ - - - + + + @@ -122245,9 +122524,9 @@ - - - + + + @@ -122260,17 +122539,17 @@ - - - - + + + + - - - - + + + + @@ -122278,10 +122557,10 @@ - - - - + + + + @@ -122289,9 +122568,9 @@ - - - + + + @@ -122316,11 +122595,11 @@ - - - - - + + + + + @@ -122392,20 +122671,20 @@ - - + + - - + + - - + + @@ -122418,8 +122697,8 @@ - - + + @@ -122441,12 +122720,12 @@ - - - - - - + + + + + + @@ -122455,40 +122734,40 @@ - - - + + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -122523,28 +122802,28 @@ - - - + + + - - + + - - - - + + + + - - - + + + @@ -122566,10 +122845,10 @@ - - - - + + + + @@ -122591,11 +122870,11 @@ - - - - - + + + + + @@ -122618,15 +122897,15 @@ - - - + + + - - - - + + + + @@ -122643,14 +122922,14 @@ - - - + + + - - - + + + @@ -122679,22 +122958,22 @@ - - + + - - + + - - - - + + + + @@ -122717,10 +122996,10 @@ - - - - + + + + @@ -122728,15 +123007,15 @@ - - - - + + + + - - - + + + @@ -122843,11 +123122,11 @@ - - - - - + + + + + @@ -123045,29 +123324,29 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -123077,10 +123356,10 @@ - - - - + + + + @@ -123119,14 +123398,14 @@ - - - - - - - - + + + + + + + + @@ -123139,9 +123418,9 @@ - - - + + + @@ -123174,8 +123453,8 @@ - - + + @@ -123242,14 +123521,14 @@ - - - + + + - - - + + + @@ -123257,10 +123536,10 @@ - - - - + + + + @@ -123305,15 +123584,15 @@ - - - - + + + + - - - + + + @@ -123412,16 +123691,16 @@ - - - + + + - - - - - + + + + + @@ -123477,46 +123756,46 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - + + - - - - - + + + + + - - + + - - + + @@ -123625,9 +123904,9 @@ - - - + + + @@ -123649,17 +123928,17 @@ - - + + - - - + + + - - + + @@ -123667,15 +123946,15 @@ - - - + + + - - - - + + + + @@ -123686,34 +123965,34 @@ - - + + - - - - + + + + - - + + - - - + + + - - - - + + + + @@ -123725,9 +124004,9 @@ - - - + + + @@ -123735,8 +124014,8 @@ - - + + @@ -123747,15 +124026,15 @@ - + - - - - - - + + + + + + @@ -123767,13 +124046,13 @@ - - - - + + + + - - + + @@ -123793,10 +124072,10 @@ - - - - + + + + @@ -123954,46 +124233,46 @@ - - - + + + - - - + + + - - + + - - - - - + + + + + - - + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -124126,8 +124405,8 @@ - - + + @@ -124160,16 +124439,16 @@ - - + + - - + + - - + + @@ -124181,8 +124460,8 @@ - - + + @@ -124193,8 +124472,8 @@ - - + + @@ -124214,9 +124493,9 @@ - - - + + + @@ -124226,62 +124505,62 @@ - - + + - - - + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -124299,14 +124578,14 @@ - - - + + + - - - + + + @@ -124341,13 +124620,13 @@ - - - - - - - + + + + + + + @@ -124571,62 +124850,62 @@ - - + + - - - + + + - - + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + @@ -124642,15 +124921,15 @@ - - + + - - - - - + + + + + @@ -124661,10 +124940,10 @@ - - - - + + + + @@ -124693,8 +124972,8 @@ - - + + @@ -124702,9 +124981,9 @@ - - - + + + @@ -124741,9 +125020,9 @@ - - - + + + @@ -124752,8 +125031,8 @@ - - + + @@ -124761,19 +125040,19 @@ - - - - - + + + + + - - - - - - + + + + + + @@ -124781,20 +125060,20 @@ - - + + - - + + - - + + - - + + @@ -124807,17 +125086,17 @@ - - + + - - + + - - - + + + @@ -124864,19 +125143,19 @@ - - - - - - - + + + + + + + - - - + + + @@ -124884,17 +125163,17 @@ - - - + + + - - + + @@ -124912,20 +125191,20 @@ - - + + - - + + - - + + @@ -124938,11 +125217,11 @@ - - - - - + + + + + @@ -124953,33 +125232,33 @@ - - - + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -125263,23 +125542,23 @@ - - + + - - - - + + + + - - - + + + @@ -125295,12 +125574,12 @@ - - - - - - + + + + + + @@ -125352,20 +125631,20 @@ - - + + - - - - + + + + - - - - + + + + @@ -125520,10 +125799,10 @@ - - - - + + + + @@ -125573,9 +125852,9 @@ - - - + + + @@ -125704,19 +125983,19 @@ - - - + + + - - - + + + - - - + + + @@ -125817,8 +126096,8 @@ - - + + @@ -125840,9 +126119,9 @@ - - - + + + @@ -125870,8 +126149,8 @@ - - + + @@ -125886,24 +126165,24 @@ - - + + - - - + + + - - - - + + + + - - - + + + @@ -125922,9 +126201,9 @@ - - - + + + @@ -125938,18 +126217,18 @@ - - + + - - - + + + - - - + + + @@ -125970,9 +126249,9 @@ - - - + + + @@ -126003,28 +126282,28 @@ - - + + - - + + - - - + + + - - - - + + + + - - - + + + @@ -126035,83 +126314,83 @@ - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - - + + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + @@ -126167,17 +126446,17 @@ - - + + - - + + - - - + + + @@ -126191,20 +126470,20 @@ - - + + - - + + - - + + @@ -126273,9 +126552,9 @@ - - - + + + @@ -126283,44 +126562,44 @@ - - + + - - - - - + + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + @@ -126330,12 +126609,12 @@ - - - - - - + + + + + + @@ -126346,9 +126625,9 @@ - - - + + + @@ -126356,18 +126635,18 @@ - - - - + + + + - - - - - - - + + + + + + + @@ -126382,30 +126661,30 @@ - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + @@ -126441,8 +126720,8 @@ - - + + @@ -126518,20 +126797,20 @@ - - + + - - - - - - - - - - + + + + + + + + + + @@ -126540,19 +126819,19 @@ - - - + + + - - - - + + + + - - - + + + @@ -126579,40 +126858,40 @@ - - - - + + + + - - + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -126851,8 +127130,8 @@ - - + + @@ -126866,13 +127145,13 @@ - - + + - - - + + + @@ -126889,10 +127168,10 @@ - - - - + + + + @@ -126925,14 +127204,14 @@ - - - + + + - - - + + + @@ -126940,17 +127219,17 @@ - - - - + + + + - - + + @@ -126965,9 +127244,9 @@ - - - + + + @@ -126981,50 +127260,50 @@ - - - - + + + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + - - + + - - - + + + - - + + @@ -127034,15 +127313,15 @@ - - - + + + - - - - + + + + @@ -127075,18 +127354,18 @@ - - - - + + + + - - - - - + + + + + @@ -127094,13 +127373,13 @@ - - - + + + - - + + @@ -127130,12 +127409,12 @@ - - + + - - + + @@ -127144,6 +127423,13 @@ + + + + + + + @@ -127162,21 +127448,27 @@ + + + + + + - - - + + + - - - - + + + + @@ -127185,30 +127477,30 @@ - - - + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -127224,26 +127516,26 @@ - - + + - - - + + + - - + + - - - + + + @@ -127260,11 +127552,11 @@ - - + + - + @@ -127279,8 +127571,8 @@ - - + + @@ -127328,18 +127620,18 @@ - - - + + + - - + + - - - + + + @@ -127347,13 +127639,13 @@ - - + + - - - + + + @@ -127361,9 +127653,9 @@ - - - + + + @@ -127427,8 +127719,8 @@ - - + + @@ -127437,9 +127729,9 @@ - - - + + + @@ -127452,8 +127744,8 @@ - - + + @@ -127496,12 +127788,12 @@ - - + + - - + + @@ -127538,35 +127830,35 @@ - - + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - + + + @@ -127578,13 +127870,13 @@ - - - + + + - - + + @@ -127627,33 +127919,37 @@ - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - + + + - - - - + + + + + + + + @@ -127702,13 +127998,13 @@ - - - + + + - - + + @@ -127740,10 +128036,10 @@ - - - - + + + + @@ -127753,12 +128049,12 @@ - - + + - - + + @@ -127769,17 +128065,17 @@ - - - - - - - + + + + + + + - - + + @@ -127796,8 +128092,8 @@ - - + + @@ -127827,15 +128123,15 @@ - - + + - - - - - + + + + + @@ -127847,8 +128143,8 @@ - - + + @@ -127874,8 +128170,8 @@ - - + + @@ -127885,20 +128181,20 @@ - - + + - + - - - + + + @@ -127912,41 +128208,41 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -127956,13 +128252,13 @@ - - + + - - - + + + @@ -127976,41 +128272,41 @@ - - - - - - + + + + + + - - + + - - - - - - + + + + + + - - - - - + + + + + - - - - - - - - + + + + + + + + @@ -128024,8 +128320,8 @@ - - + + @@ -128037,18 +128333,18 @@ - - + + - - - - + + + + @@ -128072,23 +128368,23 @@ - - - + + + - - - + + + - - - + + + - - + + @@ -128101,9 +128397,9 @@ - - - + + + @@ -128111,29 +128407,29 @@ - - - - + + + + - - + + - - - - - + + + + + - - - - - - + + + + + + @@ -128166,9 +128462,9 @@ - - - + + + @@ -128177,13 +128473,13 @@ - - + + - - - + + + @@ -128280,8 +128576,8 @@ - - + + @@ -128312,13 +128608,13 @@ - - - + + + - - + + @@ -128363,10 +128659,10 @@ - - + + - + @@ -128381,22 +128677,22 @@ - - - + + + - - - + + + - - + + - - + + @@ -128447,38 +128743,38 @@ - - - - + + + + - - - - - - - - + + + + + + + + - - + + - - + + - + - - + + @@ -128489,9 +128785,9 @@ - - - + + + @@ -128506,26 +128802,26 @@ - - + + - - - - + + + + - - - - - - + + + + + + - - + + @@ -128540,28 +128836,28 @@ - - + + - - - + + + - - + + - - - - - + + + + + - - - - + + + + @@ -128576,28 +128872,28 @@ - - - - - + + + + + - - + + - - - - - + + + + + - - - - + + + + @@ -128619,18 +128915,18 @@ - - + + - - + + - - + + @@ -128653,12 +128949,12 @@ - - + + - - + + @@ -128695,9 +128991,9 @@ - - - + + + @@ -128732,81 +129028,81 @@ - - - + + + - - - + + + - - - - + + + + - - - - - - + + + + + + - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -128936,12 +129232,12 @@ - - + + - - + + @@ -128984,14 +129280,14 @@ - - + + - - - - + + + + @@ -129012,36 +129308,36 @@ - - - - - + + + + + - - + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -129052,39 +129348,39 @@ - - + + - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - - - + + + + @@ -129110,14 +129406,14 @@ - - + + - - - - + + + + @@ -129129,12 +129425,12 @@ - - + + - - + + @@ -129150,8 +129446,8 @@ - - + + @@ -129273,21 +129569,21 @@ - - - - - + + + + + - - - - - - - - + + + + + + + + @@ -129297,8 +129593,8 @@ - - + + @@ -129321,8 +129617,8 @@ - - + + @@ -129330,9 +129626,9 @@ - - - + + + @@ -129352,24 +129648,24 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -129377,46 +129673,46 @@ - - - - - - + + + + + + - - - + + + - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - - + + + @@ -129432,8 +129728,8 @@ - - + + @@ -129454,8 +129750,8 @@ - - + + @@ -129470,8 +129766,8 @@ - - + + @@ -129504,20 +129800,20 @@ - - + + - - + + - - + + @@ -129554,12 +129850,12 @@ - - + + - - + + @@ -129567,34 +129863,34 @@ - - + + - - - + + + - - - - + + + + - - - - + + + + - - - + + + - - + + @@ -129679,9 +129975,9 @@ - - - + + + @@ -129842,74 +130138,74 @@ - - - + + + - - + + - - - + + + - - + + - - - - - - + + + + + + - - - - - - + + + + + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - + + + - - - - + + + + - - - - + + + + @@ -129917,53 +130213,53 @@ - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - - - - + + + + + + - - - - - + + + + + @@ -129998,8 +130294,8 @@ - - + + @@ -130099,13 +130395,13 @@ - - - + + + - - + + @@ -130116,15 +130412,15 @@ - - - - - - - - - + + + + + + + + + @@ -130160,9 +130456,9 @@ - - - + + + @@ -130188,30 +130484,30 @@ - - - + + + - - - - + + + + - - - - + + + + - - - + + + @@ -130224,10 +130520,10 @@ - - - - + + + + @@ -130235,11 +130531,11 @@ - - - - - + + + + + @@ -130262,10 +130558,10 @@ - - - - + + + + @@ -130278,20 +130574,20 @@ - - - + + + - - - - + + + + - + - - + + @@ -130305,9 +130601,9 @@ - - - + + + @@ -130409,14 +130705,14 @@ - - - + + + - - + + @@ -130503,8 +130799,8 @@ - - + + @@ -130514,56 +130810,56 @@ - - + + - - - - - - - - - + + + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - + + - - + + - - + + - - + + - + @@ -130574,10 +130870,10 @@ - - - - + + + + @@ -130603,21 +130899,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -130628,35 +130924,35 @@ - - - - - - - - + + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -130688,46 +130984,46 @@ - - - - + + + + - - - + + + - - + + - - + + - - - + + + - - - - - + + + + + - - + + - - - + + + @@ -130736,31 +131032,35 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + + + + + @@ -130769,23 +131069,23 @@ - - - + + + - - - + + + - - - + + + @@ -130793,34 +131093,34 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - + + + + - - - - - + + + + + @@ -130830,16 +131130,16 @@ - - - - + + + + - - - - + + + + @@ -130855,10 +131155,10 @@ - - - - + + + + @@ -130868,10 +131168,10 @@ - - - - + + + + @@ -130881,37 +131181,37 @@ - - - - + + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -130922,9 +131222,9 @@ - - - + + + @@ -130938,11 +131238,11 @@ - - - - - + + + + + @@ -130978,11 +131278,11 @@ - - + + - + @@ -130999,15 +131299,15 @@ - - + + - - + + - - + + @@ -131027,8 +131327,8 @@ - - + + @@ -131038,8 +131338,8 @@ - - + + @@ -131051,8 +131351,8 @@ - - + + @@ -131066,18 +131366,18 @@ - - - + + + - - - - + + + + - - + + @@ -131085,10 +131385,10 @@ - - - - + + + + @@ -131101,14 +131401,14 @@ - - - - + + + + - - + + @@ -131144,18 +131444,18 @@ - - - + + + - - - + + + @@ -131166,8 +131466,8 @@ - - + + @@ -131178,10 +131478,10 @@ - - - - + + + + @@ -131191,30 +131491,30 @@ - - - - - + + + + + - - + + - - + + - - - + + + - - + + @@ -131233,11 +131533,11 @@ - - - - - + + + + + @@ -131245,13 +131545,13 @@ - - + + - - + + @@ -131268,11 +131568,11 @@ - - - - - + + + + + @@ -131311,8 +131611,8 @@ - - + + @@ -131320,21 +131620,25 @@ - - - + + + - - - - + + + + + + + + - - - - + + + + @@ -131347,9 +131651,9 @@ - - - + + + @@ -131360,13 +131664,13 @@ - - + + - - - + + + @@ -131374,50 +131678,50 @@ - - - + + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -131431,8 +131735,8 @@ - - + + @@ -131508,9 +131812,9 @@ - - - + + + @@ -131548,32 +131852,32 @@ - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - + @@ -131586,9 +131890,9 @@ - - - + + + @@ -131668,10 +131972,10 @@ - - - - + + + + @@ -131699,13 +132003,30 @@ - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + @@ -131892,12 +132213,12 @@ - - + + - - + + @@ -131909,11 +132230,11 @@ - - - - - + + + + + @@ -131954,8 +132275,8 @@ - - + + @@ -132008,8 +132329,8 @@ - - + + @@ -132021,9 +132342,9 @@ - - - + + + @@ -132034,28 +132355,28 @@ - - - + + + - - - - - - + + + + + + - - + + - - - - - + + + + + @@ -132066,9 +132387,9 @@ - - - + + + @@ -132129,8 +132450,8 @@ - - + + @@ -132169,9 +132490,9 @@ - - - + + + @@ -132227,8 +132548,8 @@ - - + + @@ -132250,12 +132571,12 @@ - - + + - - + + @@ -132289,8 +132610,8 @@ - - + + @@ -132301,9 +132622,9 @@ - - - + + + @@ -132311,47 +132632,47 @@ - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - - - - + + + + + @@ -132359,18 +132680,18 @@ - - - - + + + + - - + + @@ -132378,14 +132699,14 @@ - - - - + + + + - - - + + + @@ -132421,17 +132742,17 @@ - - + + - - - + + + @@ -132441,9 +132762,9 @@ - - - + + + @@ -132485,12 +132806,12 @@ - - + + - - + + @@ -132504,67 +132825,67 @@ - - - + + + - - + + - - + + - - + + - - + + - - - - - + + + + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + @@ -132572,19 +132893,19 @@ - - + + - - + + - - - - - + + + + + @@ -132595,41 +132916,41 @@ - - - - + + + + - - - + + + - - + + - - - - - + + + + + - - + + - - + + @@ -132643,16 +132964,16 @@ - - - - + + + + - - - - + + + + @@ -132673,11 +132994,11 @@ - - - - - + + + + + @@ -132694,24 +133015,24 @@ - - - - + + + + - - - - - + + + + + - - - - - + + + + + @@ -132736,10 +133057,10 @@ - - - - + + + + @@ -132752,12 +133073,12 @@ - - + + - - + + @@ -132782,18 +133103,18 @@ - - - + + + - - + + - - - + + + @@ -132817,18 +133138,18 @@ - - - - - + + + + + - - - - - + + + + + @@ -132839,19 +133160,19 @@ - - - - + + + + - - + + - - - + + + @@ -132870,14 +133191,14 @@ - - - + + + - - - + + + @@ -132903,9 +133224,9 @@ - - - + + + @@ -132918,22 +133239,22 @@ - - - - + + + + - - + + - - + + @@ -132947,43 +133268,43 @@ - - + + - - - + + + - - - + + + - - - - - + + + + + - - - + + + - - - - - + + + + + @@ -132996,9 +133317,9 @@ - - - + + + @@ -133031,14 +133352,14 @@ - - - - + + + + - - + + @@ -133049,44 +133370,44 @@ - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -133120,16 +133441,16 @@ - - - - - - + + + + + + - - - + + + @@ -133137,9 +133458,9 @@ - - - + + + @@ -133183,33 +133504,33 @@ - - - + + + - - - + + + - - + + - - + + - - - + + + - - - - + + + + @@ -133234,9 +133555,9 @@ - - - + + + @@ -133252,32 +133573,32 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + - - - + + + @@ -133286,9 +133607,9 @@ - - - + + + @@ -133296,74 +133617,74 @@ - - - + + + - - + + - - - + + + - - - + + + - - - - + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + - - + + - - + + - - - - + + + + @@ -133375,42 +133696,42 @@ - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - + + @@ -133425,8 +133746,8 @@ - - + + @@ -133446,8 +133767,8 @@ - - + + @@ -133461,61 +133782,61 @@ - - - - - + + + + + - - - + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - + + + + @@ -133533,12 +133854,12 @@ - - - - - - + + + + + + @@ -133549,12 +133870,12 @@ - - - - - - + + + + + + @@ -133569,9 +133890,9 @@ - - - + + + @@ -133584,15 +133905,15 @@ - - + + - - - - - + + + + + @@ -133609,10 +133930,10 @@ - - - - + + + + @@ -133632,10 +133953,10 @@ - - - - + + + + @@ -133675,22 +133996,40 @@ + + + + + + + + + + + + - - - + + + + + - - + + + + + + - - + + @@ -133701,19 +134040,45 @@ + + + + + + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -133721,16 +134086,16 @@ - - - - - - + + + + + + - - + + @@ -133833,33 +134198,33 @@ - - + + - - + + - - - - - - + + + + + + - - - + + + - - + + @@ -133885,8 +134250,8 @@ - - + + @@ -133901,16 +134266,16 @@ - - + + - - + + - - + + @@ -133995,19 +134360,19 @@ - - - + + + - - - + + + - - - + + + @@ -134022,8 +134387,8 @@ - - + + @@ -134054,9 +134419,9 @@ - - - + + + @@ -134086,14 +134451,14 @@ - - - + + + - - - + + + @@ -134106,9 +134471,9 @@ - - - + + + @@ -134116,10 +134481,10 @@ - - - - + + + + @@ -134133,12 +134498,12 @@ - - + + - - + + @@ -134159,9 +134524,9 @@ - - - + + + @@ -134173,9 +134538,9 @@ - - - + + + @@ -134223,8 +134588,8 @@ - - + + @@ -134270,9 +134635,9 @@ - - - + + + @@ -134288,10 +134653,10 @@ - - - - + + + + @@ -134312,10 +134677,10 @@ - - - - + + + + @@ -134348,17 +134713,17 @@ - - + + - - - + + + @@ -134372,40 +134737,55 @@ - - + + + + - - + + - - - - - + + + + + + + + + + + + + + + + + + - - - + + + - - - + + + - - + + - - + + - - + + @@ -134427,20 +134807,20 @@ - - - - - - + + + + + + - - + + - - + + @@ -134448,21 +134828,21 @@ - - - - - + + + + + - - - - - + + + + + - - + + @@ -134473,7 +134853,7 @@ - + @@ -134492,21 +134872,21 @@ - - - - + + + + - - - - - - - - - + + + + + + + + + @@ -134540,8 +134920,8 @@ - - + + @@ -134561,9 +134941,9 @@ - - - + + + @@ -134571,23 +134951,23 @@ - - - + + + - - - + + + - - - + + + @@ -134608,37 +134988,37 @@ - - + + - - - - + + + + - - - + + + - - + + - - - - + + + + @@ -134652,11 +135032,11 @@ - - - - - + + + + + @@ -134666,12 +135046,12 @@ - - - - - - + + + + + + @@ -134715,10 +135095,10 @@ - - - - + + + + @@ -134777,24 +135157,24 @@ - - + + - - + + - - + + @@ -134804,11 +135184,11 @@ - - - - - + + + + + @@ -134853,14 +135233,14 @@ - - - - + + + + - - - + + + @@ -135093,8 +135473,8 @@ - - + + @@ -135114,29 +135494,29 @@ - - - + + + - - + + - - + + - - + + - - + + - - + + @@ -135148,14 +135528,14 @@ - - - + + + - - - + + + @@ -135168,16 +135548,16 @@ - - - - + + + + - - - - + + + + @@ -135229,12 +135609,12 @@ - - + + - - + + @@ -135281,24 +135661,24 @@ - - + + - - + + - - + + - - + + @@ -135318,13 +135698,13 @@ - - + + - - - + + + @@ -135338,16 +135718,16 @@ - - - - + + + + - - - - + + + + @@ -135368,8 +135748,8 @@ - - + + @@ -135384,8 +135764,8 @@ - - + + @@ -135419,8 +135799,8 @@ - - + + @@ -135438,10 +135818,10 @@ - - - - + + + + @@ -135454,54 +135834,54 @@ - - + + - - - + + + - - + + - + - + - - - - + + + + - - - - + + + + - - - - - - - - - + + + + + + + + + - - + + - - + + @@ -135510,36 +135890,36 @@ - - - - - + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + @@ -135550,12 +135930,12 @@ - - - - - - + + + + + + @@ -135574,9 +135954,9 @@ - - - + + + @@ -135586,8 +135966,8 @@ - - + + @@ -135601,10 +135981,10 @@ - - - - + + + + @@ -135617,13 +135997,13 @@ - - - + + + - - + + @@ -135676,11 +136056,11 @@ - - - - - + + + + + @@ -135696,24 +136076,24 @@ - - + + - - + + - - + + - - + + @@ -135721,24 +136101,24 @@ - - + + - - + + - - + + @@ -135798,8 +136178,8 @@ - - + + @@ -135809,21 +136189,28 @@ - - - - - + + + + + - - - - + + + + + + + + + + + @@ -135837,21 +136224,25 @@ - - - + + + - - + + - - - - + + + + + + + + @@ -135872,48 +136263,48 @@ - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + @@ -135922,7 +136313,7 @@ - + @@ -135962,17 +136353,17 @@ - + - + - - + + @@ -135991,8 +136382,8 @@ - - + + @@ -136018,13 +136409,19 @@ + + + + + + - - + + @@ -136036,23 +136433,23 @@ - - - - + + + + - - - + + + - - - + + + @@ -136112,8 +136509,8 @@ - - + + @@ -136154,16 +136551,16 @@ - - + + - - + + - - + + @@ -136174,8 +136571,8 @@ - - + + @@ -136198,12 +136595,12 @@ - - + + - - + + @@ -136219,8 +136616,8 @@ - - + + @@ -136339,10 +136736,10 @@ - - - - + + + + @@ -136351,20 +136748,20 @@ - - - - - - - - + + + + + + + + - - - - + + + + @@ -136372,14 +136769,14 @@ - - - + + + - - - + + + @@ -136397,8 +136794,8 @@ - - + + @@ -136410,8 +136807,8 @@ - - + + @@ -136448,8 +136845,8 @@ - - + + @@ -136500,22 +136897,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -136605,8 +137002,8 @@ - - + + @@ -136614,22 +137011,22 @@ - - + + - - - - - - + + + + + + - - - - + + + + @@ -136642,9 +137039,9 @@ - - - + + + @@ -136652,9 +137049,9 @@ - - - + + + @@ -136702,13 +137099,13 @@ - - - - - - - + + + + + + + @@ -136716,28 +137113,28 @@ - - + + - - + + - - + + - - + + - - + + @@ -136753,20 +137150,20 @@ - - - - - - + + + + + + - - - + + + - - + + @@ -136782,8 +137179,8 @@ - - + + @@ -136800,8 +137197,8 @@ - - + + @@ -136814,15 +137211,15 @@ - - - + + + - - - - + + + + @@ -136830,39 +137227,39 @@ - - - + + + - - - - + + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + @@ -136870,30 +137267,30 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -137020,8 +137417,8 @@ - - + + @@ -137145,12 +137542,12 @@ - - + + - - + + @@ -137167,9 +137564,9 @@ - - - + + + @@ -137177,9 +137574,9 @@ - - - + + + @@ -137187,9 +137584,9 @@ - - - + + + @@ -137214,12 +137611,12 @@ - - + + - - + + @@ -137237,10 +137634,10 @@ - - - - + + + + @@ -137262,67 +137659,67 @@ - - - + + + - - - - + + + + - - - + + + - - + + - - - - - - + + + + + + - - - - + + + + - - - - - - - + + + + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -137333,13 +137730,13 @@ - + - - - - + + + + @@ -137373,10 +137770,10 @@ - - - - + + + + @@ -137418,7 +137815,7 @@ - + @@ -137426,34 +137823,34 @@ - - + + - - + + - - + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -137464,10 +137861,10 @@ - - - - + + + + @@ -137477,8 +137874,8 @@ - - + + @@ -137489,8 +137886,8 @@ - - + + @@ -137528,8 +137925,8 @@ - - + + @@ -137567,14 +137964,14 @@ - - - - - + + + + + - - + + @@ -137612,11 +138009,11 @@ - - - - - + + + + + @@ -137760,21 +138157,21 @@ - - + + - - - + + + - - + + - - + + @@ -137808,11 +138205,11 @@ - - - - - + + + + + @@ -137827,21 +138224,21 @@ - - + + - - + + - - + + - - - + + + @@ -137857,11 +138254,11 @@ - - - - - + + + + + @@ -137885,12 +138282,12 @@ - - - - - - + + + + + + @@ -137910,21 +138307,21 @@ - - - - + + + + - - - - + + + + - - - + + + @@ -137945,26 +138342,26 @@ - - - - + + + + - - - + + + - - - - - + + + + + - - - + + + @@ -137973,20 +138370,20 @@ - - - + + + - - - + + + - - - + + + @@ -137996,26 +138393,26 @@ - - - + + + - - - + + + - - - - + + + + - - - - + + + + @@ -138161,9 +138558,9 @@ - - - + + + @@ -138172,9 +138569,9 @@ - - - + + + @@ -138182,26 +138579,26 @@ - - + + - - - - + + + + - - + + - - + + @@ -138209,20 +138606,20 @@ - - - + + + - - - + + + - - - - + + + + @@ -138234,33 +138631,33 @@ - - - - - - + + + + + + - - - - - - + + + + + + - - - - + + + + - - - + + + @@ -138268,28 +138665,28 @@ - - - + + + - - - - - + + + + + - - + + - - - + + + - - + + @@ -138300,13 +138697,13 @@ - - - - - - - + + + + + + + @@ -138338,14 +138735,14 @@ - - + + - - - - + + + + @@ -138353,10 +138750,10 @@ - - - - + + + + @@ -138364,10 +138761,10 @@ - - - - + + + + @@ -138390,10 +138787,10 @@ - - - - + + + + @@ -138416,29 +138813,29 @@ - + - - + + - - + + - - - - + + + + - - - - - - + + + + + + @@ -138467,13 +138864,13 @@ - - - - - - - + + + + + + + @@ -138492,10 +138889,10 @@ - - - - + + + + @@ -138512,10 +138909,10 @@ - - - - + + + + @@ -138525,30 +138922,30 @@ - - - + + + - - - - - - + + + + + + - - + + - - - - - - - + + + + + + + @@ -138565,13 +138962,13 @@ - - + + - - - + + + @@ -138590,39 +138987,39 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + - - + + @@ -138645,55 +139042,55 @@ - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -138703,9 +139100,9 @@ - - - + + + @@ -138715,16 +139112,16 @@ - - - - - + + + + + - - - + + + @@ -138759,15 +139156,15 @@ - - - + + + - - - - + + + + @@ -138780,11 +139177,11 @@ - - - - - + + + + + @@ -138794,16 +139191,16 @@ - - - - - + + + + + - - - + + + @@ -138813,8 +139210,8 @@ - - + + @@ -138826,25 +139223,25 @@ - - + + - - + + - - - - - - - + + + + + + + - - + + @@ -138855,9 +139252,9 @@ - - - + + + @@ -138909,9 +139306,9 @@ - - - + + + @@ -138945,22 +139342,22 @@ - - - - - + + + + + - - - - - + + + + + - - + + @@ -139039,8 +139436,8 @@ - - + + @@ -139050,9 +139447,9 @@ - - - + + + @@ -139104,8 +139501,8 @@ - - + + @@ -139121,9 +139518,9 @@ - - - + + + @@ -139133,17 +139530,17 @@ - - - - + + + + - - - - - + + + + + @@ -139167,17 +139564,17 @@ - - + + - - - - - - - + + + + + + + @@ -139186,23 +139583,23 @@ - - + + - - - - + + + + - - - + + + - - + + @@ -139213,12 +139610,12 @@ - - - - - - + + + + + + @@ -139230,8 +139627,8 @@ - - + + @@ -139243,18 +139640,18 @@ - - - - - - + + + + + + - - - - + + + + @@ -139262,12 +139659,12 @@ - - + + - - + + @@ -139305,9 +139702,9 @@ - - - + + + @@ -139319,8 +139716,8 @@ - - + + @@ -139352,45 +139749,45 @@ - - - + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - - + + + + + @@ -139435,9 +139832,9 @@ - - - + + + @@ -139468,10 +139865,10 @@ - - - - + + + + @@ -139563,14 +139960,14 @@ - - - - + + + + - - + + @@ -139588,8 +139985,8 @@ - - + + @@ -139840,9 +140237,9 @@ - - - + + + @@ -139931,9 +140328,9 @@ - - - + + + @@ -139954,13 +140351,13 @@ - - - - - - - + + + + + + + @@ -139972,44 +140369,44 @@ - - - + + + - - - - + + + + - - + + - - + + - - - - - - + + + + + + - - + + - - - + + + @@ -140119,9 +140516,9 @@ - - - + + + @@ -140133,17 +140530,17 @@ - - - - - - + + + + + + - - - + + + @@ -140267,9 +140664,9 @@ - - - + + + @@ -140302,34 +140699,34 @@ - - + + - - + + - - + + - - + + - - - - + + + + - - - - - - + + + + + + @@ -140352,9 +140749,9 @@ - - - + + + @@ -140372,8 +140769,8 @@ - - + + @@ -140408,10 +140805,10 @@ - - - - + + + + @@ -140420,21 +140817,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -140442,21 +140839,21 @@ - - - - + + + + - - - - - + + + + + - - - + + + @@ -140475,16 +140872,16 @@ - - - + + + - - - + + + - - + + @@ -140502,43 +140899,43 @@ - - + + - - + + - - + + - - + + - - + + - - - + + + - + - + @@ -140550,12 +140947,12 @@ - - + + - - + + @@ -140570,23 +140967,23 @@ - - + + - - + + - - - + + + - + - + @@ -140605,15 +141002,15 @@ - - - + + + - - - - + + + + @@ -140626,27 +141023,27 @@ - - - + + + - - - - + + + + - - - + + + - - - - + + + + @@ -140668,9 +141065,9 @@ - - - + + + @@ -140679,10 +141076,10 @@ - - - - + + + + @@ -140690,9 +141087,9 @@ - - - + + + @@ -140700,9 +141097,9 @@ - - - + + + @@ -140711,17 +141108,17 @@ - - - - + + + + - + @@ -140739,8 +141136,8 @@ - - + + @@ -140782,8 +141179,8 @@ - - + + @@ -140876,8 +141273,8 @@ - - + + @@ -140886,8 +141283,8 @@ - - + + @@ -140919,19 +141316,19 @@ - - - - - - - - - + + + + + + + + + - - + + @@ -140943,9 +141340,9 @@ - - - + + + @@ -140964,14 +141361,14 @@ - - - - + + + + - - + + @@ -140999,9 +141396,9 @@ - - - + + + @@ -141024,11 +141421,12 @@ + - - + + @@ -141063,19 +141461,19 @@ - - + + - - - - + + + + - - - + + + @@ -141093,8 +141491,8 @@ - - + + @@ -141104,24 +141502,24 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -141131,26 +141529,26 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + @@ -141159,10 +141557,10 @@ - - - - + + + + @@ -141179,7 +141577,7 @@ - + @@ -141191,19 +141589,31 @@ + + + + + + + + + + + + - - - + + + @@ -141341,10 +141751,10 @@ - - - - + + + + @@ -141352,10 +141762,10 @@ - - - - + + + + @@ -141434,19 +141844,19 @@ - - - + + + - - - + + + - - - + + + @@ -141476,8 +141886,8 @@ - - + + @@ -141495,16 +141905,16 @@ - - + + - - + + @@ -141557,9 +141967,9 @@ - - - + + + @@ -141573,9 +141983,9 @@ - - - + + + @@ -141614,8 +142024,8 @@ - - + + @@ -141661,9 +142071,9 @@ - - - + + + @@ -141754,11 +142164,11 @@ - - - - - + + + + + @@ -141993,9 +142403,9 @@ - - - + + + @@ -142113,27 +142523,27 @@ - - + + - - + + - - + + - - - - - + + + + + @@ -142167,8 +142577,8 @@ - - + + @@ -142207,17 +142617,17 @@ - - + + - - + + - - - + + + @@ -142228,8 +142638,8 @@ - - + + @@ -142260,12 +142670,12 @@ - - + + - - + + @@ -142288,8 +142698,8 @@ - - + + @@ -142311,8 +142721,8 @@ - - + + @@ -142348,9 +142758,9 @@ - - - + + + @@ -142373,9 +142783,9 @@ - - - + + + @@ -142390,16 +142800,16 @@ - - + + - - + + - - + + @@ -142418,8 +142828,8 @@ - - + + @@ -142434,16 +142844,16 @@ - - - - - - - - - - + + + + + + + + + + @@ -142477,8 +142887,8 @@ - - + + @@ -142493,15 +142903,15 @@ - - - - + + + + - - - + + + @@ -142512,9 +142922,9 @@ - - - + + + @@ -142526,9 +142936,9 @@ - - - + + + @@ -142547,24 +142957,24 @@ - - - + + + - - - - + + + + - - - + + + @@ -142591,8 +143001,8 @@ - - + + @@ -142611,9 +143021,9 @@ - - - + + + @@ -142622,28 +143032,28 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + @@ -142652,14 +143062,14 @@ - - - - + + + + - - + + @@ -142670,9 +143080,9 @@ - - - + + + @@ -142687,8 +143097,8 @@ - - + + @@ -142710,8 +143120,8 @@ - - + + @@ -142814,15 +143224,15 @@ - - - + + + - - - - + + + + @@ -142834,8 +143244,8 @@ - - + + @@ -142901,16 +143311,16 @@ - - + + - - + + @@ -142918,10 +143328,10 @@ - - - - + + + + @@ -142945,38 +143355,38 @@ - - + + - - + + - - - + + + - - - + + + - - + + - - + + @@ -142984,10 +143394,10 @@ - - - - + + + + @@ -143011,9 +143421,9 @@ - - - + + + @@ -143021,8 +143431,8 @@ - - + + @@ -143071,9 +143481,9 @@ - - - + + + @@ -143094,10 +143504,10 @@ - - - - + + + + @@ -143147,9 +143557,9 @@ - - - + + + @@ -143209,21 +143619,50 @@ - - - + + + - - - - - - + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -143399,9 +143838,9 @@ - - - + + + @@ -143415,27 +143854,27 @@ - - + + - - - - + + + + - - - + + + @@ -143449,33 +143888,33 @@ - - - - - + + + + + - - - + + + - - + + - - + + - - + + - - - + + + @@ -143721,15 +144160,15 @@ - - - + + + - - - - + + + + @@ -143780,15 +144219,15 @@ - - - + + + - - - + + + @@ -143801,15 +144240,15 @@ - - - - + + + + - - - + + + @@ -143887,9 +144326,9 @@ - - - + + + @@ -143905,36 +144344,36 @@ - - - + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + @@ -143949,9 +144388,9 @@ - - - + + + @@ -143966,14 +144405,14 @@ - - - + + + - - - + + + @@ -144015,15 +144454,15 @@ - - + + - - - - - + + + + + @@ -144031,11 +144470,11 @@ - - - - - + + + + + @@ -144051,8 +144490,8 @@ - - + + @@ -144072,8 +144511,8 @@ - - + + @@ -144116,8 +144555,8 @@ - - + + @@ -144153,8 +144592,8 @@ - - + + @@ -144221,9 +144660,9 @@ - - - + + + @@ -144294,26 +144733,26 @@ - - - - - - - + + + + + + + - - + + - - - + + + @@ -144350,21 +144789,21 @@ - - - + + + - - - + + + - - - + + + @@ -144418,6 +144857,9 @@ + + + @@ -144428,15 +144870,15 @@ - - + + - - + + @@ -144447,16 +144889,16 @@ - - - - - + + + + + - - - + + + @@ -144465,11 +144907,11 @@ - - - - - + + + + + @@ -144478,9 +144920,9 @@ - - - + + + @@ -144530,11 +144972,11 @@ - - - - - + + + + + @@ -144544,23 +144986,23 @@ - - - - + + + + - - - + + + - - + + - - + + @@ -144579,12 +145021,12 @@ - - + + - - + + @@ -144592,8 +145034,8 @@ - - + + @@ -144605,13 +145047,17 @@ + + + + - - + + @@ -144619,12 +145065,16 @@ - - - + + + - + + + + + @@ -144632,6 +145082,10 @@ + + + + @@ -144645,9 +145099,9 @@ - - - + + + @@ -144670,17 +145124,17 @@ - - - + + + - - + + @@ -144697,12 +145151,12 @@ - - - - - - + + + + + + @@ -144757,9 +145211,9 @@ - - - + + + @@ -144784,28 +145238,34 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - + + + + + + + + + @@ -144815,17 +145275,17 @@ - - + + - - - - + + + + @@ -144836,26 +145296,26 @@ - - - - + + + + - - - - + + + + - - + + - - - - + + + + @@ -144866,9 +145326,9 @@ - - - + + + @@ -144900,10 +145360,10 @@ - - - - + + + + @@ -144936,9 +145396,9 @@ - - - + + + @@ -144955,15 +145415,15 @@ - - - - + + + + - - - + + + @@ -144987,12 +145447,12 @@ - - - - - - + + + + + + @@ -145014,13 +145474,13 @@ - - + + - - - + + + @@ -145037,10 +145497,10 @@ - - - - + + + + @@ -145051,47 +145511,47 @@ - - - + + + - - + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index 19e78938fd20..bf898882c7b3 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -5,8 +5,10 @@ add_device_randomness add_memory add_memory_subsection + add_swap_extent add_taint add_timer + add_to_page_cache_lru add_uevent_var add_wait_queue adjust_managed_page_count @@ -148,6 +150,7 @@ class_interface_unregister __class_register class_unregister + __cleancache_get_page cleanup_srcu_struct clear_page __ClearPageMovable @@ -222,6 +225,7 @@ config_group_init_type_name config_item_get config_item_put + congestion_wait console_drivers console_printk console_stop @@ -857,6 +861,9 @@ __fdget fd_install fget + filemap_allow_speculation + filemap_check_errors + filemap_map_pages filp_close filp_open_block find_get_pid @@ -921,6 +928,7 @@ fwnode_property_read_u32_array fwnode_property_read_u64_array gcd + generate_random_uuid generic_device_group generic_file_llseek generic_handle_irq @@ -1302,6 +1310,7 @@ kobj_sysfs_ops krealloc kset_create_and_add + kset_register ksize ksoftirqd kstat @@ -1388,6 +1397,7 @@ lzorle1x_1_compress match_hex match_int + match_strdup match_string match_token mbox_chan_received_data @@ -1450,7 +1460,10 @@ __memset_io memstart_addr memunmap + migrate_page_copy + migrate_page_move_mapping migrate_pages + migrate_page_states migrate_swap mii_check_media mii_ethtool_get_link_ksettings @@ -1675,9 +1688,17 @@ on_each_cpu_cond oops_in_progress overflowuid + page_cache_ra_unbounded + page_cache_sync_ra + pagecache_write_begin + pagecache_write_end page_endio + __page_file_index + __page_file_mapping + page_get_link page_mapping __page_pinner_migration_failed + page_symlink panic panic_notifier_list panic_timeout @@ -1732,6 +1753,11 @@ pci_write_config_dword pcpu_nr_pages PDE_DATA + percpu_counter_add_batch + percpu_counter_destroy + __percpu_counter_init + percpu_counter_set + __percpu_counter_sum __per_cpu_offset per_cpu_ptr_to_phys perf_aux_output_begin @@ -1905,10 +1931,12 @@ qcom_smem_state_update_bits queue_delayed_work_on queue_work_on + radix_tree_gang_lookup radix_tree_insert radix_tree_iter_delete radix_tree_lookup radix_tree_next_chunk + radix_tree_preload ___ratelimit rational_best_approximation raw_notifier_call_chain @@ -1957,6 +1985,7 @@ rdev_get_drvdata rdev_get_id rdev_get_regmap + read_cache_page_gfp reboot_mode refcount_dec_and_lock refcount_dec_not_one @@ -2948,6 +2977,8 @@ trace_seq_printf trace_seq_putc tracing_off + truncate_inode_pages_range + truncate_pagecache_range try_module_get try_to_free_mem_cgroup_pages try_wait_for_completion @@ -3170,6 +3201,11 @@ usb_unregister_notify __usecs_to_jiffies usleep_range + utf8_casefold + utf8_load + utf8s_to_utf16s + utf8_strncasecmp_folded + utf8_unload uuid_null v4l2_async_notifier_add_fwnode_subdev v4l2_async_notifier_add_subdev @@ -3318,6 +3354,7 @@ vm_map_pages vm_map_ram vm_node_stat + vm_unmap_aliases vm_unmapped_area vm_unmap_ram vm_zone_stat @@ -3332,10 +3369,13 @@ wait_for_completion wait_for_completion_interruptible wait_for_completion_interruptible_timeout + wait_for_completion_io wait_for_completion_io_timeout wait_for_completion_killable wait_for_completion_killable_timeout wait_for_completion_timeout + wait_for_stable_page + wait_on_page_writeback wait_woken __wake_up wake_up_if_idle @@ -3359,10 +3399,12 @@ ww_mutex_lock ww_mutex_unlock __xa_alloc + __xa_clear_mark xa_destroy xa_erase xa_find xa_find_after + xa_get_mark xa_load xa_store xhci_add_endpoint From 3e45af8a72c01c7e00c26e9df6089f7412ab3ec2 Mon Sep 17 00:00:00 2001 From: Tadeusz Struk Date: Tue, 8 Mar 2022 17:20:15 -0800 Subject: [PATCH 023/156] ANDROID: incremental-fs: limit mount stack depth Syzbot recently found a number of issues related to incremental-fs (see bug numbers below). All have to do with the fact that incr-fs allows mounts of the same source and target multiple times. This is a design decision and the user space component "Data Loader" expects this to work for app re-install use case. The mounting depth needs to be controlled, however, and only allowed to be two levels deep. In case of more than two mount attempts the driver needs to return an error. In case of the issues listed below the common pattern is that the reproducer calls: mount("./file0", "./file0", "incremental-fs", 0, NULL) many times and then invokes a file operation like chmod, setxattr, or open on the ./file0. This causes a recursive call for all the mounted instances, which eventually causes a stack overflow and a kernel crash: BUG: stack guard page was hit at ffffc90000c0fff8 kernel stack overflow (double-fault): 0000 [#1] PREEMPT SMP KASAN This change also cleans up the mount error path to properly clean allocated resources and call deactivate_locked_super(), which causes the incfs_kill_sb() to be called, where the sb is freed. Bug: 211066171 Bug: 213140206 Bug: 213215835 Bug: 211914587 Bug: 211213635 Bug: 213137376 Bug: 211161296 Signed-off-by: Tadeusz Struk Change-Id: I08d9b545a2715423296bf4beb67bdbbed78d1be1 --- fs/incfs/data_mgmt.h | 6 ++++ fs/incfs/vfs.c | 79 ++++++++++++++++++++++++++++++++------------ 2 files changed, 64 insertions(+), 21 deletions(-) diff --git a/fs/incfs/data_mgmt.h b/fs/incfs/data_mgmt.h index 84db0382765d..2227913674a4 100644 --- a/fs/incfs/data_mgmt.h +++ b/fs/incfs/data_mgmt.h @@ -131,8 +131,14 @@ struct mount_info { struct path mi_backing_dir_path; struct dentry *mi_index_dir; + /* For stacking mounts, if true, this indicates if the index dir needs + * to be freed for this SB otherwise it was created by lower level SB */ + bool mi_index_free; struct dentry *mi_incomplete_dir; + /* For stacking mounts, if true, this indicates if the incomplete dir + * needs to be freed for this SB. Similar to mi_index_free */ + bool mi_incomplete_free; const struct cred *mi_owner; diff --git a/fs/incfs/vfs.c b/fs/incfs/vfs.c index ea7866fbfd6e..776640451f6f 100644 --- a/fs/incfs/vfs.c +++ b/fs/incfs/vfs.c @@ -437,7 +437,8 @@ static int incfs_init_dentry(struct dentry *dentry, struct path *path) } static struct dentry *open_or_create_special_dir(struct dentry *backing_dir, - const char *name) + const char *name, + bool *created) { struct dentry *index_dentry; struct inode *backing_inode = d_inode(backing_dir); @@ -450,6 +451,7 @@ static struct dentry *open_or_create_special_dir(struct dentry *backing_dir, return index_dentry; } else if (d_really_is_positive(index_dentry)) { /* Index already exists. */ + *created = false; return index_dentry; } @@ -469,6 +471,7 @@ static struct dentry *open_or_create_special_dir(struct dentry *backing_dir, return ERR_PTR(-EINVAL); } + *created = true; return index_dentry; } @@ -1747,6 +1750,7 @@ struct dentry *incfs_mount_fs(struct file_system_type *type, int flags, struct super_block *src_fs_sb = NULL; struct inode *root_inode = NULL; struct super_block *sb = sget(type, NULL, set_anon_super, flags, NULL); + bool dir_created = false; int error = 0; if (IS_ERR(sb)) @@ -1763,17 +1767,23 @@ struct dentry *incfs_mount_fs(struct file_system_type *type, int flags, BUILD_BUG_ON(PAGE_SIZE != INCFS_DATA_FILE_BLOCK_SIZE); + if (!dev_name) { + pr_err("incfs: Backing dir is not set, filesystem can't be mounted.\n"); + error = -ENOENT; + goto err_deactivate; + } + error = parse_options(&options, (char *)data); if (error != 0) { pr_err("incfs: Options parsing error. %d\n", error); - goto err; + goto err_deactivate; } sb->s_bdi->ra_pages = options.readahead_pages; if (!dev_name) { pr_err("incfs: Backing dir is not set, filesystem can't be mounted.\n"); error = -ENOENT; - goto err; + goto err_free_opts; } error = kern_path(dev_name, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, @@ -1782,69 +1792,80 @@ struct dentry *incfs_mount_fs(struct file_system_type *type, int flags, !d_really_is_positive(backing_dir_path.dentry)) { pr_err("incfs: Error accessing: %s.\n", dev_name); - goto err; + goto err_free_opts; } src_fs_sb = backing_dir_path.dentry->d_sb; sb->s_maxbytes = src_fs_sb->s_maxbytes; + sb->s_stack_depth = src_fs_sb->s_stack_depth + 1; + + if (sb->s_stack_depth > FILESYSTEM_MAX_STACK_DEPTH) { + error = -EINVAL; + goto err_put_path; + } mi = incfs_alloc_mount_info(sb, &options, &backing_dir_path); - if (IS_ERR_OR_NULL(mi)) { error = PTR_ERR(mi); pr_err("incfs: Error allocating mount info. %d\n", error); - mi = NULL; - goto err; + goto err_put_path; } + sb->s_fs_info = mi; + mi->mi_backing_dir_path = backing_dir_path; index_dir = open_or_create_special_dir(backing_dir_path.dentry, - INCFS_INDEX_NAME); + INCFS_INDEX_NAME, &dir_created); if (IS_ERR_OR_NULL(index_dir)) { error = PTR_ERR(index_dir); pr_err("incfs: Can't find or create .index dir in %s\n", dev_name); /* No need to null index_dir since we don't put it */ - goto err; + goto err_put_path; } + mi->mi_index_dir = index_dir; + mi->mi_index_free = dir_created; incomplete_dir = open_or_create_special_dir(backing_dir_path.dentry, - INCFS_INCOMPLETE_NAME); + INCFS_INCOMPLETE_NAME, + &dir_created); if (IS_ERR_OR_NULL(incomplete_dir)) { error = PTR_ERR(incomplete_dir); pr_err("incfs: Can't find or create .incomplete dir in %s\n", dev_name); /* No need to null incomplete_dir since we don't put it */ - goto err; + goto err_put_path; } mi->mi_incomplete_dir = incomplete_dir; + mi->mi_incomplete_free = dir_created; - sb->s_fs_info = mi; root_inode = fetch_regular_inode(sb, backing_dir_path.dentry); if (IS_ERR(root_inode)) { error = PTR_ERR(root_inode); - goto err; + goto err_put_path; } sb->s_root = d_make_root(root_inode); if (!sb->s_root) { error = -ENOMEM; - goto err; + goto err_put_path; } error = incfs_init_dentry(sb->s_root, &backing_dir_path); if (error) - goto err; + goto err_put_path; path_put(&backing_dir_path); sb->s_flags |= SB_ACTIVE; pr_debug("incfs: mount\n"); return dget(sb->s_root); -err: - sb->s_fs_info = NULL; + +err_put_path: path_put(&backing_dir_path); - incfs_free_mount_info(mi); - deactivate_locked_super(sb); +err_free_opts: free_options(&options); +err_deactivate: + deactivate_locked_super(sb); + pr_err("incfs: mount failed %d\n", error); return ERR_PTR(error); } @@ -1879,10 +1900,26 @@ out: void incfs_kill_sb(struct super_block *sb) { struct mount_info *mi = sb->s_fs_info; + struct inode *dinode = NULL; pr_debug("incfs: unmount\n"); - generic_shutdown_super(sb); - incfs_free_mount_info(mi); + + if (mi) { + if (mi->mi_backing_dir_path.dentry) + dinode = d_inode(mi->mi_backing_dir_path.dentry); + + if (dinode) { + if (mi->mi_index_dir && mi->mi_index_free) + vfs_rmdir(dinode, mi->mi_index_dir); + + if (mi->mi_incomplete_dir && mi->mi_incomplete_free) + vfs_rmdir(dinode, mi->mi_incomplete_dir); + } + + incfs_free_mount_info(mi); + sb->s_fs_info = NULL; + } + kill_anon_super(sb); } static int show_options(struct seq_file *m, struct dentry *root) From 3b25a439ce056a09b5d8ed1b98759726bfb75209 Mon Sep 17 00:00:00 2001 From: Tadeusz Struk Date: Wed, 9 Mar 2022 17:30:38 -0800 Subject: [PATCH 024/156] ANDROID: selftests: incfs: Add -fno-omit-frame-pointer Without it incfs/incfs_perf runtime fails in format_signature: malloc(): invalid size (unsorted) Aborted When compiled with gcc version 11.2.0. Also add check for NULL after the malloc, and remove unneeded space for uint32_t in signing_section. Bug: 211066171 Signed-off-by: Tadeusz Struk Change-Id: I62b775140e4b89f75335cbd65665cf6a3e0fe964 --- tools/testing/selftests/filesystems/incfs/Makefile | 4 ++-- tools/testing/selftests/filesystems/incfs/utils.c | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/filesystems/incfs/Makefile b/tools/testing/selftests/filesystems/incfs/Makefile index f3798029247a..5a2f6301c4b2 100644 --- a/tools/testing/selftests/filesystems/incfs/Makefile +++ b/tools/testing/selftests/filesystems/incfs/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -CFLAGS += -D_FILE_OFFSET_BITS=64 -Wall -Werror -I../.. -I../../../../.. -LDLIBS := -llz4 -lzstd -lcrypto -lpthread +CFLAGS += -D_FILE_OFFSET_BITS=64 -Wall -Werror -I../.. -I../../../../.. -fno-omit-frame-pointer -fsanitize=address -g +LDLIBS := -llz4 -lzstd -lcrypto -lpthread -fsanitize=address TEST_GEN_PROGS := incfs_test incfs_stress incfs_perf include ../../lib.mk diff --git a/tools/testing/selftests/filesystems/incfs/utils.c b/tools/testing/selftests/filesystems/incfs/utils.c index e60b0d36f49d..0411d912ae05 100644 --- a/tools/testing/selftests/filesystems/incfs/utils.c +++ b/tools/testing/selftests/filesystems/incfs/utils.c @@ -116,6 +116,9 @@ size_t format_signature(void **buf, const char *root_hash, const char *add_data) size_t size = sizeof(struct signature_blob) + strlen(add_data) + 1; struct signature_blob *sb = malloc(size); + if (!sb) + return 0; + *sb = (struct signature_blob){ .version = INCFS_SIGNATURE_VERSION, .hash_section_size = sizeof(struct hash_section), @@ -126,7 +129,7 @@ size_t format_signature(void **buf, const char *root_hash, const char *add_data) .salt_size = 0, .hash_size = SHA256_DIGEST_SIZE, }, - .signing_section_size = sizeof(uint32_t) + strlen(add_data) + 1, + .signing_section_size = strlen(add_data) + 1, }; memcpy(sb->hash_section.hash, root_hash, SHA256_DIGEST_SIZE); From 5b1bb43708876e57dbf63c90340403f2ba5b31a6 Mon Sep 17 00:00:00 2001 From: Tadeusz Struk Date: Wed, 9 Mar 2022 15:33:45 -0800 Subject: [PATCH 025/156] ANDROID: selftests: incfs: skip large_file_test test is not enough free space Make the large_file_test check if there is at least 3GB of free disk space and skip the test if there is not. This is to make the tests pass on a VM with limited disk size, now all functional tests are passing. TAP version 13 1..26 ok 1 basic_file_ops_test ok 2 cant_touch_index_test ok 3 dynamic_files_and_data_test ok 4 concurrent_reads_and_writes_test ok 5 attribute_test ok 6 work_after_remount_test ok 7 child_procs_waiting_for_data_test ok 8 multiple_providers_test ok 9 hash_tree_test ok 10 read_log_test ok 11 get_blocks_test ok 12 get_hash_blocks_test ok 13 large_file_test ok 14 mapped_file_test ok 15 compatibility_test ok 16 data_block_count_test ok 17 hash_block_count_test ok 18 per_uid_read_timeouts_test ok 19 inotify_test ok 20 verity_test ok 21 enable_verity_test ok 22 mmap_test ok 23 truncate_test ok 24 stat_test ok 25 sysfs_test Error mounting fs.: File exists Error mounting fs.: File exists ok 26 sysfs_rename_test Bug: 211066171 Signed-off-by: Tadeusz Struk Change-Id: I2260e2b314429251070d0163c70173f237f86476 --- .../selftests/filesystems/incfs/incfs_test.c | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/filesystems/incfs/incfs_test.c b/tools/testing/selftests/filesystems/incfs/incfs_test.c index 2d2bc933fefc..bfd5f1ac7c7a 100644 --- a/tools/testing/selftests/filesystems/incfs/incfs_test.c +++ b/tools/testing/selftests/filesystems/incfs/incfs_test.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -43,6 +44,7 @@ #define FS_IOC_GETFLAGS _IOR('f', 1, long) #define FS_VERITY_FL 0x00100000 /* Verity protected inode */ +#define TEST_SKIP 2 #define TEST_FAILURE 1 #define TEST_SUCCESS 0 @@ -2793,14 +2795,16 @@ failure: return TEST_FAILURE; } +#define THREE_GB (3LL * 1024 * 1024 * 1024) +#define FOUR_GB (4LL * 1024 * 1024 * 1024) /* Have 1GB of margin */ static int large_file_test(const char *mount_dir) { char *backing_dir; int cmd_fd = -1; int i; - int result = TEST_FAILURE; + int result = TEST_FAILURE, ret; uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE] = {}; - int block_count = 3LL * 1024 * 1024 * 1024 / INCFS_DATA_FILE_BLOCK_SIZE; + int block_count = THREE_GB / INCFS_DATA_FILE_BLOCK_SIZE; struct incfs_fill_block *block_buf = calloc(block_count, sizeof(struct incfs_fill_block)); struct incfs_fill_blocks fill_blocks = { @@ -2809,6 +2813,22 @@ static int large_file_test(const char *mount_dir) }; incfs_uuid_t id; int fd = -1; + struct statvfs svfs; + unsigned long long free_disksz; + + ret = statvfs(mount_dir, &svfs); + if (ret) { + ksft_print_msg("Can't get disk size. Skipping %s...\n", __func__); + return TEST_SKIP; + } + + free_disksz = (unsigned long long)svfs.f_bavail * svfs.f_bsize; + + if (FOUR_GB > free_disksz) { + ksft_print_msg("Not enough free disk space (%lldMB). Skipping %s...\n", + free_disksz >> 20, __func__); + return TEST_SKIP; + } backing_dir = create_backing_dir(mount_dir); if (!backing_dir) @@ -2849,6 +2869,7 @@ static int large_file_test(const char *mount_dir) failure: close(fd); close(cmd_fd); + unlink("very_large_file"); umount(mount_dir); free(backing_dir); return result; @@ -4665,9 +4686,15 @@ struct test_case { void run_one_test(const char *mount_dir, struct test_case *test_case) { + int ret; + ksft_print_msg("Running %s\n", test_case->name); - if (test_case->pfunc(mount_dir) == TEST_SUCCESS) + ret = test_case->pfunc(mount_dir); + + if (ret == TEST_SUCCESS) ksft_test_result_pass("%s\n", test_case->name); + else if (ret == TEST_SKIP) + ksft_test_result_skip("%s\n", test_case->name); else ksft_test_result_fail("%s\n", test_case->name); } From 3c06a5ce5e5857a1dff88a784e58e32bbddaf759 Mon Sep 17 00:00:00 2001 From: Jaskaran Singh Date: Wed, 6 Apr 2022 10:57:39 +0530 Subject: [PATCH 026/156] ANDROID: Split ANDROID_STRUCT_PADDING into separate configs Not all non-GKI platforms support disabling ANDROID_STRUCT_PADDING, as some modules may require Android vendor data. However, it would be beneficial to have the option to disable some of the struct paddings, such as ANDROID_KABI_RESERVE, for memory savings given a situation where the ANDROID_STRUCT_PADDING config cannot be disabled. Split the ANDROID_STRUCT_PADDING config into two configs, one to control ANDROID_VENDOR_DATA and ANDROID_OEM_DATA, and another to control ANDROID_KABI_RESERVE. Bug: 206561931 Change-Id: Iea4b962dff386a17c9bef20ae048be4e17bf43ab Signed-off-by: Jaskaran Singh --- drivers/android/Kconfig | 24 +++++++++++++++++++++--- include/linux/android_kabi.h | 2 +- include/linux/android_vendor.h | 2 +- init/init_task.c | 2 +- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig index c9636c3c6dbd..491751ab0dbf 100644 --- a/drivers/android/Kconfig +++ b/drivers/android/Kconfig @@ -74,14 +74,32 @@ config ANDROID_VENDOR_HOOKS Allow vendor modules to attach to tracepoint "hooks" defined via DECLARE_HOOK or DECLARE_RESTRICTED_HOOK. -config ANDROID_STRUCT_PADDING - bool "Android Struct Padding" +config ANDROID_KABI_RESERVE + bool "Android KABI reserve padding" + default y + help + This option enables the padding that the Android GKI kernel adds + to many different kernel structures to support an in-kernel stable ABI + over the lifespan of support for the kernel. + + Only disable this option if you have a system that needs the Android + kernel drivers, but is NOT an Android GKI kernel image. If disabled + it has the possibility to make the kernel static and runtime image + slightly smaller but will NOT be supported by the Google Android + kernel team. + + If even slightly unsure, say Y. + +config ANDROID_VENDOR_OEM_DATA + bool "Android vendor and OEM data padding" default y help This option enables the padding that the Android GKI kernel adds to many different kernel structures to support an in-kernel stable ABI over the lifespan of support for the kernel as well as OEM additional - fields that are needed by some of the Android kernel tracepoints. + fields that are needed by some of the Android kernel tracepoints. The + macros enabled by this option are used to enable padding in vendor modules + used for the above specified purposes. Only disable this option if you have a system that needs the Android kernel drivers, but is NOT an Android GKI kernel image and you do NOT diff --git a/include/linux/android_kabi.h b/include/linux/android_kabi.h index dc0da1ab45d6..f6dd7f00b386 100644 --- a/include/linux/android_kabi.h +++ b/include/linux/android_kabi.h @@ -83,7 +83,7 @@ * number: the "number" of the padding variable in the structure. Start with * 1 and go up. */ -#ifdef CONFIG_ANDROID_STRUCT_PADDING +#ifdef CONFIG_ANDROID_KABI_RESERVE #define ANDROID_KABI_RESERVE(number) _ANDROID_KABI_RESERVE(number) #else #define ANDROID_KABI_RESERVE(number) diff --git a/include/linux/android_vendor.h b/include/linux/android_vendor.h index ab3dca23966a..af3014ccc82e 100644 --- a/include/linux/android_vendor.h +++ b/include/linux/android_vendor.h @@ -26,7 +26,7 @@ * Same as ANDROID_VENDOR_DATA but allocates an array of u64 with * the specified size */ -#ifdef CONFIG_ANDROID_STRUCT_PADDING +#ifdef CONFIG_ANDROID_VENDOR_OEM_DATA #define ANDROID_VENDOR_DATA(n) u64 android_vendor_data##n #define ANDROID_VENDOR_DATA_ARRAY(n, s) u64 android_vendor_data##n[s] diff --git a/init/init_task.c b/init/init_task.c index 26e018fbbf78..9b8ad6861316 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -213,7 +213,7 @@ struct task_struct init_task #ifdef CONFIG_SECCOMP_FILTER .seccomp = { .filter_count = ATOMIC_INIT(0) }, #endif -#ifdef CONFIG_ANDROID_STRUCT_PADDING +#ifdef CONFIG_ANDROID_VENDOR_OEM_DATA .android_vendor_data1 = {0, }, .android_oem_data1 = {0, }, #endif From eff1ffbf0c097512fe2c92462b6183f716afb770 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 7 Apr 2022 19:58:31 +0200 Subject: [PATCH 027/156] ANDROID: GKI: set more vfs-only exports into their own namespace There are more vfs-only symbols that OEMs want to use, so place them in the proper vfs-only namespace. Bug: 157965270 Bug: 210074446 Bug: 227656251 Cc: Matthias Maennich Signed-off-by: Greg Kroah-Hartman Change-Id: I99b9facc8da45fb329f6627d204180d1f89bcf97 --- fs/crypto/Makefile | 2 ++ fs/fs_types.c | 2 +- fs/fscache/Makefile | 2 ++ fs/jbd2/Makefile | 2 ++ fs/verity/Makefile | 2 ++ 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fs/crypto/Makefile b/fs/crypto/Makefile index 652c7180ec6d..d39077502e06 100644 --- a/fs/crypto/Makefile +++ b/fs/crypto/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_FS_ENCRYPTION) += fscrypto.o +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=ANDROID_GKI_VFS_EXPORT_ONLY + fscrypto-y := crypto.o \ fname.o \ hkdf.o \ diff --git a/fs/fs_types.c b/fs/fs_types.c index 78365e5dc08c..a11a1d8c7811 100644 --- a/fs/fs_types.c +++ b/fs/fs_types.c @@ -41,7 +41,7 @@ unsigned char fs_ftype_to_dtype(unsigned int filetype) return fs_dtype_by_ftype[filetype]; } -EXPORT_SYMBOL_GPL(fs_ftype_to_dtype); +EXPORT_SYMBOL_NS_GPL(fs_ftype_to_dtype, ANDROID_GKI_VFS_EXPORT_ONLY); /* * dirent file type to fs on-disk file type conversion diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile index 79e08e05ef84..9121382cf16d 100644 --- a/fs/fscache/Makefile +++ b/fs/fscache/Makefile @@ -3,6 +3,8 @@ # Makefile for general filesystem caching code # +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=ANDROID_GKI_VFS_EXPORT_ONLY + fscache-y := \ cache.o \ cookie.o \ diff --git a/fs/jbd2/Makefile b/fs/jbd2/Makefile index 126b4da6c7de..b64f93331643 100644 --- a/fs/jbd2/Makefile +++ b/fs/jbd2/Makefile @@ -3,6 +3,8 @@ # Makefile for the linux journaling routines. # +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=ANDROID_GKI_VFS_EXPORT_ONLY + obj-$(CONFIG_JBD2) += jbd2.o jbd2-objs := transaction.o commit.o recovery.o checkpoint.o revoke.o journal.o diff --git a/fs/verity/Makefile b/fs/verity/Makefile index 435559a4fa9e..4b8323450f90 100644 --- a/fs/verity/Makefile +++ b/fs/verity/Makefile @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=ANDROID_GKI_VFS_EXPORT_ONLY + obj-$(CONFIG_FS_VERITY) += enable.o \ hash_algs.o \ init.o \ From 20e11d796943d53e49a0ef21d1fd96c4c94f1992 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Thu, 7 Apr 2022 18:27:08 -0600 Subject: [PATCH 028/156] ANDROID: GKI: Enable CRYPTO_DES This algorithm is still used in wifi calling scenarios. Bug: 228528489 Change-Id: I06041ec023d023cfee175bd02b6db8e6c9656518 Signed-off-by: Subash Abhinov Kasiviswanathan --- arch/arm64/configs/gki_defconfig | 1 + arch/x86/configs/gki_defconfig | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 36901bc73efd..9703932d9262 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -652,6 +652,7 @@ CONFIG_CRYPTO_ADIANTUM=y CONFIG_CRYPTO_XCBC=y CONFIG_CRYPTO_BLAKE2B=y CONFIG_CRYPTO_MD5=y +CONFIG_CRYPTO_DES=y CONFIG_CRYPTO_LZ4=y CONFIG_CRYPTO_ZSTD=y CONFIG_CRYPTO_ANSI_CPRNG=y diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index c9ce1777eb58..8525a0e365c5 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -586,6 +586,7 @@ CONFIG_CRYPTO_BLAKE2B=y CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_SHA256_SSSE3=y CONFIG_CRYPTO_AES_NI_INTEL=y +CONFIG_CRYPTO_DES=y CONFIG_CRYPTO_LZ4=y CONFIG_CRYPTO_ZSTD=y CONFIG_CRYPTO_ANSI_CPRNG=y From b5bcf0d66787c5591804e87efc24715926950b39 Mon Sep 17 00:00:00 2001 From: Lina Wang Date: Sat, 26 Feb 2022 15:48:01 +0800 Subject: [PATCH 029/156] UPSTREAM: xfrm: fix tunnel model fragmentation behavior in tunnel mode, if outer interface(ipv4) is less, it is easily to let inner IPV6 mtu be less than 1280. If so, a Packet Too Big ICMPV6 message is received. When send again, packets are fragmentized with 1280, they are still rejected with ICMPV6(Packet Too Big) by xfrmi_xmit2(). According to RFC4213 Section3.2.2: if (IPv4 path MTU - 20) is less than 1280 if packet is larger than 1280 bytes Send ICMPv6 "packet too big" with MTU=1280 Drop packet else Encapsulate but do not set the Don't Fragment flag in the IPv4 header. The resulting IPv4 packet might be fragmented by the IPv4 layer on the encapsulator or by some router along the IPv4 path. endif else if packet is larger than (IPv4 path MTU - 20) Send ICMPv6 "packet too big" with MTU = (IPv4 path MTU - 20). Drop packet. else Encapsulate and set the Don't Fragment flag in the IPv4 header. endif endif Packets should be fragmentized with ipv4 outer interface, so change it. After it is fragemtized with ipv4, there will be double fragmenation. No.48 & No.51 are ipv6 fragment packets, No.48 is double fragmentized, then tunneled with IPv4(No.49& No.50), which obey spec. And received peer cannot decrypt it rightly. 48 2002::10 2002::11 1296(length) IPv6 fragment (off=0 more=y ident=0xa20da5bc nxt=50) 49 0x0000 (0) 2002::10 2002::11 1304 IPv6 fragment (off=0 more=y ident=0x7448042c nxt=44) 50 0x0000 (0) 2002::10 2002::11 200 ESP (SPI=0x00035000) 51 2002::10 2002::11 180 Echo (ping) request 52 0x56dc 2002::10 2002::11 248 IPv6 fragment (off=1232 more=n ident=0xa20da5bc nxt=50) xfrm6_noneed_fragment has fixed above issues. Finally, it acted like below: 1 0x6206 192.168.1.138 192.168.1.1 1316 Fragmented IP protocol (proto=Encap Security Payload 50, off=0, ID=6206) [Reassembled in #2] 2 0x6206 2002::10 2002::11 88 IPv6 fragment (off=0 more=y ident=0x1f440778 nxt=50) 3 0x0000 2002::10 2002::11 248 ICMPv6 Echo (ping) request Signed-off-by: Lina Wang Signed-off-by: Steffen Klassert Bug: 226699354 Change-Id: Ideec82bea6a1efa26352680cb3113f7c36b945ef Signed-off-by: Lina Wang --- net/ipv6/xfrm6_output.c | 16 ++++++++++++++++ net/xfrm/xfrm_interface.c | 5 ++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/net/ipv6/xfrm6_output.c b/net/ipv6/xfrm6_output.c index 6abb45a67199..ee349c243878 100644 --- a/net/ipv6/xfrm6_output.c +++ b/net/ipv6/xfrm6_output.c @@ -52,6 +52,19 @@ static int __xfrm6_output_finish(struct net *net, struct sock *sk, struct sk_buf return xfrm_output(sk, skb); } +static int xfrm6_noneed_fragment(struct sk_buff *skb) +{ + struct frag_hdr *fh; + u8 prevhdr = ipv6_hdr(skb)->nexthdr; + + if (prevhdr != NEXTHDR_FRAGMENT) + return 0; + fh = (struct frag_hdr *)(skb->data + sizeof(struct ipv6hdr)); + if (fh->nexthdr == NEXTHDR_ESP || fh->nexthdr == NEXTHDR_AUTH) + return 1; + return 0; +} + static int __xfrm6_output(struct net *net, struct sock *sk, struct sk_buff *skb) { struct dst_entry *dst = skb_dst(skb); @@ -80,6 +93,9 @@ static int __xfrm6_output(struct net *net, struct sock *sk, struct sk_buff *skb) xfrm6_local_rxpmtu(skb, mtu); kfree_skb(skb); return -EMSGSIZE; + } else if (toobig && xfrm6_noneed_fragment(skb)) { + skb->ignore_df = 1; + goto skip_frag; } else if (!skb->ignore_df && toobig && skb->sk) { xfrm_local_error(skb, mtu); kfree_skb(skb); diff --git a/net/xfrm/xfrm_interface.c b/net/xfrm/xfrm_interface.c index e1fae61a5bb9..aa539f8de402 100644 --- a/net/xfrm/xfrm_interface.c +++ b/net/xfrm/xfrm_interface.c @@ -303,7 +303,10 @@ xfrmi_xmit2(struct sk_buff *skb, struct net_device *dev, struct flowi *fl) if (mtu < IPV6_MIN_MTU) mtu = IPV6_MIN_MTU; - icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); + if (skb->len > 1280) + icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); + else + goto xmit; } else { if (!(ip_hdr(skb)->frag_off & htons(IP_DF))) goto xmit; From 9608dc38a0d051efad3932ed5e9d604a70917be3 Mon Sep 17 00:00:00 2001 From: Mukesh Ojha Date: Thu, 7 Apr 2022 11:12:11 +0530 Subject: [PATCH 030/156] FROMLIST: remoteproc: Use unbounded workqueue for recovery work There could be a scenario where there is too much load on a core (n number of tasks which is affined) or in a case when multiple rproc subsystem is going for a recovery and they queued recovery work to one core so even though subsystem are independent there recovery will be delayed if one of the subsystem recovery work is taking more time in completing. If we make this queue unbounded, the recovery work could be picked on any cpu. This patch try to address this. Signed-off-by: Mukesh Ojha Bug: 228429683 Change-Id: If18b39db6c0861989a6a3b36d9efde5f488b9b73 Link: https://lore.kernel.org/lkml/1649313998-1086-1-git-send-email-quic_mojha@quicinc.com/ Signed-off-by: Mukesh Ojha --- drivers/remoteproc/remoteproc_core.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index f9612eeb714e..8dd8b9138789 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -59,6 +59,7 @@ static int rproc_release_carveout(struct rproc *rproc, /* Unique indices for remoteproc devices */ static DEFINE_IDA(rproc_dev_index); +static struct workqueue_struct *rproc_recovery_wq; static const char * const rproc_crash_names[] = { [RPROC_MMUFAULT] = "mmufault", @@ -2488,8 +2489,11 @@ void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type) dev_err(&rproc->dev, "crash detected in %s: type %s\n", rproc->name, rproc_crash_to_string(type)); + if (rproc_recovery_wq) + queue_work(rproc_recovery_wq, &rproc->crash_handler); + else /* Have a worker handle the error; ensure system is not suspended */ - queue_work(system_freezable_wq, &rproc->crash_handler); + queue_work(system_freezable_wq, &rproc->crash_handler); } EXPORT_SYMBOL(rproc_report_crash); @@ -2534,6 +2538,11 @@ static void __exit rproc_exit_panic(void) static int __init remoteproc_init(void) { + rproc_recovery_wq = alloc_workqueue("rproc_recovery_wq", + WQ_UNBOUND | WQ_FREEZABLE, 0); + if (!rproc_recovery_wq) + pr_err("remoteproc: creation of rproc_recovery_wq failed\n"); + rproc_init_sysfs(); rproc_init_debugfs(); rproc_init_cdev(); @@ -2550,6 +2559,8 @@ static void __exit remoteproc_exit(void) rproc_exit_panic(); rproc_exit_debugfs(); rproc_exit_sysfs(); + if (rproc_recovery_wq) + destroy_workqueue(rproc_recovery_wq); } module_exit(remoteproc_exit); From 0dcfc2c03655b59ec9581b712970ba07bd56930d Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Mon, 18 Apr 2022 17:13:55 +0800 Subject: [PATCH 031/156] ANDROID: GKI: Update symbols to symbol list Update symbols to symbol list from oppo Leaf changes summary: 3 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 2 Added functions Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 1 Added variable 2 Added functions: [A] 'function sock* __inet6_lookup_established(net*, inet_hashinfo*, const in6_addr*, const __be16, const in6_addr*, const u16, const int, const int)' [A] 'function sock* __inet_lookup_established(net*, inet_hashinfo*, const __be32, const __be16, const __be32, const u16, const int, const int)' 1 Added variable: [A] 'inet_hashinfo tcp_hashinfo' Bug: 193384408 Signed-off-by: Wei Liu Change-Id: I807fb047fdfb08d17fe19f26f753bd118c1f19ee --- android/abi_gki_aarch64.xml | 11373 ++++++++++++++++---------------- android/abi_gki_aarch64_oplus | 3 + 2 files changed, 5603 insertions(+), 5773 deletions(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index b3258f6a0bcb..5dd16c5ddd55 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -131,6 +131,8 @@ + + @@ -6454,6 +6456,7 @@ + @@ -7140,7 +7143,6 @@ - @@ -8916,9 +8918,9 @@ - - - + + + @@ -8939,7 +8941,6 @@ - @@ -9021,7 +9022,6 @@ - @@ -11897,6 +11897,7 @@ + @@ -12303,8 +12304,8 @@ - - + + @@ -13593,6 +13594,7 @@ + @@ -15142,12 +15144,6 @@ - - - - - - @@ -15642,6 +15638,9 @@ + + + @@ -15841,7 +15840,29 @@ - + + + + + + + + + + + + + + + + + + + + + + + @@ -18731,7 +18752,6 @@ - @@ -19122,11 +19142,7 @@ - - - - - + @@ -20081,7 +20097,6 @@ - @@ -20279,7 +20294,6 @@ - @@ -20720,14 +20734,6 @@ - - - - - - - - @@ -21078,7 +21084,7 @@ - + @@ -21810,6 +21816,7 @@ + @@ -22550,6 +22557,11 @@ + + + + + @@ -24733,7 +24745,7 @@ - + @@ -24772,7 +24784,6 @@ - @@ -24959,6 +24970,7 @@ + @@ -25263,7 +25275,23 @@ - + + + + + + + + + + + + + + + + + @@ -25926,7 +25954,7 @@ - + @@ -26082,7 +26110,7 @@ - + @@ -26303,7 +26331,6 @@ - @@ -26381,7 +26408,7 @@ - + @@ -26438,6 +26465,7 @@ + @@ -26794,6 +26822,7 @@ + @@ -27432,7 +27461,7 @@ - + @@ -28374,7 +28403,6 @@ - @@ -31238,9 +31266,6 @@ - - - @@ -31354,6 +31379,7 @@ + @@ -32226,9 +32252,9 @@ - - - + + + @@ -32719,6 +32745,7 @@ + @@ -33115,7 +33142,6 @@ - @@ -33370,7 +33396,68 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -33877,10 +33964,9 @@ - - + @@ -34292,6 +34378,7 @@ + @@ -34490,7 +34577,7 @@ - + @@ -36127,10 +36214,6 @@ - - - - @@ -36812,7 +36895,17 @@ - + + + + + + + + + + + @@ -38040,7 +38133,6 @@ - @@ -38831,8 +38923,8 @@ - - + + @@ -39805,17 +39897,6 @@ - - - - - - - - - - - @@ -40096,9 +40177,9 @@ - - - + + + @@ -40605,6 +40686,7 @@ + @@ -41588,7 +41670,6 @@ - @@ -44061,13 +44142,6 @@ - - - - - - - @@ -44097,15 +44171,7 @@ - - - - - - - - @@ -44442,18 +44508,7 @@ - - - - - - - - - - - - + @@ -44549,12 +44604,12 @@ - - - - - - + + + + + + @@ -44659,7 +44714,6 @@ - @@ -45440,21 +45494,21 @@ - + - + - + - + - + - + @@ -45901,7 +45955,6 @@ - @@ -47553,8 +47606,8 @@ - - + + @@ -47900,7 +47953,35 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -48081,6 +48162,7 @@ + @@ -48114,7 +48196,7 @@ - + @@ -48465,7 +48547,14 @@ - + + + + + + + + @@ -49111,11 +49200,6 @@ - - - - - @@ -49527,7 +49611,7 @@ - + @@ -50170,56 +50254,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -50560,6 +50595,7 @@ + @@ -51507,7 +51543,7 @@ - + @@ -52352,26 +52388,8 @@ - - - - - - - - - - - - - - - - - - @@ -52630,26 +52648,6 @@ - - - - - - - - - - - - - - - - - - - - @@ -52787,7 +52785,6 @@ - @@ -52838,6 +52835,7 @@ + @@ -53448,7 +53446,23 @@ - + + + + + + + + + + + + + + + + + @@ -53810,7 +53824,6 @@ - @@ -54741,9 +54754,9 @@ - - - + + + @@ -55463,6 +55476,7 @@ + @@ -56047,7 +56061,7 @@ - + @@ -56512,7 +56526,7 @@ - + @@ -57383,29 +57397,6 @@ - - - - - - - - - - - - - - - - - - - - - - - @@ -57614,6 +57605,7 @@ + @@ -58750,7 +58742,6 @@ - @@ -59213,11 +59204,6 @@ - - - - - @@ -60442,44 +60428,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -61705,65 +61654,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -62540,7 +62431,6 @@ - @@ -63181,7 +63071,6 @@ - @@ -63486,7 +63375,17 @@ - + + + + + + + + + + + @@ -65060,10 +64959,10 @@ - - - - + + + + @@ -65114,6 +65013,11 @@ + + + + + @@ -65697,47 +65601,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -66333,11 +66197,6 @@ - - - - - @@ -67372,11 +67231,6 @@ - - - - - @@ -67838,7 +67692,7 @@ - + @@ -69704,7 +69558,6 @@ - @@ -69901,6 +69754,7 @@ + @@ -70611,6 +70465,7 @@ + @@ -71650,14 +71505,6 @@ - - - - - - - - @@ -72346,9 +72193,10 @@ + - - + + @@ -72467,7 +72315,6 @@ - @@ -72792,7 +72639,6 @@ - @@ -73156,8 +73002,8 @@ - - + + @@ -73229,6 +73075,7 @@ + @@ -73696,38 +73543,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -73838,7 +73654,7 @@ - + @@ -75393,6 +75209,7 @@ + @@ -76529,7 +76346,6 @@ - @@ -77937,12 +77753,12 @@ - + - + @@ -78998,12 +78814,12 @@ - - - - - - + + + + + + @@ -80300,6 +80116,7 @@ + @@ -80539,7 +80356,6 @@ - @@ -81265,6 +81081,7 @@ + @@ -81852,6 +81669,7 @@ + @@ -81924,6 +81742,9 @@ + + + @@ -81940,11 +81761,6 @@ - - - - - @@ -81967,7 +81783,6 @@ - @@ -82428,7 +82243,7 @@ - + @@ -82504,14 +82319,6 @@ - - - - - - - - @@ -85619,6 +85426,7 @@ + @@ -86237,7 +86045,7 @@ - + @@ -86918,9 +86726,9 @@ - - - + + + @@ -87389,26 +87197,7 @@ - - - - - - - - - - - - - - - - - - - - + @@ -87433,7 +87222,7 @@ - + @@ -87458,6 +87247,7 @@ + @@ -87496,6 +87286,7 @@ + @@ -88910,7 +88701,7 @@ - + @@ -89557,8 +89348,8 @@ - - + + @@ -90633,7 +90424,6 @@ - @@ -90794,6 +90584,14 @@ + + + + + + + + @@ -91228,7 +91026,20 @@ - + + + + + + + + + + + + + + @@ -91785,7 +91596,7 @@ - + @@ -91960,7 +91771,7 @@ - + @@ -92255,6 +92066,7 @@ + @@ -92395,12 +92207,12 @@ - + - + @@ -92492,7 +92304,6 @@ - @@ -93967,7 +93778,6 @@ - @@ -94480,17 +94290,6 @@ - - - - - - - - - - - @@ -95253,7 +95052,7 @@ - + @@ -95765,7 +95564,7 @@ - + @@ -95811,7 +95610,7 @@ - + @@ -96249,7 +96048,7 @@ - + @@ -96956,9 +96755,6 @@ - - - @@ -98198,6 +97994,7 @@ + @@ -98473,8 +98270,8 @@ - - + + @@ -98684,8 +98481,8 @@ - - + + @@ -99842,6 +99639,7 @@ + @@ -100707,6 +100505,7 @@ + @@ -100819,7 +100618,14 @@ - + + + + + + + + @@ -100896,8 +100702,8 @@ - - + + @@ -102304,9 +102110,6 @@ - - - @@ -102319,6 +102122,9 @@ + + + @@ -102342,7 +102148,7 @@ - + @@ -102483,8 +102289,8 @@ - - + + @@ -103084,7 +102890,6 @@ - @@ -103353,6 +103158,7 @@ + @@ -103887,7 +103693,20 @@ - + + + + + + + + + + + + + + @@ -103946,6 +103765,9 @@ + + + @@ -107390,6 +107212,7 @@ + @@ -107404,20 +107227,6 @@ - - - - - - - - - - - - - - @@ -107752,6 +107561,7 @@ + @@ -111687,11 +111497,6 @@ - - - - - @@ -112018,6 +111823,7 @@ + @@ -113785,7 +113591,6 @@ - @@ -113870,7 +113675,6 @@ - @@ -113994,8 +113798,8 @@ - - + + @@ -114576,6 +114380,7 @@ + @@ -114758,6 +114563,28 @@ + + + + + + + + + + + + + + + + + + + + + + @@ -118415,9 +118242,9 @@ - - - + + + @@ -118428,33 +118255,33 @@ - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + @@ -118464,27 +118291,27 @@ - - - + + + - - - + + + - - - + + + - - - + + + @@ -118494,8 +118321,8 @@ - - + + @@ -118514,8 +118341,8 @@ - - + + @@ -118531,57 +118358,57 @@ - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + @@ -118592,16 +118419,16 @@ - - + + - - + + @@ -118625,12 +118452,12 @@ - - - - - - + + + + + + @@ -118647,9 +118474,9 @@ - - - + + + @@ -118665,29 +118492,29 @@ - - + + - - - - - + + + + + - - - + + + - - + + - - - + + + @@ -118704,15 +118531,15 @@ - - - + + + - - - + + + @@ -118725,22 +118552,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -118819,13 +118646,13 @@ - - - - - - - + + + + + + + @@ -118834,29 +118661,29 @@ - - - + + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -118898,11 +118725,11 @@ - - - - - + + + + + @@ -118922,14 +118749,14 @@ - - + + - - - - + + + + @@ -118952,15 +118779,15 @@ - - - - + + + + - - - + + + @@ -118968,11 +118795,11 @@ - - - - - + + + + + @@ -119023,21 +118850,21 @@ - - - - - + + + + + - - - - - - + + + + + + - - + + @@ -119078,18 +118905,18 @@ - - + + - - - + + + - - - + + + @@ -119122,17 +118949,17 @@ - - - - + + + + - - - - - + + + + + @@ -119143,42 +118970,42 @@ - - - - + + + + - - + + - - - + + + - - - + + + - - + + - - - - + + + + - - + + - - + + @@ -119188,15 +119015,15 @@ - - + + - - - - - + + + + + @@ -119209,9 +119036,9 @@ - - - + + + @@ -119220,49 +119047,49 @@ - - - - + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -119272,10 +119099,10 @@ - - - - + + + + @@ -119325,17 +119152,17 @@ - - + + - - + + - - - + + + @@ -119370,10 +119197,10 @@ - - - - + + + + @@ -119405,13 +119232,13 @@ - - - + + + - - + + @@ -119442,8 +119269,8 @@ - - + + @@ -119452,8 +119279,8 @@ - - + + @@ -119475,8 +119302,8 @@ - - + + @@ -119523,8 +119350,8 @@ - - + + @@ -119546,10 +119373,10 @@ - - - - + + + + @@ -119585,9 +119412,9 @@ - - - + + + @@ -119736,16 +119563,16 @@ - - + + - - + + @@ -119794,34 +119621,34 @@ - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - + + + - - - + + + @@ -119842,31 +119669,31 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -119884,9 +119711,9 @@ - - - + + + @@ -119934,67 +119761,67 @@ - - - - + + + + - - - - - + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - + + + + + + + + + + @@ -120010,16 +119837,16 @@ - - + + - - - - - - + + + + + + @@ -120040,22 +119867,22 @@ - - + + - - + + - - + + - - + + @@ -120079,23 +119906,23 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + @@ -120105,8 +119932,8 @@ - - + + @@ -120119,8 +119946,8 @@ - - + + @@ -120190,9 +120017,9 @@ - - - + + + @@ -120236,11 +120063,11 @@ - - - - - + + + + + @@ -120256,19 +120083,19 @@ - - - + + + - - - + + + - - - + + + @@ -120300,9 +120127,9 @@ - - - + + + @@ -120391,14 +120218,14 @@ - - - + + + - - - + + + @@ -120407,47 +120234,47 @@ - - - + + + - - - - + + + + - - + + - - - - - + + + + + - - - - - + + + + + - - + + - - - - + + + + - - + + @@ -120460,26 +120287,26 @@ - - + + - - + + - - - + + + - - - + + + @@ -120488,48 +120315,48 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -120540,8 +120367,8 @@ - - + + @@ -120659,8 +120486,8 @@ - - + + @@ -120768,28 +120595,28 @@ - - - + + + - - + + - - - - - + + + + + @@ -120800,13 +120627,13 @@ - - - - - - - + + + + + + + @@ -120817,11 +120644,11 @@ - - - - - + + + + + @@ -120833,11 +120660,11 @@ - - - - - + + + + + @@ -120856,8 +120683,8 @@ - - + + @@ -120894,10 +120721,10 @@ - - - - + + + + @@ -120935,22 +120762,22 @@ - - + + - - - - + + + + - - + + - - + + @@ -120961,9 +120788,9 @@ - - - + + + @@ -120982,22 +120809,22 @@ - - + + - - - - + + + + - - - + + + @@ -121058,8 +120885,8 @@ - - + + @@ -121090,18 +120917,18 @@ - - + + - - - + + + - - - + + + @@ -121113,8 +120940,8 @@ - - + + @@ -121130,9 +120957,9 @@ - - - + + + @@ -121165,12 +120992,12 @@ - - + + - - + + @@ -121181,23 +121008,23 @@ - - + + - - - + + + - - - - + + + + @@ -121212,9 +121039,9 @@ - - - + + + @@ -121233,8 +121060,8 @@ - - + + @@ -121254,58 +121081,58 @@ - - - + + + - - - + + + - - - - + + + + - - - - - + + + + + - - - - - + + + + + - + - + - - - + + + - - - - - + + + + + - - - - - + + + + + @@ -121354,12 +121181,12 @@ - - + + - - + + @@ -121388,10 +121215,10 @@ - - - - + + + + @@ -121446,42 +121273,42 @@ - - - - - - + + + + + + - - - - - - + + + + + + - - + + - - - + + + - - - + + + - + @@ -121524,10 +121351,10 @@ - - - - + + + + @@ -121535,9 +121362,9 @@ - - - + + + @@ -121547,7 +121374,7 @@ - + @@ -121568,8 +121395,8 @@ - - + + @@ -121590,22 +121417,22 @@ - - + + - - - + + + - - - + + + @@ -121655,11 +121482,11 @@ - - - - - + + + + + @@ -121667,44 +121494,44 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - - + + + + @@ -121725,8 +121552,8 @@ - - + + @@ -121751,18 +121578,18 @@ - - - + + + - - - + + + @@ -121779,9 +121606,9 @@ - - - + + + @@ -121792,11 +121619,11 @@ - - - - - + + + + + @@ -121806,11 +121633,11 @@ - - + + - + @@ -121832,27 +121659,27 @@ - - + + - - - + + + - - + + - - - - + + + + @@ -121888,12 +121715,12 @@ - - + + - - + + @@ -121902,7 +121729,7 @@ - + @@ -121933,11 +121760,11 @@ - - - - - + + + + + @@ -121947,25 +121774,25 @@ - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -122002,11 +121829,11 @@ - - - - - + + + + + @@ -122016,11 +121843,11 @@ - - - - - + + + + + @@ -122052,9 +121879,9 @@ - - - + + + @@ -122065,8 +121892,8 @@ - - + + @@ -122076,9 +121903,9 @@ - - - + + + @@ -122087,11 +121914,11 @@ - - - - - + + + + + @@ -122104,16 +121931,16 @@ - - + + - - + + - - + + @@ -122128,8 +121955,8 @@ - - + + @@ -122171,49 +121998,49 @@ - - + + - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -122235,9 +122062,9 @@ - - - + + + @@ -122250,9 +122077,9 @@ - - - + + + @@ -122274,10 +122101,10 @@ - - - - + + + + @@ -122308,15 +122135,15 @@ - - - - + + + + - - - + + + @@ -122336,8 +122163,8 @@ - - + + @@ -122353,8 +122180,8 @@ - - + + @@ -122362,13 +122189,13 @@ - - + + - - - + + + @@ -122379,9 +122206,9 @@ - - - + + + @@ -122398,16 +122225,16 @@ - - + + - - + + @@ -122432,9 +122259,9 @@ - - - + + + @@ -122524,9 +122351,9 @@ - - - + + + @@ -122539,17 +122366,17 @@ - - - - + + + + - - - - + + + + @@ -122557,10 +122384,10 @@ - - - - + + + + @@ -122568,9 +122395,9 @@ - - - + + + @@ -122595,11 +122422,11 @@ - - - - - + + + + + @@ -122671,20 +122498,20 @@ - - + + - - + + - - + + @@ -122697,8 +122524,8 @@ - - + + @@ -122720,12 +122547,12 @@ - - - - - - + + + + + + @@ -122734,40 +122561,40 @@ - - - + + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -122802,28 +122629,28 @@ - - - + + + - - + + - - - - + + + + - - - + + + @@ -122845,10 +122672,10 @@ - - - - + + + + @@ -122870,11 +122697,11 @@ - - - - - + + + + + @@ -122897,15 +122724,15 @@ - - - + + + - - - - + + + + @@ -122922,14 +122749,14 @@ - - - + + + - - - + + + @@ -122958,22 +122785,22 @@ - - + + - - + + - - - - + + + + @@ -122996,10 +122823,10 @@ - - - - + + + + @@ -123007,15 +122834,15 @@ - - - - + + + + - - - + + + @@ -123122,11 +122949,11 @@ - - - - - + + + + + @@ -123324,29 +123151,29 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -123356,10 +123183,10 @@ - - - - + + + + @@ -123398,14 +123225,14 @@ - - - - - - - - + + + + + + + + @@ -123418,9 +123245,9 @@ - - - + + + @@ -123453,8 +123280,8 @@ - - + + @@ -123521,14 +123348,14 @@ - - - + + + - - - + + + @@ -123536,10 +123363,10 @@ - - - - + + + + @@ -123584,15 +123411,15 @@ - - - - + + + + - - - + + + @@ -123661,17 +123488,17 @@ - - - + + + - - - - - - + + + + + + @@ -123691,16 +123518,16 @@ - - - + + + - - - - - + + + + + @@ -123756,46 +123583,46 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - + + - - - - - + + + + + - - + + - - + + @@ -123904,9 +123731,9 @@ - - - + + + @@ -123928,17 +123755,17 @@ - - + + - - - + + + - - + + @@ -123946,15 +123773,15 @@ - - - + + + - - - - + + + + @@ -123965,34 +123792,34 @@ - - + + - - - - + + + + - - + + - - - + + + - - - - + + + + @@ -124004,9 +123831,9 @@ - - - + + + @@ -124014,8 +123841,8 @@ - - + + @@ -124026,15 +123853,15 @@ - + - - - - - - + + + + + + @@ -124046,13 +123873,13 @@ - - - - + + + + - - + + @@ -124072,10 +123899,10 @@ - - - - + + + + @@ -124233,46 +124060,46 @@ - - - + + + - - - + + + - - + + - - - - - + + + + + - - + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -124405,8 +124232,8 @@ - - + + @@ -124439,16 +124266,16 @@ - - + + - - + + - - + + @@ -124460,8 +124287,8 @@ - - + + @@ -124472,8 +124299,8 @@ - - + + @@ -124493,9 +124320,9 @@ - - - + + + @@ -124505,62 +124332,62 @@ - - + + - - - + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -124578,14 +124405,14 @@ - - - + + + - - - + + + @@ -124620,13 +124447,13 @@ - - - - - - - + + + + + + + @@ -124850,62 +124677,62 @@ - - + + - - - + + + - - + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + @@ -124921,15 +124748,15 @@ - - + + - - - - - + + + + + @@ -124940,10 +124767,10 @@ - - - - + + + + @@ -124972,8 +124799,8 @@ - - + + @@ -124981,9 +124808,9 @@ - - - + + + @@ -125020,9 +124847,9 @@ - - - + + + @@ -125031,8 +124858,8 @@ - - + + @@ -125040,19 +124867,19 @@ - - - - - + + + + + - - - - - - + + + + + + @@ -125060,20 +124887,20 @@ - - + + - - + + - - + + - - + + @@ -125086,17 +124913,17 @@ - - + + - - + + - - - + + + @@ -125143,19 +124970,19 @@ - - - - - - - + + + + + + + - - - + + + @@ -125163,17 +124990,17 @@ - - - + + + - - + + @@ -125191,20 +125018,20 @@ - - + + - - + + - - + + @@ -125217,11 +125044,11 @@ - - - - - + + + + + @@ -125232,33 +125059,33 @@ - - - + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -125542,23 +125369,23 @@ - - + + - - - - + + + + - - - + + + @@ -125574,12 +125401,12 @@ - - - - - - + + + + + + @@ -125631,20 +125458,20 @@ - - + + - - - - + + + + - - - - + + + + @@ -125799,10 +125626,10 @@ - - - - + + + + @@ -125852,9 +125679,9 @@ - - - + + + @@ -125983,19 +125810,19 @@ - - - + + + - - - + + + - - - + + + @@ -126096,8 +125923,8 @@ - - + + @@ -126119,9 +125946,9 @@ - - - + + + @@ -126149,8 +125976,8 @@ - - + + @@ -126165,24 +125992,24 @@ - - + + - - - + + + - - - - + + + + - - - + + + @@ -126201,9 +126028,9 @@ - - - + + + @@ -126217,18 +126044,18 @@ - - + + - - - + + + - - - + + + @@ -126249,9 +126076,9 @@ - - - + + + @@ -126282,28 +126109,28 @@ - - + + - - + + - - - + + + - - - - + + + + - - - + + + @@ -126314,83 +126141,83 @@ - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - - + + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + @@ -126446,17 +126273,17 @@ - - + + - - + + - - - + + + @@ -126470,20 +126297,20 @@ - - + + - - + + - - + + @@ -126552,9 +126379,9 @@ - - - + + + @@ -126562,44 +126389,44 @@ - - + + - - - - - + + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + @@ -126609,12 +126436,12 @@ - - - - - - + + + + + + @@ -126625,9 +126452,9 @@ - - - + + + @@ -126635,18 +126462,18 @@ - - - - + + + + - - - - - - - + + + + + + + @@ -126661,30 +126488,30 @@ - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + @@ -126720,8 +126547,8 @@ - - + + @@ -126797,20 +126624,20 @@ - - + + - - - - - - - - - - + + + + + + + + + + @@ -126819,19 +126646,19 @@ - - - + + + - - - - + + + + - - - + + + @@ -126858,40 +126685,40 @@ - - - - + + + + - - + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -127130,8 +126957,8 @@ - - + + @@ -127145,13 +126972,13 @@ - - + + - - - + + + @@ -127168,10 +126995,10 @@ - - - - + + + + @@ -127204,14 +127031,14 @@ - - - + + + - - - + + + @@ -127219,17 +127046,17 @@ - - - - + + + + - - + + @@ -127244,9 +127071,9 @@ - - - + + + @@ -127260,50 +127087,50 @@ - - - - + + + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + - - + + - - - + + + - - + + @@ -127313,15 +127140,15 @@ - - - + + + - - - - + + + + @@ -127354,18 +127181,18 @@ - - - - + + + + - - - - - + + + + + @@ -127373,13 +127200,13 @@ - - - + + + - - + + @@ -127409,12 +127236,12 @@ - - + + - - + + @@ -127460,15 +127287,15 @@ - - - + + + - - - - + + + + @@ -127477,30 +127304,30 @@ - - - + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -127516,26 +127343,26 @@ - - + + - - - + + + - - + + - - - + + + @@ -127552,11 +127379,11 @@ - - + + - + @@ -127571,8 +127398,8 @@ - - + + @@ -127620,18 +127447,18 @@ - - - + + + - - + + - - - + + + @@ -127639,13 +127466,13 @@ - - + + - - - + + + @@ -127653,9 +127480,9 @@ - - - + + + @@ -127719,8 +127546,8 @@ - - + + @@ -127729,9 +127556,9 @@ - - - + + + @@ -127744,8 +127571,8 @@ - - + + @@ -127788,12 +127615,12 @@ - - + + - - + + @@ -127830,35 +127657,35 @@ - - + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - + + + @@ -127870,13 +127697,13 @@ - - - + + + - - + + @@ -127919,36 +127746,36 @@ - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - + + + - - - - + + + + - - + + @@ -127998,13 +127825,13 @@ - - - + + + - - + + @@ -128036,10 +127863,10 @@ - - - - + + + + @@ -128049,12 +127876,12 @@ - - + + - - + + @@ -128065,17 +127892,17 @@ - - - - - - - + + + + + + + - - + + @@ -128092,8 +127919,8 @@ - - + + @@ -128123,15 +127950,15 @@ - - + + - - - - - + + + + + @@ -128143,8 +127970,8 @@ - - + + @@ -128170,8 +127997,8 @@ - - + + @@ -128181,20 +128008,20 @@ - - + + - + - - - + + + @@ -128208,41 +128035,41 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -128252,13 +128079,13 @@ - - + + - - - + + + @@ -128272,41 +128099,41 @@ - - - - - - + + + + + + - - + + - - - - - - + + + + + + - - - - - + + + + + - - - - - - - - + + + + + + + + @@ -128320,8 +128147,8 @@ - - + + @@ -128333,18 +128160,18 @@ - - + + - - - - + + + + @@ -128368,23 +128195,23 @@ - - - + + + - - - + + + - - - + + + - - + + @@ -128397,9 +128224,9 @@ - - - + + + @@ -128407,29 +128234,29 @@ - - - - + + + + - - + + - - - - - + + + + + - - - - - - + + + + + + @@ -128462,9 +128289,9 @@ - - - + + + @@ -128473,13 +128300,13 @@ - - + + - - - + + + @@ -128576,8 +128403,8 @@ - - + + @@ -128608,13 +128435,13 @@ - - - + + + - - + + @@ -128659,10 +128486,10 @@ - - + + - + @@ -128677,22 +128504,22 @@ - - - + + + - - - + + + - - + + - - + + @@ -128743,38 +128570,38 @@ - - - - + + + + - - - - - - - - + + + + + + + + - - + + - - + + - + - - + + @@ -128785,9 +128612,9 @@ - - - + + + @@ -128802,26 +128629,26 @@ - - + + - - - - + + + + - - - - - - + + + + + + - - + + @@ -128836,28 +128663,28 @@ - - + + - - - + + + - - + + - - - - - + + + + + - - - - + + + + @@ -128872,28 +128699,28 @@ - - - - - + + + + + - - + + - - - - - + + + + + - - - - + + + + @@ -128915,18 +128742,18 @@ - - + + - - + + - - + + @@ -128949,12 +128776,12 @@ - - + + - - + + @@ -128991,9 +128818,9 @@ - - - + + + @@ -129028,81 +128855,81 @@ - - - + + + - - - + + + - - - - + + + + - - - - - - + + + + + + - - - - + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -129232,12 +129059,12 @@ - - + + - - + + @@ -129280,14 +129107,14 @@ - - + + - - - - + + + + @@ -129308,36 +129135,36 @@ - - - - - + + + + + - - + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -129348,39 +129175,39 @@ - - + + - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - - - + + + + @@ -129406,14 +129233,14 @@ - - + + - - - - + + + + @@ -129425,12 +129252,12 @@ - - + + - - + + @@ -129446,8 +129273,8 @@ - - + + @@ -129569,21 +129396,21 @@ - - - - - + + + + + - - - - - - - - + + + + + + + + @@ -129593,8 +129420,8 @@ - - + + @@ -129617,8 +129444,8 @@ - - + + @@ -129626,9 +129453,9 @@ - - - + + + @@ -129648,24 +129475,24 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -129673,46 +129500,46 @@ - - - - - - + + + + + + - - - + + + - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - - + + + @@ -129728,8 +129555,8 @@ - - + + @@ -129750,8 +129577,8 @@ - - + + @@ -129766,8 +129593,8 @@ - - + + @@ -129800,20 +129627,20 @@ - - + + - - + + - - + + @@ -129850,12 +129677,12 @@ - - + + - - + + @@ -129863,34 +129690,34 @@ - - + + - - - + + + - - - - + + + + - - - - + + + + - - - + + + - - + + @@ -129975,9 +129802,9 @@ - - - + + + @@ -130138,74 +129965,74 @@ - - - + + + - - + + - - - + + + - - + + - - - - - - + + + + + + - - - - - - + + + + + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - + + + - - - - + + + + - - - - + + + + @@ -130213,53 +130040,53 @@ - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - - - - + + + + + + - - - - - + + + + + @@ -130294,8 +130121,8 @@ - - + + @@ -130395,13 +130222,13 @@ - - - + + + - - + + @@ -130412,15 +130239,15 @@ - - - - - - - - - + + + + + + + + + @@ -130456,9 +130283,9 @@ - - - + + + @@ -130484,30 +130311,30 @@ - - - + + + - - - - + + + + - - - - + + + + - - - + + + @@ -130520,10 +130347,10 @@ - - - - + + + + @@ -130531,11 +130358,11 @@ - - - - - + + + + + @@ -130558,10 +130385,10 @@ - - - - + + + + @@ -130574,20 +130401,20 @@ - - - + + + - - - - + + + + - + - - + + @@ -130601,9 +130428,9 @@ - - - + + + @@ -130705,14 +130532,14 @@ - - - + + + - - + + @@ -130799,8 +130626,8 @@ - - + + @@ -130810,56 +130637,56 @@ - - + + - - - - - - - - - + + + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - + + - - + + - - + + - - + + - + @@ -130870,10 +130697,10 @@ - - - - + + + + @@ -130899,21 +130726,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -130924,35 +130751,35 @@ - - - - - - - - + + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -130984,46 +130811,46 @@ - - - - + + + + - - - + + + - - + + - - + + - - - + + + - - - - - + + + + + - - + + - - - + + + @@ -131032,35 +130859,35 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + - - + + @@ -131069,23 +130896,23 @@ - - - + + + - - - + + + - - - + + + @@ -131093,34 +130920,34 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - + + + + - - - - - + + + + + @@ -131130,16 +130957,16 @@ - - - - + + + + - - - - + + + + @@ -131155,10 +130982,10 @@ - - - - + + + + @@ -131168,10 +130995,10 @@ - - - - + + + + @@ -131181,37 +131008,37 @@ - - - - + + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -131222,9 +131049,9 @@ - - - + + + @@ -131238,11 +131065,11 @@ - - - - - + + + + + @@ -131278,11 +131105,11 @@ - - + + - + @@ -131299,15 +131126,15 @@ - - + + - - + + - - + + @@ -131327,8 +131154,8 @@ - - + + @@ -131338,8 +131165,8 @@ - - + + @@ -131351,8 +131178,8 @@ - - + + @@ -131366,18 +131193,18 @@ - - - + + + - - - - + + + + - - + + @@ -131385,10 +131212,10 @@ - - - - + + + + @@ -131401,14 +131228,14 @@ - - - - + + + + - - + + @@ -131444,18 +131271,18 @@ - - - + + + - - - + + + @@ -131466,8 +131293,8 @@ - - + + @@ -131478,10 +131305,10 @@ - - - - + + + + @@ -131491,30 +131318,30 @@ - - - - - + + + + + - - + + - - + + - - - + + + - - + + @@ -131533,11 +131360,11 @@ - - - - - + + + + + @@ -131545,13 +131372,13 @@ - - + + - - + + @@ -131568,11 +131395,11 @@ - - - - - + + + + + @@ -131611,8 +131438,8 @@ - - + + @@ -131620,25 +131447,25 @@ - - - + + + - - + + - - - - + + + + - - - - + + + + @@ -131651,9 +131478,9 @@ - - - + + + @@ -131664,13 +131491,13 @@ - - + + - - - + + + @@ -131678,50 +131505,50 @@ - - - + + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -131735,8 +131562,8 @@ - - + + @@ -131812,9 +131639,9 @@ - - - + + + @@ -131852,32 +131679,32 @@ - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - + @@ -131890,9 +131717,9 @@ - - - + + + @@ -131972,10 +131799,10 @@ - - - - + + + + @@ -132020,13 +131847,13 @@ - - - - - - - + + + + + + + @@ -132213,12 +132040,12 @@ - - + + - - + + @@ -132230,11 +132057,11 @@ - - - - - + + + + + @@ -132275,8 +132102,8 @@ - - + + @@ -132329,8 +132156,8 @@ - - + + @@ -132342,9 +132169,9 @@ - - - + + + @@ -132355,28 +132182,28 @@ - - - + + + - - - - - - + + + + + + - - + + - - - - - + + + + + @@ -132387,9 +132214,9 @@ - - - + + + @@ -132450,8 +132277,8 @@ - - + + @@ -132490,9 +132317,9 @@ - - - + + + @@ -132548,8 +132375,8 @@ - - + + @@ -132571,12 +132398,12 @@ - - + + - - + + @@ -132610,8 +132437,8 @@ - - + + @@ -132622,9 +132449,9 @@ - - - + + + @@ -132632,47 +132459,47 @@ - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - - - - + + + + + @@ -132680,18 +132507,18 @@ - - - - + + + + - - + + @@ -132699,14 +132526,14 @@ - - - - + + + + - - - + + + @@ -132742,17 +132569,17 @@ - - + + - - - + + + @@ -132762,9 +132589,9 @@ - - - + + + @@ -132806,12 +132633,12 @@ - - + + - - + + @@ -132825,67 +132652,67 @@ - - - + + + - - + + - - + + - - + + - - + + - - - - - + + + + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + @@ -132893,19 +132720,19 @@ - - + + - - + + - - - - - + + + + + @@ -132916,41 +132743,41 @@ - - - - + + + + - - - + + + - - + + - - - - - + + + + + - - + + - - + + @@ -132964,16 +132791,16 @@ - - - - + + + + - - - - + + + + @@ -132994,11 +132821,11 @@ - - - - - + + + + + @@ -133015,24 +132842,24 @@ - - - - + + + + - - - - - + + + + + - - - - - + + + + + @@ -133057,10 +132884,10 @@ - - - - + + + + @@ -133073,12 +132900,12 @@ - - + + - - + + @@ -133103,18 +132930,18 @@ - - - + + + - - + + - - - + + + @@ -133138,18 +132965,18 @@ - - - - - + + + + + - - - - - + + + + + @@ -133160,19 +132987,19 @@ - - - - + + + + - - + + - - - + + + @@ -133191,14 +133018,14 @@ - - - + + + - - - + + + @@ -133224,9 +133051,9 @@ - - - + + + @@ -133239,22 +133066,22 @@ - - - - + + + + - - + + - - + + @@ -133268,43 +133095,43 @@ - - + + - - - + + + - - - + + + - - - - - + + + + + - - - + + + - - - - - + + + + + @@ -133317,9 +133144,9 @@ - - - + + + @@ -133352,14 +133179,14 @@ - - - - + + + + - - + + @@ -133370,44 +133197,44 @@ - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -133441,16 +133268,16 @@ - - - - - - + + + + + + - - - + + + @@ -133458,9 +133285,9 @@ - - - + + + @@ -133504,33 +133331,33 @@ - - - + + + - - - + + + - - + + - - + + - - - + + + - - - - + + + + @@ -133555,9 +133382,9 @@ - - - + + + @@ -133573,32 +133400,32 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + - - - + + + @@ -133607,9 +133434,9 @@ - - - + + + @@ -133617,74 +133444,74 @@ - - - + + + - - + + - - - + + + - - - + + + - - - - + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + - - + + - - + + - - - - + + + + @@ -133696,42 +133523,42 @@ - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - + + @@ -133746,8 +133573,8 @@ - - + + @@ -133767,8 +133594,8 @@ - - + + @@ -133782,61 +133609,61 @@ - - - - - + + + + + - - - + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - + + + + @@ -133854,12 +133681,12 @@ - - - - - - + + + + + + @@ -133870,12 +133697,12 @@ - - - - - - + + + + + + @@ -133890,9 +133717,9 @@ - - - + + + @@ -133905,15 +133732,15 @@ - - + + - - - - - + + + + + @@ -133930,10 +133757,10 @@ - - - - + + + + @@ -133953,10 +133780,10 @@ - - - - + + + + @@ -134002,10 +133829,10 @@ - - - - + + + + @@ -134020,16 +133847,16 @@ - - - + + + - - + + - - + + @@ -134052,11 +133879,11 @@ - - - - - + + + + + @@ -134086,16 +133913,16 @@ - - - - - - + + + + + + - - + + @@ -134198,33 +134025,33 @@ - - + + - - + + - - - - - - + + + + + + - - - + + + - - + + @@ -134250,8 +134077,8 @@ - - + + @@ -134266,16 +134093,16 @@ - - + + - - + + - - + + @@ -134360,19 +134187,19 @@ - - - + + + - - - + + + - - - + + + @@ -134387,8 +134214,8 @@ - - + + @@ -134419,9 +134246,9 @@ - - - + + + @@ -134451,14 +134278,14 @@ - - - + + + - - - + + + @@ -134471,9 +134298,9 @@ - - - + + + @@ -134481,10 +134308,10 @@ - - - - + + + + @@ -134498,12 +134325,12 @@ - - + + - - + + @@ -134524,9 +134351,9 @@ - - - + + + @@ -134538,9 +134365,9 @@ - - - + + + @@ -134588,8 +134415,8 @@ - - + + @@ -134635,9 +134462,9 @@ - - - + + + @@ -134653,10 +134480,10 @@ - - - - + + + + @@ -134677,10 +134504,10 @@ - - - - + + + + @@ -134713,17 +134540,17 @@ - - + + - - - + + + @@ -134737,55 +134564,55 @@ - - - - + + + + - - + + - - - + + + - - + + - - + + - - - - - + + + + + - - - + + + - - - + + + - - + + - - + + - - + + @@ -134807,20 +134634,20 @@ - - - - - - + + + + + + - - + + - - + + @@ -134828,21 +134655,21 @@ - - - - - + + + + + - - - - - + + + + + - - + + @@ -134853,7 +134680,7 @@ - + @@ -134872,21 +134699,21 @@ - - - - + + + + - - - - - - - - - + + + + + + + + + @@ -134920,8 +134747,8 @@ - - + + @@ -134941,9 +134768,9 @@ - - - + + + @@ -134951,23 +134778,23 @@ - - - + + + - - - + + + - - - + + + @@ -134988,37 +134815,37 @@ - - + + - - - - + + + + - - - + + + - - + + - - - - + + + + @@ -135032,11 +134859,11 @@ - - - - - + + + + + @@ -135046,12 +134873,12 @@ - - - - - - + + + + + + @@ -135095,10 +134922,10 @@ - - - - + + + + @@ -135157,24 +134984,24 @@ - - + + - - + + - - + + @@ -135184,11 +135011,11 @@ - - - - - + + + + + @@ -135233,14 +135060,14 @@ - - - - + + + + - - - + + + @@ -135473,8 +135300,8 @@ - - + + @@ -135494,29 +135321,29 @@ - - - + + + - - + + - - + + - - + + - - + + - - + + @@ -135528,14 +135355,14 @@ - - - + + + - - - + + + @@ -135548,16 +135375,16 @@ - - - - + + + + - - - - + + + + @@ -135609,12 +135436,12 @@ - - + + - - + + @@ -135661,24 +135488,24 @@ - - + + - - + + - - + + - - + + @@ -135698,13 +135525,13 @@ - - + + - - - + + + @@ -135718,16 +135545,16 @@ - - - - + + + + - - - - + + + + @@ -135748,8 +135575,8 @@ - - + + @@ -135764,8 +135591,8 @@ - - + + @@ -135799,8 +135626,8 @@ - - + + @@ -135818,10 +135645,10 @@ - - - - + + + + @@ -135834,54 +135661,54 @@ - - + + - - - + + + - - + + - + - + - - - - + + + + - - - - + + + + - - - - - - - - - + + + + + + + + + - - + + - - + + @@ -135890,36 +135717,36 @@ - - - - - + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + @@ -135930,12 +135757,12 @@ - - - - - - + + + + + + @@ -135954,9 +135781,9 @@ - - - + + + @@ -135966,8 +135793,8 @@ - - + + @@ -135981,10 +135808,10 @@ - - - - + + + + @@ -135997,13 +135824,13 @@ - - - + + + - - + + @@ -136056,11 +135883,11 @@ - - - - - + + + + + @@ -136076,24 +135903,24 @@ - - + + - - + + - - + + - - + + @@ -136101,24 +135928,24 @@ - - + + - - + + - - + + @@ -136178,8 +136005,8 @@ - - + + @@ -136189,28 +136016,28 @@ - - - - - + + + + + - - - - - + + + + + - - - - + + + + @@ -136224,23 +136051,23 @@ - - - + + + - - + + - - - - + + + + - - + + @@ -136263,48 +136090,48 @@ - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + @@ -136313,7 +136140,7 @@ - + @@ -136353,17 +136180,17 @@ - + - + - - + + @@ -136382,8 +136209,8 @@ - - + + @@ -136420,8 +136247,8 @@ - - + + @@ -136433,23 +136260,23 @@ - - - - + + + + - - - + + + - - - + + + @@ -136509,8 +136336,8 @@ - - + + @@ -136551,16 +136378,16 @@ - - + + - - + + - - + + @@ -136571,8 +136398,8 @@ - - + + @@ -136595,12 +136422,12 @@ - - + + - - + + @@ -136616,8 +136443,8 @@ - - + + @@ -136736,10 +136563,10 @@ - - - - + + + + @@ -136748,20 +136575,20 @@ - - - - - - - - + + + + + + + + - - - - + + + + @@ -136769,14 +136596,14 @@ - - - + + + - - - + + + @@ -136794,8 +136621,8 @@ - - + + @@ -136807,8 +136634,8 @@ - - + + @@ -136845,8 +136672,8 @@ - - + + @@ -136897,22 +136724,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -137002,8 +136829,8 @@ - - + + @@ -137011,22 +136838,22 @@ - - + + - - - - - - + + + + + + - - - - + + + + @@ -137039,9 +136866,9 @@ - - - + + + @@ -137049,9 +136876,9 @@ - - - + + + @@ -137099,13 +136926,13 @@ - - - - - - - + + + + + + + @@ -137113,28 +136940,28 @@ - - + + - - + + - - + + - - + + - - + + @@ -137150,20 +136977,20 @@ - - - - - - + + + + + + - - - + + + - - + + @@ -137179,8 +137006,8 @@ - - + + @@ -137197,8 +137024,8 @@ - - + + @@ -137211,15 +137038,15 @@ - - - + + + - - - - + + + + @@ -137227,39 +137054,39 @@ - - - + + + - - - - + + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + @@ -137267,30 +137094,30 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -137373,30 +137200,30 @@ - - + + - - - + + + - - - + + + - - - - - - + + + + + + - - + + @@ -137417,8 +137244,8 @@ - - + + @@ -137431,15 +137258,15 @@ - - - - - + + + + + - - + + @@ -137467,57 +137294,57 @@ - - + + - - + + - - + + - - - - - - - - - + + + + + + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - - + + + - - + + @@ -137542,12 +137369,12 @@ - - + + - - + + @@ -137564,9 +137391,9 @@ - - - + + + @@ -137574,9 +137401,9 @@ - - - + + + @@ -137584,9 +137411,9 @@ - - - + + + @@ -137611,12 +137438,12 @@ - - + + - - + + @@ -137634,10 +137461,10 @@ - - - - + + + + @@ -137659,67 +137486,67 @@ - - - + + + - - - - + + + + - - - + + + - - + + - - - - - - + + + + + + - - - - + + + + - - - - - - - + + + + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -137730,13 +137557,13 @@ - + - - - - + + + + @@ -137770,10 +137597,10 @@ - - - - + + + + @@ -137815,7 +137642,7 @@ - + @@ -137823,34 +137650,34 @@ - - + + - - + + - - + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -137861,10 +137688,10 @@ - - - - + + + + @@ -137874,8 +137701,8 @@ - - + + @@ -137886,8 +137713,8 @@ - - + + @@ -137925,8 +137752,8 @@ - - + + @@ -137964,14 +137791,14 @@ - - - - - + + + + + - - + + @@ -138009,11 +137836,11 @@ - - - - - + + + + + @@ -138157,21 +137984,21 @@ - - + + - - - + + + - - + + - - + + @@ -138205,11 +138032,11 @@ - - - - - + + + + + @@ -138224,21 +138051,21 @@ - - + + - - + + - - + + - - - + + + @@ -138254,11 +138081,11 @@ - - - - - + + + + + @@ -138282,12 +138109,12 @@ - - - - - - + + + + + + @@ -138307,21 +138134,21 @@ - - - - + + + + - - - - + + + + - - - + + + @@ -138342,26 +138169,26 @@ - - - - + + + + - - - + + + - - - - - + + + + + - - - + + + @@ -138370,20 +138197,20 @@ - - - + + + - - - + + + - - - + + + @@ -138393,26 +138220,26 @@ - - - + + + - - - + + + - - - - + + + + - - - - + + + + @@ -138558,9 +138385,9 @@ - - - + + + @@ -138569,9 +138396,9 @@ - - - + + + @@ -138579,26 +138406,26 @@ - - + + - - - - + + + + - - + + - - + + @@ -138606,20 +138433,20 @@ - - - + + + - - - + + + - - - - + + + + @@ -138631,33 +138458,33 @@ - - - - - - + + + + + + - - - - - - + + + + + + - - - - + + + + - - - + + + @@ -138665,28 +138492,28 @@ - - - + + + - - - - - + + + + + - - + + - - - + + + - - + + @@ -138697,13 +138524,13 @@ - - - - - - - + + + + + + + @@ -138735,14 +138562,14 @@ - - + + - - - - + + + + @@ -138750,10 +138577,10 @@ - - - - + + + + @@ -138761,10 +138588,10 @@ - - - - + + + + @@ -138787,10 +138614,10 @@ - - - - + + + + @@ -138813,29 +138640,29 @@ - + - - + + - - + + - - - - + + + + - - - - - - + + + + + + @@ -138864,13 +138691,13 @@ - - - - - - - + + + + + + + @@ -138889,10 +138716,10 @@ - - - - + + + + @@ -138909,10 +138736,10 @@ - - - - + + + + @@ -138922,30 +138749,30 @@ - - - + + + - - - - - - + + + + + + - - + + - - - - - - - + + + + + + + @@ -138962,13 +138789,13 @@ - - + + - - - + + + @@ -138987,39 +138814,39 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + - - + + @@ -139042,55 +138869,55 @@ - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -139100,9 +138927,9 @@ - - - + + + @@ -139112,16 +138939,16 @@ - - - - - + + + + + - - - + + + @@ -139156,15 +138983,15 @@ - - - + + + - - - - + + + + @@ -139177,11 +139004,11 @@ - - - - - + + + + + @@ -139191,16 +139018,16 @@ - - - - - + + + + + - - - + + + @@ -139210,8 +139037,8 @@ - - + + @@ -139223,25 +139050,25 @@ - - + + - - + + - - - - - - - + + + + + + + - - + + @@ -139252,9 +139079,9 @@ - - - + + + @@ -139306,9 +139133,9 @@ - - - + + + @@ -139342,22 +139169,22 @@ - - - - - + + + + + - - - - - + + + + + - - + + @@ -139436,8 +139263,8 @@ - - + + @@ -139447,9 +139274,9 @@ - - - + + + @@ -139501,8 +139328,8 @@ - - + + @@ -139518,9 +139345,9 @@ - - - + + + @@ -139530,17 +139357,17 @@ - - - - + + + + - - - - - + + + + + @@ -139564,17 +139391,17 @@ - - + + - - - - - - - + + + + + + + @@ -139583,23 +139410,23 @@ - - + + - - - - + + + + - - - + + + - - + + @@ -139610,12 +139437,12 @@ - - - - - - + + + + + + @@ -139627,8 +139454,8 @@ - - + + @@ -139640,18 +139467,18 @@ - - - - - - + + + + + + - - - - + + + + @@ -139659,12 +139486,12 @@ - - + + - - + + @@ -139702,9 +139529,9 @@ - - - + + + @@ -139716,8 +139543,8 @@ - - + + @@ -139749,45 +139576,45 @@ - - - + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - - + + + + + @@ -139832,9 +139659,9 @@ - - - + + + @@ -139865,10 +139692,10 @@ - - - - + + + + @@ -139960,14 +139787,14 @@ - - - - + + + + - - + + @@ -139985,8 +139812,8 @@ - - + + @@ -140237,9 +140064,9 @@ - - - + + + @@ -140328,9 +140155,9 @@ - - - + + + @@ -140351,13 +140178,13 @@ - - - - - - - + + + + + + + @@ -140369,44 +140196,44 @@ - - - + + + - - - - + + + + - - + + - - + + - - - - - - + + + + + + - - + + - - - + + + @@ -140516,9 +140343,9 @@ - - - + + + @@ -140530,17 +140357,17 @@ - - - - - - + + + + + + - - - + + + @@ -140664,9 +140491,9 @@ - - - + + + @@ -140699,34 +140526,34 @@ - - + + - - + + - - + + - - + + - - - - + + + + - - - - - - + + + + + + @@ -140749,9 +140576,9 @@ - - - + + + @@ -140769,8 +140596,8 @@ - - + + @@ -140805,10 +140632,10 @@ - - - - + + + + @@ -140817,21 +140644,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -140839,21 +140666,21 @@ - - - - + + + + - - - - - + + + + + - - - + + + @@ -140872,16 +140699,16 @@ - - - + + + - - - + + + - - + + @@ -140899,43 +140726,43 @@ - - + + - - + + - - + + - - + + - - + + - - - + + + - + - + @@ -140947,12 +140774,12 @@ - - + + - - + + @@ -140967,23 +140794,23 @@ - - + + - - + + - - - + + + - + - + @@ -141002,15 +140829,15 @@ - - - + + + - - - - + + + + @@ -141023,27 +140850,27 @@ - - - + + + - - - - + + + + - - - + + + - - - - + + + + @@ -141065,9 +140892,9 @@ - - - + + + @@ -141076,10 +140903,10 @@ - - - - + + + + @@ -141087,9 +140914,9 @@ - - - + + + @@ -141097,9 +140924,9 @@ - - - + + + @@ -141108,17 +140935,17 @@ - - - - + + + + - + @@ -141136,8 +140963,8 @@ - - + + @@ -141171,6 +140998,7 @@ + @@ -141179,8 +141007,8 @@ - - + + @@ -141273,8 +141101,8 @@ - - + + @@ -141283,8 +141111,8 @@ - - + + @@ -141316,19 +141144,19 @@ - - - - - - - - - + + + + + + + + + - - + + @@ -141340,9 +141168,9 @@ - - - + + + @@ -141361,14 +141189,14 @@ - - - - + + + + - - + + @@ -141396,9 +141224,9 @@ - - - + + + @@ -141421,12 +141249,11 @@ - - - + + @@ -141461,19 +141288,19 @@ - - + + - - - - + + + + - - - + + + @@ -141491,8 +141318,8 @@ - - + + @@ -141502,24 +141329,24 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -141529,26 +141356,26 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + @@ -141557,10 +141384,10 @@ - - - - + + + + @@ -141577,7 +141404,7 @@ - + @@ -141611,9 +141438,9 @@ - - - + + + @@ -141751,10 +141578,10 @@ - - - - + + + + @@ -141762,10 +141589,10 @@ - - - - + + + + @@ -141844,19 +141671,19 @@ - - - + + + - - - + + + - - - + + + @@ -141886,8 +141713,8 @@ - - + + @@ -141905,16 +141732,16 @@ - - + + - - + + @@ -141967,9 +141794,9 @@ - - - + + + @@ -141983,9 +141810,9 @@ - - - + + + @@ -142024,8 +141851,8 @@ - - + + @@ -142071,9 +141898,9 @@ - - - + + + @@ -142164,11 +141991,11 @@ - - - - - + + + + + @@ -142403,9 +142230,9 @@ - - - + + + @@ -142523,27 +142350,27 @@ - - + + - - + + - - + + - - - - - + + + + + @@ -142577,8 +142404,8 @@ - - + + @@ -142617,17 +142444,17 @@ - - + + - - + + - - - + + + @@ -142638,8 +142465,8 @@ - - + + @@ -142670,12 +142497,12 @@ - - + + - - + + @@ -142698,8 +142525,8 @@ - - + + @@ -142721,8 +142548,8 @@ - - + + @@ -142758,9 +142585,9 @@ - - - + + + @@ -142783,9 +142610,9 @@ - - - + + + @@ -142800,16 +142627,16 @@ - - + + - - + + - - + + @@ -142828,8 +142655,8 @@ - - + + @@ -142844,16 +142671,16 @@ - - - - - - - - - - + + + + + + + + + + @@ -142887,8 +142714,8 @@ - - + + @@ -142903,15 +142730,15 @@ - - - - + + + + - - - + + + @@ -142922,9 +142749,9 @@ - - - + + + @@ -142936,9 +142763,9 @@ - - - + + + @@ -142957,24 +142784,24 @@ - - - + + + - - - - + + + + - - - + + + @@ -143001,8 +142828,8 @@ - - + + @@ -143021,9 +142848,9 @@ - - - + + + @@ -143032,28 +142859,28 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + @@ -143062,14 +142889,14 @@ - - - - + + + + - - + + @@ -143080,9 +142907,9 @@ - - - + + + @@ -143097,8 +142924,8 @@ - - + + @@ -143120,8 +142947,8 @@ - - + + @@ -143224,15 +143051,15 @@ - - - + + + - - - - + + + + @@ -143244,8 +143071,8 @@ - - + + @@ -143311,16 +143138,16 @@ - - + + - - + + @@ -143328,10 +143155,10 @@ - - - - + + + + @@ -143355,38 +143182,38 @@ - - + + - - + + - - - + + + - - - + + + - - + + - - + + @@ -143394,10 +143221,10 @@ - - - - + + + + @@ -143421,9 +143248,9 @@ - - - + + + @@ -143431,8 +143258,8 @@ - - + + @@ -143481,9 +143308,9 @@ - - - + + + @@ -143504,10 +143331,10 @@ - - - - + + + + @@ -143557,9 +143384,9 @@ - - - + + + @@ -143619,50 +143446,50 @@ - - - + + + - - - - - - + + + + + + - - - - - + + + + + - - + + - - - - + + + + - - - - - - + + + + + + - - + + @@ -143838,9 +143665,9 @@ - - - + + + @@ -143854,27 +143681,27 @@ - - + + - - - - + + + + - - - + + + @@ -143888,33 +143715,33 @@ - - - - - + + + + + - - - + + + - - + + - - + + - - + + - - - + + + @@ -144160,15 +143987,15 @@ - - - + + + - - - - + + + + @@ -144219,15 +144046,15 @@ - - - + + + - - - + + + @@ -144240,15 +144067,15 @@ - - - - + + + + - - - + + + @@ -144326,9 +144153,9 @@ - - - + + + @@ -144344,36 +144171,36 @@ - - - + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + @@ -144388,9 +144215,9 @@ - - - + + + @@ -144405,14 +144232,14 @@ - - - + + + - - - + + + @@ -144454,15 +144281,15 @@ - - + + - - - - - + + + + + @@ -144470,11 +144297,11 @@ - - - - - + + + + + @@ -144490,8 +144317,8 @@ - - + + @@ -144511,8 +144338,8 @@ - - + + @@ -144555,8 +144382,8 @@ - - + + @@ -144592,8 +144419,8 @@ - - + + @@ -144660,9 +144487,9 @@ - - - + + + @@ -144733,26 +144560,26 @@ - - - - - - - + + + + + + + - - + + - - - + + + @@ -144789,21 +144616,21 @@ - - - + + + - - - + + + - - - + + + @@ -144870,15 +144697,15 @@ - - + + - - + + @@ -144889,16 +144716,16 @@ - - - - - + + + + + - - - + + + @@ -144907,11 +144734,11 @@ - - - - - + + + + + @@ -144920,9 +144747,9 @@ - - - + + + @@ -144972,11 +144799,11 @@ - - - - - + + + + + @@ -144986,23 +144813,23 @@ - - - - + + + + - - - + + + - - + + - - + + @@ -145021,12 +144848,12 @@ - - + + - - + + @@ -145034,8 +144861,8 @@ - - + + @@ -145056,8 +144883,8 @@ - - + + @@ -145065,16 +144892,16 @@ - - - + + + - + - - + + @@ -145082,8 +144909,8 @@ - - + + @@ -145099,9 +144926,9 @@ - - - + + + @@ -145124,17 +144951,17 @@ - - - + + + - - + + @@ -145151,12 +144978,12 @@ - - - - - - + + + + + + @@ -145211,9 +145038,9 @@ - - - + + + @@ -145238,34 +145065,34 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + @@ -145275,17 +145102,17 @@ - - + + - - - - + + + + @@ -145296,26 +145123,26 @@ - - - - + + + + - - - - + + + + - - + + - - - - + + + + @@ -145326,9 +145153,9 @@ - - - + + + @@ -145360,10 +145187,10 @@ - - - - + + + + @@ -145396,9 +145223,9 @@ - - - + + + @@ -145415,15 +145242,15 @@ - - - - + + + + - - - + + + @@ -145447,12 +145274,12 @@ - - - - - - + + + + + + @@ -145474,13 +145301,13 @@ - - + + - - - + + + @@ -145497,10 +145324,10 @@ - - - - + + + + @@ -145511,47 +145338,47 @@ - - - + + + - - + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index bf898882c7b3..7664c86d1179 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -1110,7 +1110,9 @@ inc_node_page_state inc_zone_page_state in_egroup_p + __inet6_lookup_established inet_csk_get_port + __inet_lookup_established init_iova_domain init_net init_pseudo @@ -2514,6 +2516,7 @@ __task_pid_nr_ns __task_rq_lock task_rq_lock + tcp_hashinfo tcp_parse_options thermal_cooling_device_register thermal_cooling_device_unregister From 00f2b55cc4654d8a1ed4fb8b95c8379a03f5141b Mon Sep 17 00:00:00 2001 From: Ashish Mhetre Date: Tue, 10 Aug 2021 10:14:00 +0530 Subject: [PATCH 032/156] BACKPORT: iommu: Fix race condition during default domain allocation When two devices with same SID are getting probed concurrently through iommu_probe_device(), the iommu_domain sometimes is getting allocated more than once as call to iommu_alloc_default_domain() is not protected for concurrency. Furthermore, it leads to each device holding a different iommu_domain pointer, separate IOVA space and only one of the devices' domain is used for translations from IOMMU. This causes accesses from other device to fault or see incorrect translations. Fix this by protecting iommu_alloc_default_domain() call with group->mutex and let all devices with same SID share same iommu_domain. Signed-off-by: Ashish Mhetre Link: https://lore.kernel.org/r/1628570641-9127-2-git-send-email-amhetre@nvidia.com Signed-off-by: Will Deacon Bug: 229173748 (cherry picked from commit 211ff31b3d33b56aa12937e898c9280d07daf0d9) Signed-off-by: Suren Baghdasaryan Change-Id: Iafd361871c0f06f75f71f7dbd7575b74d9c4253f --- drivers/iommu/iommu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 1d320eec94aa..51ecbae3c0bd 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -268,7 +268,9 @@ int iommu_probe_device(struct device *dev) * support default domains, so the return value is not yet * checked. */ + mutex_lock(&group->mutex); iommu_alloc_default_domain(group, dev); + mutex_unlock(&group->mutex); if (group->default_domain) { ret = __iommu_attach_device(group->default_domain, dev); From b8bb3b43a492d48924b40c543384d86b2c188760 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 8 Nov 2021 14:13:49 +0800 Subject: [PATCH 033/156] BACKPORT: iommu: Extend mutex lock scope in iommu_probe_device() Extend the scope of holding group->mutex so that it can cover the default domain check/attachment and direct mappings of reserved regions. Cc: Ashish Mhetre Fixes: 211ff31b3d33b ("iommu: Fix race condition during default domain allocation") Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211108061349.1985579-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel (cherry picked from commit 556f99ac886635e8da15528995f06d1d7028cfca) Bug: 229173748 Signed-off-by: Suren Baghdasaryan Change-Id: I59bdee605a029bb28e7b023b90ab85983619be97 --- drivers/iommu/iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 51ecbae3c0bd..ed6902eb8d91 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -270,11 +270,11 @@ int iommu_probe_device(struct device *dev) */ mutex_lock(&group->mutex); iommu_alloc_default_domain(group, dev); - mutex_unlock(&group->mutex); if (group->default_domain) { ret = __iommu_attach_device(group->default_domain, dev); if (ret) { + mutex_unlock(&group->mutex); iommu_group_put(group); goto err_release; } @@ -282,6 +282,7 @@ int iommu_probe_device(struct device *dev) iommu_create_device_direct_mappings(group, dev); + mutex_unlock(&group->mutex); iommu_group_put(group); if (ops->probe_finalize) From 3628acf6b8d7fceeea2522b51218eb9feb4350be Mon Sep 17 00:00:00 2001 From: wuzhe Date: Wed, 23 Feb 2022 12:46:15 +0800 Subject: [PATCH 034/156] ANDROID: GKI: Update symbols to abi_gki_aarch64_oplus Leaf changes summary: 119 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 119 Added functions Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 0 Added variable 119 Added functions: [A] 'function int __dquot_alloc_space(inode*, qsize_t, int)' [A] 'function void __dquot_free_space(inode*, qsize_t, int)' [A] 'function int __dquot_transfer(inode*, dquot**)' [A] 'function int __fscrypt_encrypt_symlink(inode*, const char*, unsigned int, fscrypt_str*)' [A] 'function bool __fscrypt_inode_uses_inline_crypto(const inode*)' [A] 'function int __fscrypt_prepare_link(inode*, inode*, dentry*)' [A] 'function int __fscrypt_prepare_lookup(inode*, dentry*, fscrypt_name*)' [A] 'function int __fscrypt_prepare_readdir(inode*)' [A] 'function int __fscrypt_prepare_rename(inode*, dentry*, inode*, dentry*, unsigned int)' [A] 'function int __fscrypt_prepare_setattr(dentry*, iattr*)' [A] 'function ssize_t __generic_file_write_iter(kiocb*, iov_iter*)' [A] 'function void bio_associate_blkg_from_css(bio*, cgroup_subsys_state*)' [A] 'function const char* blk_op_str(unsigned int)' [A] 'function int blkdev_issue_zeroout(block_device*, long long unsigned int, long long unsigned int, unsigned int, unsigned int)' [A] 'function void clear_nlink(inode*)' [A] 'function dentry* d_find_alias(inode*)' [A] 'function void d_instantiate_new(dentry*, inode*)' [A] 'function void d_invalidate(dentry*)' [A] 'function void d_tmpfile(dentry*, inode*)' [A] 'function char* dentry_path_raw(dentry*, char*, int)' [A] 'function dquot* dqget(super_block*, kqid)' [A] 'function void dqput(dquot*)' [A] 'function int dquot_acquire(dquot*)' [A] 'function dquot* dquot_alloc(super_block*, int)' [A] 'function int dquot_alloc_inode(inode*)' [A] 'function int dquot_claim_space_nodirty(inode*, qsize_t)' [A] 'function int dquot_commit(dquot*)' [A] 'function int dquot_commit_info(super_block*, int)' [A] 'function void dquot_destroy(dquot*)' [A] 'function int dquot_disable(super_block*, int, unsigned int)' [A] 'function void dquot_drop(inode*)' [A] 'function int dquot_file_open(inode*, file*)' [A] 'function void dquot_free_inode(inode*)' [A] 'function int dquot_get_dqblk(super_block*, kqid, qc_dqblk*)' [A] 'function int dquot_get_next_dqblk(super_block*, kqid*, qc_dqblk*)' [A] 'function int dquot_get_next_id(super_block*, kqid*)' [A] 'function int dquot_get_state(super_block*, qc_state*)' [A] 'function int dquot_initialize(inode*)' [A] 'function bool dquot_initialize_needed(inode*)' [A] 'function int dquot_load_quota_inode(inode*, int, int, unsigned int)' [A] 'function int dquot_mark_dquot_dirty(dquot*)' [A] 'function int dquot_quota_off(super_block*, int)' [A] 'function int dquot_quota_on(super_block*, int, int, const path*)' [A] 'function int dquot_quota_on_mount(super_block*, char*, int, int)' [A] 'function int dquot_release(dquot*)' [A] 'function int dquot_resume(super_block*, int)' [A] 'function int dquot_set_dqblk(super_block*, kqid, qc_dqblk*)' [A] 'function int dquot_set_dqinfo(super_block*, int, qc_info*)' [A] 'function int dquot_transfer(inode*, iattr*)' [A] 'function int dquot_writeback_dquots(super_block*, int)' [A] 'function void evict_inodes(super_block*)' [A] 'function inode* find_inode_nowait(super_block*, unsigned long int, int (inode*, unsigned long int, void*)*, void*)' [A] 'function int freeze_bdev(block_device*)' [A] 'function int freeze_super(super_block*)' [A] 'function void fscrypt_decrypt_bio(bio*)' [A] 'function bool fscrypt_dio_supported(kiocb*, iov_iter*)' [A] 'function int fscrypt_drop_inode(inode*)' [A] 'function page* fscrypt_encrypt_pagecache_blocks(page*, unsigned int, unsigned int, gfp_t)' [A] 'function int fscrypt_file_open(inode*, file*)' [A] 'function int fscrypt_fname_alloc_buffer(u32, fscrypt_str*)' [A] 'function int fscrypt_fname_disk_to_usr(const inode*, u32, u32, const fscrypt_str*, fscrypt_str*)' [A] 'function void fscrypt_fname_free_buffer(fscrypt_str*)' [A] 'function u64 fscrypt_fname_siphash(const inode*, const qstr*)' [A] 'function void fscrypt_free_bounce_page(page*)' [A] 'function void fscrypt_free_inode(inode*)' [A] 'function const char* fscrypt_get_symlink(inode*, void*, unsigned int, delayed_call*)' [A] 'function int fscrypt_has_permitted_context(inode*, inode*)' [A] 'function int fscrypt_ioctl_add_key(file*, void*)' [A] 'function int fscrypt_ioctl_get_key_status(file*, void*)' [A] 'function int fscrypt_ioctl_get_nonce(file*, void*)' [A] 'function int fscrypt_ioctl_get_policy(file*, void*)' [A] 'function int fscrypt_ioctl_get_policy_ex(file*, void*)' [A] 'function int fscrypt_ioctl_remove_key(file*, void*)' [A] 'function int fscrypt_ioctl_remove_key_all_users(file*, void*)' [A] 'function int fscrypt_ioctl_set_policy(file*, void*)' [A] 'function bool fscrypt_match_name(const fscrypt_name*, const u8*, u32)' [A] 'function bool fscrypt_mergeable_bio(bio*, const inode*, long long unsigned int)' [A] 'function int fscrypt_prepare_new_inode(inode*, inode*, bool*)' [A] 'function int fscrypt_prepare_symlink(inode*, const char*, unsigned int, unsigned int, fscrypt_str*)' [A] 'function void fscrypt_put_encryption_info(inode*)' [A] 'function void fscrypt_set_bio_crypt_ctx(bio*, const inode*, long long unsigned int, unsigned int)' [A] 'function int fscrypt_set_context(inode*, void*)' [A] 'function int fscrypt_set_test_dummy_encryption(super_block*, const char*, fscrypt_dummy_policy*)' [A] 'function int fscrypt_setup_filename(inode*, const qstr*, int, fscrypt_name*)' [A] 'function void fscrypt_show_test_dummy_encryption(seq_file*, char, super_block*)' [A] 'function int fscrypt_symlink_getattr(const path*, kstat*)' [A] 'function int fscrypt_zeroout_range(const inode*, unsigned long int, sector_t, unsigned int)' [A] 'function void fsverity_cleanup_inode(inode*)' [A] 'function void fsverity_enqueue_verify_work(work_struct*)' [A] 'function int fsverity_file_open(inode*, file*)' [A] 'function int fsverity_ioctl_enable(file*, void*)' [A] 'function int fsverity_ioctl_measure(file*, void*)' [A] 'function int fsverity_ioctl_read_metadata(file*, void*)' [A] 'function int fsverity_prepare_setattr(dentry*, iattr*)' [A] 'function void fsverity_verify_bio(bio*)' [A] 'function bool fsverity_verify_page(page*)' [A] 'function dentry* generic_fh_to_dentry(super_block*, fid*, int, int, inode* (super_block*, typedef u64, typedef u32)*)' [A] 'function dentry* generic_fh_to_parent(super_block*, fid*, int, int, inode* (super_block*, typedef u64, typedef u32)*)' [A] 'function long long int generic_file_llseek_size(file*, long long int, int, long long int, long long int)' [A] 'function void generic_set_encrypted_ci_d_ops(dentry*)' [A] 'function void iget_failed(inode*)' [A] 'function inode* iget_locked(super_block*, unsigned long int)' [A] 'function inode* ilookup(super_block*, unsigned long int)' [A] 'function void inode_nohighmem(inode*)' [A] 'function int insert_inode_locked(inode*)' [A] 'function posix_acl* posix_acl_alloc(int, gfp_t)' [A] 'function int posix_acl_chmod(inode*, unsigned short int)' [A] 'function int posix_acl_equiv_mode(const posix_acl*, umode_t*)' [A] 'function void seq_escape(seq_file*, const char*, const char*)' [A] 'function void set_cached_acl(inode*, int, posix_acl*)' [A] 'function int set_task_ioprio(task_struct*, int)' [A] 'function void shrink_dcache_sb(super_block*)' [A] 'function void sync_inodes_sb(super_block*)' [A] 'function int thaw_bdev(block_device*)' [A] 'function int thaw_super(super_block*)' [A] 'function int vfs_ioc_fssetxattr_check(inode*, const fsxattr*, fsxattr*)' [A] 'function int vfs_ioc_setflags_prepare(inode*, unsigned int, unsigned int)' [A] 'function long long int vfs_setpos(file*, long long int, long long int)' [A] 'function void wbc_account_cgroup_owner(writeback_control*, page*, size_t)' Bug: 220957464 Signed-off-by: wuzhe Change-Id: I85c3d05d169e20a451ba9d808c785c4965e026d1 --- android/abi_gki_aarch64.xml | 12102 +++++++++++++++++--------------- android/abi_gki_aarch64_oplus | 139 +- 2 files changed, 6729 insertions(+), 5512 deletions(-) mode change 100755 => 100644 android/abi_gki_aarch64.xml diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml old mode 100755 new mode 100644 index 5dd16c5ddd55..952ed23ff18d --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -85,6 +85,9 @@ + + + @@ -112,6 +115,14 @@ + + + + + + + + @@ -806,6 +817,7 @@ + @@ -883,6 +895,7 @@ + @@ -935,6 +948,7 @@ + @@ -1046,6 +1060,7 @@ + @@ -1313,11 +1328,15 @@ + + + + @@ -1363,6 +1382,7 @@ + @@ -1796,8 +1816,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -2339,6 +2389,7 @@ + @@ -2376,6 +2427,7 @@ + @@ -2411,17 +2463,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -2466,8 +2562,11 @@ + + + @@ -2478,6 +2577,7 @@ + @@ -2756,6 +2856,8 @@ + + @@ -2785,6 +2887,7 @@ + @@ -2809,6 +2912,7 @@ + @@ -2840,6 +2944,7 @@ + @@ -3989,6 +4094,9 @@ + + + @@ -4508,6 +4616,7 @@ + @@ -4555,6 +4664,7 @@ + @@ -4564,6 +4674,7 @@ + @@ -4601,6 +4712,7 @@ + @@ -4997,6 +5109,7 @@ + @@ -5066,6 +5179,8 @@ + + @@ -5694,7 +5809,10 @@ + + + @@ -5831,6 +5949,7 @@ + @@ -7143,6 +7262,7 @@ + @@ -8918,9 +9038,9 @@ - - - + + + @@ -8941,6 +9061,7 @@ + @@ -9022,6 +9143,7 @@ + @@ -9711,6 +9833,7 @@ + @@ -10412,6 +10535,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -11897,7 +12055,6 @@ - @@ -12304,8 +12461,8 @@ - - + + @@ -13594,7 +13751,6 @@ - @@ -14004,6 +14160,7 @@ + @@ -14872,6 +15029,7 @@ + @@ -15144,6 +15302,12 @@ + + + + + + @@ -15638,9 +15802,6 @@ - - - @@ -15797,6 +15958,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -15840,29 +16021,7 @@ - - - - - - - - - - - - - - - - - - - - - - - + @@ -18752,6 +18911,7 @@ + @@ -19142,7 +19302,11 @@ - + + + + + @@ -20097,6 +20261,7 @@ + @@ -20294,6 +20459,7 @@ + @@ -20734,6 +20900,14 @@ + + + + + + + + @@ -21085,6 +21259,7 @@ + @@ -21816,7 +21991,6 @@ - @@ -22557,11 +22731,6 @@ - - - - - @@ -24745,7 +24914,7 @@ - + @@ -24784,6 +24953,7 @@ + @@ -24970,7 +25140,6 @@ - @@ -25275,23 +25444,7 @@ - - - - - - - - - - - - - - - - - + @@ -25954,7 +26107,7 @@ - + @@ -26110,7 +26263,7 @@ - + @@ -26331,6 +26484,7 @@ + @@ -26408,7 +26562,7 @@ - + @@ -26465,7 +26619,6 @@ - @@ -26822,7 +26975,6 @@ - @@ -27461,7 +27613,7 @@ - + @@ -28403,6 +28555,7 @@ + @@ -31266,6 +31419,9 @@ + + + @@ -31379,7 +31535,6 @@ - @@ -31888,7 +32043,20 @@ - + + + + + + + + + + + + + + @@ -31912,6 +32080,14 @@ + + + + + + + + @@ -32109,6 +32285,7 @@ + @@ -32252,9 +32429,9 @@ - - - + + + @@ -33142,6 +33319,7 @@ + @@ -33396,68 +33574,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -33685,6 +33802,11 @@ + + + + + @@ -33868,6 +33990,9 @@ + + + @@ -33964,9 +34089,10 @@ + - + @@ -34378,7 +34504,6 @@ - @@ -34577,7 +34702,7 @@ - + @@ -34728,7 +34853,7 @@ - + @@ -36214,6 +36339,10 @@ + + + + @@ -36528,6 +36657,12 @@ + + + + + + @@ -38133,6 +38268,7 @@ + @@ -38923,8 +39059,8 @@ - - + + @@ -39897,6 +40033,17 @@ + + + + + + + + + + + @@ -40177,9 +40324,9 @@ - - - + + + @@ -40686,7 +40833,6 @@ - @@ -41670,6 +41816,7 @@ + @@ -44142,6 +44289,13 @@ + + + + + + + @@ -44171,7 +44325,15 @@ + + + + + + + + @@ -44508,7 +44670,18 @@ - + + + + + + + + + + + + @@ -44604,12 +44777,12 @@ - - - - - - + + + + + + @@ -44714,6 +44887,7 @@ + @@ -44809,6 +44983,7 @@ + @@ -45494,21 +45669,21 @@ - + - + - + - + - + - + @@ -45955,6 +46130,7 @@ + @@ -47246,6 +47422,12 @@ + + + + + + @@ -47606,8 +47788,8 @@ - - + + @@ -47953,35 +48135,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -48097,6 +48251,7 @@ + @@ -48162,7 +48317,6 @@ - @@ -48196,7 +48350,7 @@ - + @@ -48547,14 +48701,7 @@ - - - - - - - - + @@ -49200,6 +49347,11 @@ + + + + + @@ -49611,7 +49763,7 @@ - + @@ -50254,7 +50406,56 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -50595,7 +50796,6 @@ - @@ -51543,7 +51743,7 @@ - + @@ -52388,8 +52588,26 @@ + + + + + + + + + + + + + + + + + + @@ -52648,6 +52866,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -52785,6 +53023,7 @@ + @@ -52835,7 +53074,6 @@ - @@ -53446,23 +53684,7 @@ - - - - - - - - - - - - - - - - - + @@ -53824,6 +54046,7 @@ + @@ -54754,9 +54977,9 @@ - - - + + + @@ -55476,7 +55699,6 @@ - @@ -56061,7 +56283,7 @@ - + @@ -56165,6 +56387,7 @@ + @@ -56526,7 +56749,7 @@ - + @@ -57397,6 +57620,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -57603,9 +57849,9 @@ + - @@ -58742,6 +58988,7 @@ + @@ -59204,6 +59451,11 @@ + + + + + @@ -60428,7 +60680,44 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -61654,7 +61943,65 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -61686,6 +62033,7 @@ + @@ -62431,6 +62779,7 @@ + @@ -63071,6 +63420,7 @@ + @@ -63375,17 +63725,7 @@ - - - - - - - - - - - + @@ -64959,10 +65299,10 @@ - - - - + + + + @@ -65013,11 +65353,6 @@ - - - - - @@ -65601,7 +65936,47 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -66197,6 +66572,11 @@ + + + + + @@ -66447,6 +66827,7 @@ + @@ -67231,6 +67612,11 @@ + + + + + @@ -67692,7 +68078,7 @@ - + @@ -69558,6 +69944,7 @@ + @@ -69754,7 +70141,6 @@ - @@ -70465,7 +70851,6 @@ - @@ -70633,6 +71018,7 @@ + @@ -71505,6 +71891,14 @@ + + + + + + + + @@ -72193,10 +72587,9 @@ - - - + + @@ -72315,6 +72708,7 @@ + @@ -72639,6 +73033,7 @@ + @@ -72955,6 +73350,7 @@ + @@ -73002,8 +73398,8 @@ - - + + @@ -73075,7 +73471,6 @@ - @@ -73543,7 +73938,38 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -73654,7 +74080,7 @@ - + @@ -75209,7 +75635,6 @@ - @@ -76346,6 +76771,7 @@ + @@ -77753,12 +78179,12 @@ - + - + @@ -78814,12 +79240,12 @@ - - - - - - + + + + + + @@ -80116,7 +80542,6 @@ - @@ -80356,6 +80781,7 @@ + @@ -81081,7 +81507,6 @@ - @@ -81134,6 +81559,23 @@ + + + + + + + + + + + + + + + + + @@ -81669,7 +82111,6 @@ - @@ -81742,9 +82183,6 @@ - - - @@ -81761,6 +82199,11 @@ + + + + + @@ -81783,6 +82226,7 @@ + @@ -82243,7 +82687,7 @@ - + @@ -82319,6 +82763,14 @@ + + + + + + + + @@ -85426,7 +85878,6 @@ - @@ -86045,7 +86496,7 @@ - + @@ -86112,6 +86563,7 @@ + @@ -86726,9 +87178,9 @@ - - - + + + @@ -87197,7 +87649,26 @@ - + + + + + + + + + + + + + + + + + + + + @@ -87222,7 +87693,7 @@ - + @@ -87247,7 +87718,6 @@ - @@ -87286,7 +87756,6 @@ - @@ -88701,7 +89170,7 @@ - + @@ -89348,8 +89817,8 @@ - - + + @@ -90424,6 +90893,7 @@ + @@ -90584,14 +91054,6 @@ - - - - - - - - @@ -91026,20 +91488,7 @@ - - - - - - - - - - - - - - + @@ -91596,7 +92045,7 @@ - + @@ -91771,7 +92220,7 @@ - + @@ -92066,7 +92515,6 @@ - @@ -92207,12 +92655,12 @@ - + - + @@ -92304,6 +92752,7 @@ + @@ -93778,6 +94227,7 @@ + @@ -94290,6 +94740,17 @@ + + + + + + + + + + + @@ -95052,7 +95513,7 @@ - + @@ -95564,7 +96025,7 @@ - + @@ -95610,7 +96071,7 @@ - + @@ -96048,7 +96509,7 @@ - + @@ -96302,6 +96763,7 @@ + @@ -96755,6 +97217,9 @@ + + + @@ -97994,7 +98459,6 @@ - @@ -98270,8 +98734,8 @@ - - + + @@ -98481,8 +98945,8 @@ - - + + @@ -99639,7 +100103,6 @@ - @@ -100505,7 +100968,6 @@ - @@ -100618,14 +101080,7 @@ - - - - - - - - + @@ -100702,8 +101157,8 @@ - - + + @@ -102110,6 +102565,9 @@ + + + @@ -102122,9 +102580,6 @@ - - - @@ -102148,7 +102603,7 @@ - + @@ -102289,8 +102744,8 @@ - - + + @@ -102561,6 +103016,7 @@ + @@ -102890,6 +103346,7 @@ + @@ -103158,7 +103615,6 @@ - @@ -103693,20 +104149,7 @@ - - - - - - - - - - - - - - + @@ -103765,9 +104208,6 @@ - - - @@ -107212,7 +107652,6 @@ - @@ -107227,6 +107666,20 @@ + + + + + + + + + + + + + + @@ -107561,7 +108014,6 @@ - @@ -109651,6 +110103,7 @@ + @@ -111497,6 +111950,11 @@ + + + + + @@ -111823,7 +112281,6 @@ - @@ -113110,6 +113567,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -113591,6 +114068,7 @@ + @@ -113675,6 +114153,7 @@ + @@ -113798,8 +114277,8 @@ - - + + @@ -114314,6 +114793,23 @@ + + + + + + + + + + + + + + + + + @@ -114380,7 +114876,6 @@ - @@ -114443,6 +114938,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -118242,9 +118782,9 @@ - - - + + + @@ -118255,33 +118795,33 @@ - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + @@ -118291,27 +118831,27 @@ - - - + + + - - - + + + - - - + + + - - - + + + @@ -118321,8 +118861,8 @@ - - + + @@ -118341,8 +118881,8 @@ - - + + @@ -118358,57 +118898,57 @@ - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + @@ -118419,16 +118959,16 @@ - - + + - - + + @@ -118452,12 +118992,12 @@ - - - - - - + + + + + + @@ -118474,9 +119014,9 @@ - - - + + + @@ -118492,29 +119032,29 @@ - - + + - - - - - + + + + + - - - + + + - - + + - - - + + + @@ -118531,15 +119071,15 @@ - - - + + + - - - + + + @@ -118552,22 +119092,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -118646,13 +119186,13 @@ - - - - - - - + + + + + + + @@ -118661,29 +119201,29 @@ - - - + + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -118725,11 +119265,11 @@ - - - - - + + + + + @@ -118749,14 +119289,14 @@ - - + + - - - - + + + + @@ -118779,15 +119319,15 @@ - - - - + + + + - - - + + + @@ -118795,11 +119335,11 @@ - - - - - + + + + + @@ -118850,21 +119390,21 @@ - - - - - + + + + + - - - - - - + + + + + + - - + + @@ -118905,18 +119445,18 @@ - - + + - - - + + + - - - + + + @@ -118949,17 +119489,17 @@ - - - - + + + + - - - - - + + + + + @@ -118970,42 +119510,47 @@ - - - - + + + + - - + + - - - + + + - - - + + + - - + + + - - - - + + - - + + + + - - + + + + + + @@ -119015,15 +119560,15 @@ - - + + - - - - - + + + + + @@ -119036,9 +119581,9 @@ - - - + + + @@ -119047,49 +119592,49 @@ - - - - + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -119099,10 +119644,10 @@ - - - - + + + + @@ -119152,17 +119697,17 @@ - - + + - - + + - - - + + + @@ -119197,10 +119742,10 @@ - - - - + + + + @@ -119232,13 +119777,13 @@ - - - + + + - - + + @@ -119269,8 +119814,8 @@ - - + + @@ -119279,8 +119824,8 @@ - - + + @@ -119302,8 +119847,8 @@ - - + + @@ -119350,8 +119895,8 @@ - - + + @@ -119373,10 +119918,14 @@ - - - - + + + + + + + + @@ -119412,9 +119961,9 @@ - - - + + + @@ -119563,16 +120112,16 @@ - - + + - - + + @@ -119621,34 +120170,42 @@ - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - + + + - - - + + + + + + + + + + + @@ -119669,31 +120226,31 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -119711,9 +120268,9 @@ - - - + + + @@ -119761,67 +120318,67 @@ - - - - + + + + - - - - - + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - + + + + + + + + + + @@ -119837,16 +120394,16 @@ - - + + - - - - - - + + + + + + @@ -119867,22 +120424,22 @@ - - + + - - + + - - + + - - + + @@ -119906,23 +120463,23 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + @@ -119932,8 +120489,8 @@ - - + + @@ -119946,8 +120503,8 @@ - - + + @@ -120017,9 +120574,9 @@ - - - + + + @@ -120063,11 +120620,11 @@ - - - - - + + + + + @@ -120083,19 +120640,19 @@ - - - + + + - - - + + + - - - + + + @@ -120127,9 +120684,9 @@ - - - + + + @@ -120218,14 +120775,14 @@ - - - + + + - - - + + + @@ -120234,47 +120791,47 @@ - - - + + + - - - - + + + + - - + + - - - - - + + + + + - - - - - + + + + + - - + + - - - - + + + + - - + + @@ -120287,26 +120844,30 @@ - - + + - - + + + + + + - - - + + + - - - + + + @@ -120315,48 +120876,48 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -120367,8 +120928,8 @@ - - + + @@ -120486,8 +121047,8 @@ - - + + @@ -120595,28 +121156,28 @@ - - - + + + - - + + - - - - - + + + + + @@ -120627,13 +121188,13 @@ - - - - - - - + + + + + + + @@ -120644,11 +121205,11 @@ - - - - - + + + + + @@ -120660,11 +121221,11 @@ - - - - - + + + + + @@ -120683,8 +121244,8 @@ - - + + @@ -120721,10 +121282,10 @@ - - - - + + + + @@ -120762,22 +121323,22 @@ - - + + - - - - + + + + - - + + - - + + @@ -120788,9 +121349,9 @@ - - - + + + @@ -120809,22 +121370,22 @@ - - + + - - - - + + + + - - - + + + @@ -120885,8 +121446,8 @@ - - + + @@ -120917,18 +121478,18 @@ - - + + - - - + + + - - - + + + @@ -120940,8 +121501,8 @@ - - + + @@ -120957,9 +121518,9 @@ - - - + + + @@ -120992,12 +121553,12 @@ - - + + - - + + @@ -121008,23 +121569,23 @@ - - + + - - - + + + - - - - + + + + @@ -121039,9 +121600,9 @@ - - - + + + @@ -121060,8 +121621,8 @@ - - + + @@ -121081,58 +121642,58 @@ - - - + + + - - - + + + - - - - + + + + - - - - - + + + + + - - - - - + + + + + - + - + - - - + + + - - - - - + + + + + - - - - - + + + + + @@ -121181,12 +121742,12 @@ - - + + - - + + @@ -121215,10 +121776,10 @@ - - - - + + + + @@ -121273,42 +121834,42 @@ - - - - - - + + + + + + - - - - - - + + + + + + - - + + - - - + + + - - - + + + - + @@ -121351,10 +121912,10 @@ - - - - + + + + @@ -121362,9 +121923,9 @@ - - - + + + @@ -121374,7 +121935,7 @@ - + @@ -121395,8 +121956,8 @@ - - + + @@ -121417,22 +121978,22 @@ - - + + - - - + + + - - - + + + @@ -121482,11 +122043,11 @@ - - - - - + + + + + @@ -121494,44 +122055,44 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - - + + + + @@ -121552,8 +122113,8 @@ - - + + @@ -121578,18 +122139,18 @@ - - - + + + - - - + + + @@ -121606,9 +122167,9 @@ - - - + + + @@ -121619,11 +122180,11 @@ - - - - - + + + + + @@ -121633,11 +122194,11 @@ - - + + - + @@ -121659,27 +122220,40 @@ - - + + - - - + + + + + + + - - + + + + + + + + + + + - - - - + + + + @@ -121687,6 +122261,11 @@ + + + + + @@ -121715,12 +122294,12 @@ - - + + - - + + @@ -121729,7 +122308,7 @@ - + @@ -121760,11 +122339,11 @@ - - - - - + + + + + @@ -121774,25 +122353,25 @@ - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -121829,11 +122408,11 @@ - - - - - + + + + + @@ -121843,11 +122422,11 @@ - - - - - + + + + + @@ -121879,9 +122458,9 @@ - - - + + + @@ -121892,8 +122471,8 @@ - - + + @@ -121903,9 +122482,9 @@ - - - + + + @@ -121914,11 +122493,11 @@ - - - - - + + + + + @@ -121931,16 +122510,16 @@ - - + + - - + + - - + + @@ -121951,12 +122530,18 @@ + + + + + + - - + + @@ -121998,49 +122583,49 @@ - - + + - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -122062,9 +122647,9 @@ - - - + + + @@ -122077,9 +122662,9 @@ - - - + + + @@ -122101,10 +122686,10 @@ - - - - + + + + @@ -122135,15 +122720,15 @@ - - - - + + + + - - - + + + @@ -122163,8 +122748,8 @@ - - + + @@ -122180,8 +122765,8 @@ - - + + @@ -122189,13 +122774,13 @@ - - + + - - - + + + @@ -122206,9 +122791,9 @@ - - - + + + @@ -122225,16 +122810,16 @@ - - + + - - + + @@ -122259,9 +122844,9 @@ - - - + + + @@ -122351,9 +122936,9 @@ - - - + + + @@ -122366,17 +122951,17 @@ - - - - + + + + - - - - + + + + @@ -122384,10 +122969,10 @@ - - - - + + + + @@ -122395,9 +122980,9 @@ - - - + + + @@ -122422,11 +123007,11 @@ - - - - - + + + + + @@ -122498,20 +123083,20 @@ - - + + - - + + - - + + @@ -122524,8 +123109,8 @@ - - + + @@ -122547,12 +123132,12 @@ - - - - - - + + + + + + @@ -122561,40 +123146,40 @@ - - - + + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -122629,28 +123214,28 @@ - - - + + + - - + + - - - - + + + + - - - + + + @@ -122672,10 +123257,10 @@ - - - - + + + + @@ -122697,11 +123282,11 @@ - - - - - + + + + + @@ -122724,15 +123309,15 @@ - - - + + + - - - - + + + + @@ -122749,14 +123334,14 @@ - - - + + + - - - + + + @@ -122785,22 +123370,22 @@ - - + + - - + + - - - - + + + + @@ -122823,10 +123408,10 @@ - - - - + + + + @@ -122834,15 +123419,15 @@ - - - - + + + + - - - + + + @@ -122949,11 +123534,11 @@ - - - - - + + + + + @@ -123151,29 +123736,29 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -123183,10 +123768,10 @@ - - - - + + + + @@ -123225,14 +123810,14 @@ - - - - - - - - + + + + + + + + @@ -123245,9 +123830,9 @@ - - - + + + @@ -123280,8 +123865,8 @@ - - + + @@ -123348,14 +123933,14 @@ - - - + + + - - - + + + @@ -123363,10 +123948,10 @@ - - - - + + + + @@ -123411,15 +123996,15 @@ - - - - + + + + - - - + + + @@ -123518,16 +124103,16 @@ - - - + + + - - - - - + + + + + @@ -123583,46 +124168,46 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - + + - - - - - + + + + + - - + + - - + + @@ -123731,9 +124316,9 @@ - - - + + + @@ -123755,17 +124340,17 @@ - - + + - - - + + + - - + + @@ -123773,15 +124358,15 @@ - - - + + + - - - - + + + + @@ -123792,34 +124377,34 @@ - - + + - - - - + + + + - - + + - - - + + + - - - - + + + + @@ -123831,9 +124416,9 @@ - - - + + + @@ -123841,8 +124426,8 @@ - - + + @@ -123853,15 +124438,15 @@ - + - - - - - - + + + + + + @@ -123873,13 +124458,13 @@ - - - - + + + + - - + + @@ -123899,10 +124484,10 @@ - - - - + + + + @@ -124060,46 +124645,46 @@ - - - + + + - - - + + + - - + + - - - - - + + + + + - - + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -124232,8 +124817,8 @@ - - + + @@ -124266,16 +124851,16 @@ - - + + - - + + - - + + @@ -124287,8 +124872,8 @@ - - + + @@ -124299,10 +124884,15 @@ - - + + + + + + + @@ -124312,6 +124902,151 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -124320,9 +125055,9 @@ - - - + + + @@ -124332,62 +125067,62 @@ - - + + - - - + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -124405,14 +125140,14 @@ - - - + + + - - - + + + @@ -124447,13 +125182,13 @@ - - - - - - - + + + + + + + @@ -124677,62 +125412,62 @@ - - + + - - - + + + - - + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + @@ -124748,15 +125483,15 @@ - - + + - - - - - + + + + + @@ -124767,10 +125502,10 @@ - - - - + + + + @@ -124799,8 +125534,8 @@ - - + + @@ -124808,9 +125543,9 @@ - - - + + + @@ -124847,9 +125582,9 @@ - - - + + + @@ -124858,8 +125593,8 @@ - - + + @@ -124867,19 +125602,19 @@ - - - - - + + + + + - - - - - - + + + + + + @@ -124887,20 +125622,20 @@ - - + + - - + + - - + + - - + + @@ -124913,17 +125648,17 @@ - - + + - - + + - - - + + + @@ -124970,19 +125705,19 @@ - - - - - - - + + + + + + + - - - + + + @@ -124990,17 +125725,17 @@ - - - + + + - - + + @@ -125018,20 +125753,20 @@ - - + + - - + + - - + + @@ -125044,11 +125779,11 @@ - - - - - + + + + + @@ -125059,33 +125794,33 @@ - - - + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -125369,23 +126104,23 @@ - - + + - - - - + + + + - - - + + + @@ -125401,12 +126136,12 @@ - - - - - - + + + + + + @@ -125458,20 +126193,20 @@ - - + + - - - - + + + + - - - - + + + + @@ -125626,10 +126361,10 @@ - - - - + + + + @@ -125679,9 +126414,9 @@ - - - + + + @@ -125810,19 +126545,19 @@ - - - + + + - - - + + + - - - + + + @@ -125923,8 +126658,8 @@ - - + + @@ -125946,9 +126681,9 @@ - - - + + + @@ -125976,8 +126711,8 @@ - - + + @@ -125992,24 +126727,24 @@ - - + + - - - + + + - - - - + + + + - - - + + + @@ -126028,9 +126763,9 @@ - - - + + + @@ -126044,18 +126779,18 @@ - - + + - - - + + + - - - + + + @@ -126076,9 +126811,9 @@ - - - + + + @@ -126109,28 +126844,28 @@ - - + + - - + + - - - + + + - - - - + + + + - - - + + + @@ -126141,83 +126876,83 @@ - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - - + + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + @@ -126273,17 +127008,17 @@ - - + + - - + + - - - + + + @@ -126297,20 +127032,20 @@ - - + + - - + + - - + + @@ -126379,9 +127114,9 @@ - - - + + + @@ -126389,44 +127124,44 @@ - - + + - - - - - + + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + @@ -126436,12 +127171,12 @@ - - - - - - + + + + + + @@ -126452,9 +127187,9 @@ - - - + + + @@ -126462,18 +127197,18 @@ - - - - + + + + - - - - - - - + + + + + + + @@ -126488,30 +127223,30 @@ - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + @@ -126547,8 +127282,8 @@ - - + + @@ -126624,20 +127359,20 @@ - - + + - - - - - - - - - - + + + + + + + + + + @@ -126646,19 +127381,19 @@ - - - + + + - - - - + + + + - - - + + + @@ -126685,40 +127420,40 @@ - - - - + + + + - - + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -126957,8 +127692,8 @@ - - + + @@ -126972,13 +127707,13 @@ - - + + - - - + + + @@ -126995,10 +127730,10 @@ - - - - + + + + @@ -127031,14 +127766,14 @@ - - - + + + - - - + + + @@ -127046,17 +127781,17 @@ - - - - + + + + - - + + @@ -127071,9 +127806,9 @@ - - - + + + @@ -127087,50 +127822,54 @@ - - - - + + + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + - - + + + + + + - - - + + + - - + + @@ -127140,15 +127879,15 @@ - - - + + + - - - - + + + + @@ -127181,18 +127920,18 @@ - - - - + + + + - - - - - + + + + + @@ -127200,13 +127939,13 @@ - - - + + + - - + + @@ -127236,12 +127975,12 @@ - - + + - - + + @@ -127287,15 +128026,15 @@ - - - + + + - - - - + + + + @@ -127304,30 +128043,37 @@ - - - + + + - - + + - - - + + + + + + + + + + - - - - + + + + - - - - + + + + @@ -127343,26 +128089,26 @@ - - + + - - - + + + - - + + - - - + + + @@ -127379,11 +128125,11 @@ - - + + - + @@ -127398,8 +128144,8 @@ - - + + @@ -127447,18 +128193,18 @@ - - - + + + - - + + - - - + + + @@ -127466,23 +128212,31 @@ - - + + - - - + + + + + + + + + + + - - - + + + @@ -127514,6 +128268,186 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -127542,12 +128476,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + @@ -127556,9 +128531,9 @@ - - - + + + @@ -127571,8 +128546,8 @@ - - + + @@ -127615,12 +128590,12 @@ - - + + - - + + @@ -127657,35 +128632,35 @@ - - + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - + + + @@ -127697,13 +128672,13 @@ - - - + + + - - + + @@ -127746,36 +128721,36 @@ - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - + + + - - - - + + + + - - + + @@ -127791,6 +128766,22 @@ + + + + + + + + + + + + + + + + @@ -127802,6 +128793,14 @@ + + + + + + + + @@ -127825,13 +128824,13 @@ - - - + + + - - + + @@ -127859,14 +128858,18 @@ + + + + - - - - + + + + @@ -127876,12 +128879,12 @@ - - + + - - + + @@ -127892,17 +128895,17 @@ - - - - - - - + + + + + + + - - + + @@ -127919,8 +128922,8 @@ - - + + @@ -127950,15 +128953,15 @@ - - + + - - - - - + + + + + @@ -127970,8 +128973,8 @@ - - + + @@ -127997,8 +129000,8 @@ - - + + @@ -128008,20 +129011,20 @@ - - + + - + - - - + + + @@ -128035,41 +129038,41 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -128079,13 +129082,13 @@ - - + + - - - + + + @@ -128099,41 +129102,41 @@ - - - - - - + + + + + + - - + + - - - - - - + + + + + + - - - - - + + + + + - - - - - - - - + + + + + + + + @@ -128147,8 +129150,8 @@ - - + + @@ -128160,18 +129163,18 @@ - - + + - - - - + + + + @@ -128195,23 +129198,23 @@ - - - + + + - - - + + + - - - + + + - - + + @@ -128224,9 +129227,9 @@ - - - + + + @@ -128234,29 +129237,29 @@ - - - - + + + + - - + + - - - - - + + + + + - - - - - - + + + + + + @@ -128289,9 +129292,9 @@ - - - + + + @@ -128300,13 +129303,13 @@ - - + + - - - + + + @@ -128403,8 +129406,8 @@ - - + + @@ -128435,13 +129438,13 @@ - - - + + + - - + + @@ -128486,10 +129489,10 @@ - - + + - + @@ -128504,22 +129507,22 @@ - - - + + + - - - + + + - - + + - - + + @@ -128570,38 +129573,38 @@ - - - - + + + + - - - - - - - - + + + + + + + + - - + + - - + + - + - - + + @@ -128612,9 +129615,9 @@ - - - + + + @@ -128629,26 +129632,26 @@ - - + + - - - - + + + + - - - - - - + + + + + + - - + + @@ -128663,28 +129666,28 @@ - - + + - - - + + + - - + + - - - - - + + + + + - - - - + + + + @@ -128699,28 +129702,28 @@ - - - - - + + + + + - - + + - - - - - + + + + + - - - - + + + + @@ -128742,18 +129745,18 @@ - - + + - - + + - - + + @@ -128776,12 +129779,12 @@ - - + + - - + + @@ -128818,9 +129821,9 @@ - - - + + + @@ -128855,81 +129858,81 @@ - - - + + + - - - + + + - - - - + + + + - - - - - - + + + + + + - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -129059,12 +130062,12 @@ - - + + - - + + @@ -129107,14 +130110,14 @@ - - + + - - - - + + + + @@ -129135,36 +130138,36 @@ - - - - - + + + + + - - + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -129175,39 +130178,39 @@ - - + + - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - - - + + + + @@ -129233,14 +130236,14 @@ - - + + - - - - + + + + @@ -129251,13 +130254,22 @@ - - - + + + + + + + - - + + + + + + + @@ -129273,8 +130285,8 @@ - - + + @@ -129396,21 +130408,26 @@ - - - - - + + + - - - - - - - - + + + + + + + + + + + + + + + @@ -129420,8 +130437,8 @@ - - + + @@ -129444,8 +130461,8 @@ - - + + @@ -129453,9 +130470,9 @@ - - - + + + @@ -129475,24 +130492,24 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -129500,46 +130517,50 @@ - - - - - - + + + + + + - - - + + + - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - - + + + + + + + @@ -129555,8 +130576,8 @@ - - + + @@ -129577,8 +130598,8 @@ - - + + @@ -129593,8 +130614,8 @@ - - + + @@ -129627,20 +130648,20 @@ - - + + - - + + - - + + @@ -129677,47 +130698,51 @@ - - + + - - + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + - - - + + + - - + + @@ -129802,9 +130827,9 @@ - - - + + + @@ -129965,74 +130990,74 @@ - - - + + + - - + + - - - + + + - - + + - - - - - - + + + + + + - - - - - - + + + + + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - + + + - - - - + + + + - - - - + + + + @@ -130040,53 +131065,53 @@ - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - - - - + + + + + + - - - - - + + + + + @@ -130121,8 +131146,8 @@ - - + + @@ -130222,13 +131247,13 @@ - - - + + + - - + + @@ -130239,15 +131264,15 @@ - - - - - - - - - + + + + + + + + + @@ -130283,9 +131308,9 @@ - - - + + + @@ -130311,30 +131336,30 @@ - - - + + + - - - - + + + + - - - - + + + + - - - + + + @@ -130347,10 +131372,10 @@ - - - - + + + + @@ -130358,11 +131383,11 @@ - - - - - + + + + + @@ -130385,10 +131410,10 @@ - - - - + + + + @@ -130401,20 +131426,20 @@ - - - + + + - - - - + + + + - + - - + + @@ -130428,9 +131453,9 @@ - - - + + + @@ -130532,14 +131557,14 @@ - - - + + + - - + + @@ -130626,8 +131651,8 @@ - - + + @@ -130637,56 +131662,56 @@ - - + + - - - - - - - - - + + + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - + + - - + + - - + + - - + + - + @@ -130697,10 +131722,10 @@ - - - - + + + + @@ -130726,21 +131751,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -130751,35 +131776,35 @@ - - - - - - - - + + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -130811,46 +131836,46 @@ - - - - + + + + - - - + + + - - + + - - + + - - - + + + - - - - - + + + + + - - + + - - - + + + @@ -130859,35 +131884,35 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + - - + + @@ -130896,23 +131921,23 @@ - - - + + + - - - + + + - - - + + + @@ -130920,34 +131945,34 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - + + + + - - - - - + + + + + @@ -130957,16 +131982,16 @@ - - - - + + + + - - - - + + + + @@ -130982,10 +132007,10 @@ - - - - + + + + @@ -130995,10 +132020,10 @@ - - - - + + + + @@ -131008,37 +132033,37 @@ - - - - + + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -131049,9 +132074,9 @@ - - - + + + @@ -131065,11 +132090,11 @@ - - - - - + + + + + @@ -131105,11 +132130,11 @@ - - + + - + @@ -131126,15 +132151,15 @@ - - + + - - + + - - + + @@ -131154,8 +132179,8 @@ - - + + @@ -131165,8 +132190,8 @@ - - + + @@ -131178,8 +132203,8 @@ - - + + @@ -131193,18 +132218,18 @@ - - - + + + - - - - + + + + - - + + @@ -131212,10 +132237,10 @@ - - - - + + + + @@ -131228,14 +132253,14 @@ - - - - + + + + - - + + @@ -131271,18 +132296,18 @@ - - - + + + - - - + + + @@ -131293,8 +132318,8 @@ - - + + @@ -131305,10 +132330,10 @@ - - - - + + + + @@ -131318,30 +132343,30 @@ - - - - - + + + + + - - + + - - + + - - - + + + - - + + @@ -131360,11 +132385,11 @@ - - - - - + + + + + @@ -131372,13 +132397,13 @@ - - + + - - + + @@ -131395,11 +132420,11 @@ - - - - - + + + + + @@ -131438,8 +132463,8 @@ - - + + @@ -131447,25 +132472,25 @@ - - - + + + - - + + - - - - + + + + - - - - + + + + @@ -131478,9 +132503,9 @@ - - - + + + @@ -131491,13 +132516,13 @@ - - + + - - - + + + @@ -131505,50 +132530,50 @@ - - - + + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -131562,8 +132587,8 @@ - - + + @@ -131639,9 +132664,9 @@ - - - + + + @@ -131679,32 +132704,32 @@ - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - + @@ -131717,9 +132742,9 @@ - - - + + + @@ -131799,10 +132824,10 @@ - - - - + + + + @@ -131847,13 +132872,13 @@ - - - - - - - + + + + + + + @@ -132040,12 +133065,12 @@ - - + + - - + + @@ -132057,11 +133082,11 @@ - - - - - + + + + + @@ -132102,8 +133127,8 @@ - - + + @@ -132156,8 +133181,8 @@ - - + + @@ -132169,9 +133194,9 @@ - - - + + + @@ -132182,28 +133207,28 @@ - - - + + + - - - - - - + + + + + + - - + + - - - - - + + + + + @@ -132214,9 +133239,9 @@ - - - + + + @@ -132277,8 +133302,8 @@ - - + + @@ -132317,9 +133342,9 @@ - - - + + + @@ -132375,8 +133400,8 @@ - - + + @@ -132398,12 +133423,12 @@ - - + + - - + + @@ -132437,8 +133462,8 @@ - - + + @@ -132449,9 +133474,9 @@ - - - + + + @@ -132459,47 +133484,47 @@ - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - - - - + + + + + @@ -132507,18 +133532,18 @@ - - - - + + + + - - + + @@ -132526,14 +133551,14 @@ - - - - + + + + - - - + + + @@ -132569,17 +133594,17 @@ - - + + - - - + + + @@ -132589,9 +133614,9 @@ - - - + + + @@ -132633,12 +133658,12 @@ - - + + - - + + @@ -132652,67 +133677,67 @@ - - - + + + - - + + - - + + - - + + - - + + - - - - - + + + + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + @@ -132720,19 +133745,19 @@ - - + + - - + + - - - - - + + + + + @@ -132743,41 +133768,41 @@ - - - - + + + + - - - + + + - - + + - - - - - + + + + + - - + + - - + + @@ -132791,16 +133816,16 @@ - - - - + + + + - - - - + + + + @@ -132821,11 +133846,11 @@ - - - - - + + + + + @@ -132842,24 +133867,24 @@ - - - - + + + + - - - - - + + + + + - - - - - + + + + + @@ -132884,10 +133909,10 @@ - - - - + + + + @@ -132900,12 +133925,12 @@ - - + + - - + + @@ -132930,18 +133955,18 @@ - - - + + + - - + + - - - + + + @@ -132965,18 +133990,18 @@ - - - - - + + + + + - - - - - + + + + + @@ -132987,19 +134012,19 @@ - - - - + + + + - - + + - - - + + + @@ -133018,14 +134043,14 @@ - - - + + + - - - + + + @@ -133051,9 +134076,9 @@ - - - + + + @@ -133066,22 +134091,22 @@ - - - - + + + + - - + + - - + + @@ -133095,43 +134120,43 @@ - - + + - - - + + + - - - + + + - - - - - + + + + + - - - + + + - - - - - + + + + + @@ -133144,9 +134169,9 @@ - - - + + + @@ -133179,14 +134204,14 @@ - - - - + + + + - - + + @@ -133197,44 +134222,44 @@ - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -133268,16 +134293,16 @@ - - - - - - + + + + + + - - - + + + @@ -133285,9 +134310,9 @@ - - - + + + @@ -133331,33 +134356,33 @@ - - - + + + - - - + + + - - + + - - + + - - - + + + - - - - + + + + @@ -133382,9 +134407,9 @@ - - - + + + @@ -133400,32 +134425,32 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + - - - + + + @@ -133434,9 +134459,9 @@ - - - + + + @@ -133444,74 +134469,74 @@ - - - + + + - - + + - - - + + + - - - + + + - - - - + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + - - + + - - + + - - - - + + + + @@ -133523,42 +134548,42 @@ - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - + + @@ -133573,8 +134598,8 @@ - - + + @@ -133594,8 +134619,8 @@ - - + + @@ -133609,61 +134634,61 @@ - - - - - + + + + + - - - + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - + + + + @@ -133681,12 +134706,12 @@ - - - - - - + + + + + + @@ -133697,12 +134722,12 @@ - - - - - - + + + + + + @@ -133717,9 +134742,9 @@ - - - + + + @@ -133732,15 +134757,15 @@ - - + + - - - - - + + + + + @@ -133757,10 +134782,10 @@ - - - - + + + + @@ -133780,10 +134805,10 @@ - - - - + + + + @@ -133829,10 +134854,10 @@ - - - - + + + + @@ -133847,16 +134872,16 @@ - - - + + + - - + + - - + + @@ -133879,11 +134904,11 @@ - - - - - + + + + + @@ -133913,16 +134938,16 @@ - - - - - - + + + + + + - - + + @@ -134025,33 +135050,33 @@ - - + + - - + + - - - - - - + + + + + + - - - + + + - - + + @@ -134077,8 +135102,8 @@ - - + + @@ -134093,16 +135118,16 @@ - - + + - - + + - - + + @@ -134187,19 +135212,19 @@ - - - + + + - - - + + + - - - + + + @@ -134214,8 +135239,8 @@ - - + + @@ -134246,9 +135271,9 @@ - - - + + + @@ -134278,14 +135303,14 @@ - - - + + + - - - + + + @@ -134298,9 +135323,9 @@ - - - + + + @@ -134308,10 +135333,10 @@ - - - - + + + + @@ -134325,12 +135350,12 @@ - - + + - - + + @@ -134351,9 +135376,9 @@ - - - + + + @@ -134365,9 +135390,9 @@ - - - + + + @@ -134415,8 +135440,8 @@ - - + + @@ -134462,9 +135487,9 @@ - - - + + + @@ -134480,10 +135505,10 @@ - - - - + + + + @@ -134504,10 +135529,10 @@ - - - - + + + + @@ -134540,17 +135565,17 @@ - - + + - - - + + + @@ -134564,55 +135589,55 @@ - - - - + + + + - - + + - - - + + + - - + + - - + + - - - - - + + + + + - - - + + + - - - + + + - - + + - - + + - - + + @@ -134634,20 +135659,20 @@ - - - - - - + + + + + + - - + + - - + + @@ -134655,21 +135680,21 @@ - - - - - + + + + + - - - - - + + + + + - - + + @@ -134680,7 +135705,7 @@ - + @@ -134699,21 +135724,21 @@ - - - - + + + + - - - - - - - - - + + + + + + + + + @@ -134747,8 +135772,8 @@ - - + + @@ -134768,9 +135793,9 @@ - - - + + + @@ -134778,23 +135803,23 @@ - - - + + + - - - + + + - - - + + + @@ -134815,37 +135840,37 @@ - - + + - - - - + + + + - - - + + + - - + + - - - - + + + + @@ -134859,11 +135884,11 @@ - - - - - + + + + + @@ -134873,12 +135898,12 @@ - - - - - - + + + + + + @@ -134922,10 +135947,10 @@ - - - - + + + + @@ -134984,24 +136009,24 @@ - - + + - - + + - - + + @@ -135011,11 +136036,11 @@ - - - - - + + + + + @@ -135060,14 +136085,14 @@ - - - - + + + + - - - + + + @@ -135300,8 +136325,8 @@ - - + + @@ -135321,29 +136346,29 @@ - - - + + + - - + + - - + + - - + + - - + + - - + + @@ -135355,14 +136380,14 @@ - - - + + + - - - + + + @@ -135375,16 +136400,16 @@ - - - - + + + + - - - - + + + + @@ -135436,12 +136461,12 @@ - - + + - - + + @@ -135488,24 +136513,24 @@ - - + + - - + + - - + + - - + + @@ -135525,13 +136550,13 @@ - - + + - - - + + + @@ -135545,16 +136570,16 @@ - - - - + + + + - - - - + + + + @@ -135562,6 +136587,21 @@ + + + + + + + + + + + + + + + @@ -135575,8 +136615,8 @@ - - + + @@ -135591,8 +136631,8 @@ - - + + @@ -135626,8 +136666,8 @@ - - + + @@ -135645,10 +136685,10 @@ - - - - + + + + @@ -135661,54 +136701,54 @@ - - + + - - - + + + - - + + - + - + - - - - + + + + - - - - + + + + - - - - - - - - - + + + + + + + + + - - + + - - + + @@ -135717,36 +136757,36 @@ - - - - - + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + @@ -135757,12 +136797,12 @@ - - - - - - + + + + + + @@ -135781,9 +136821,9 @@ - - - + + + @@ -135793,8 +136833,8 @@ - - + + @@ -135808,10 +136848,10 @@ - - - - + + + + @@ -135824,13 +136864,13 @@ - - - + + + - - + + @@ -135883,11 +136923,11 @@ - - - - - + + + + + @@ -135903,24 +136943,24 @@ - - + + - - + + - - + + - - + + @@ -135928,24 +136968,24 @@ - - + + - - + + - - + + @@ -136005,8 +137045,8 @@ - - + + @@ -136016,28 +137056,28 @@ - - - - - + + + + + - - - - - + + + + + - - - - + + + + @@ -136051,23 +137091,23 @@ - - - + + + - - + + - - - - + + + + - - + + @@ -136090,48 +137130,48 @@ - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + @@ -136140,7 +137180,7 @@ - + @@ -136180,17 +137220,17 @@ - + - + - - + + @@ -136209,8 +137249,8 @@ - - + + @@ -136247,8 +137287,8 @@ - - + + @@ -136260,23 +137300,23 @@ - - - - + + + + - - - + + + - - - + + + @@ -136336,8 +137376,8 @@ - - + + @@ -136378,16 +137418,16 @@ - - + + - - + + - - + + @@ -136398,8 +137438,8 @@ - - + + @@ -136422,12 +137462,12 @@ - - + + - - + + @@ -136443,8 +137483,8 @@ - - + + @@ -136563,10 +137603,10 @@ - - - - + + + + @@ -136575,20 +137615,20 @@ - - - - - - - - + + + + + + + + - - - - + + + + @@ -136596,14 +137636,14 @@ - - - + + + - - - + + + @@ -136621,8 +137661,8 @@ - - + + @@ -136634,8 +137674,8 @@ - - + + @@ -136672,8 +137712,8 @@ - - + + @@ -136724,22 +137764,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -136829,8 +137869,8 @@ - - + + @@ -136838,22 +137878,22 @@ - - + + - - - - - - + + + + + + - - - - + + + + @@ -136866,9 +137906,9 @@ - - - + + + @@ -136876,9 +137916,9 @@ - - - + + + @@ -136926,13 +137966,13 @@ - - - - - - - + + + + + + + @@ -136940,28 +137980,28 @@ - - + + - - + + - - + + - - + + - - + + @@ -136977,20 +138017,20 @@ - - - - - - + + + + + + - - - + + + - - + + @@ -137006,8 +138046,8 @@ - - + + @@ -137024,8 +138064,8 @@ - - + + @@ -137038,15 +138078,15 @@ - - - + + + - - - - + + + + @@ -137054,39 +138094,39 @@ - - - + + + - - - - + + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + @@ -137094,30 +138134,30 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -137244,8 +138284,8 @@ - - + + @@ -137369,12 +138409,12 @@ - - + + - - + + @@ -137391,9 +138431,9 @@ - - - + + + @@ -137401,9 +138441,9 @@ - - - + + + @@ -137411,9 +138451,9 @@ - - - + + + @@ -137438,12 +138478,12 @@ - - + + - - + + @@ -137461,10 +138501,10 @@ - - - - + + + + @@ -137486,67 +138526,67 @@ - - - + + + - - - - + + + + - - - + + + - - + + - - - - - - + + + + + + - - - - + + + + - - - - - - - + + + + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -137557,13 +138597,13 @@ - + - - - - + + + + @@ -137597,10 +138637,10 @@ - - - - + + + + @@ -137642,7 +138682,7 @@ - + @@ -137650,34 +138690,34 @@ - - + + - - + + - - + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -137688,10 +138728,10 @@ - - - - + + + + @@ -137701,8 +138741,8 @@ - - + + @@ -137713,8 +138753,8 @@ - - + + @@ -137752,8 +138792,8 @@ - - + + @@ -137791,14 +138831,14 @@ - - - - - + + + + + - - + + @@ -137836,11 +138876,11 @@ - - - - - + + + + + @@ -137984,21 +139024,21 @@ - - + + - - - + + + - - + + - - + + @@ -138032,11 +139072,11 @@ - - - - - + + + + + @@ -138051,21 +139091,21 @@ - - + + - - + + - - + + - - - + + + @@ -138081,11 +139121,11 @@ - - - - - + + + + + @@ -138109,12 +139149,12 @@ - - - - - - + + + + + + @@ -138134,24 +139174,30 @@ - - - - + + + + - - - - + + + + - - - + + + + + + + + + @@ -138169,26 +139215,26 @@ - - - - + + + + - - - + + + - - - - - + + + + + - - - + + + @@ -138197,20 +139243,20 @@ - - - + + + - - - + + + - - - + + + @@ -138220,26 +139266,26 @@ - - - + + + - - - + + + - - - - + + + + - - - - + + + + @@ -138385,20 +139431,26 @@ - - - + + + + + + + + + - - - + + + @@ -138406,26 +139458,26 @@ - - + + - - - - + + + + - - + + - - + + @@ -138433,20 +139485,25 @@ - - - - - - - - + + + - - - - + + + + + + + + + + + + + + @@ -138458,33 +139515,33 @@ - - - - - - + + + + + + - - - - - - + + + + + + - - - - + + + + - - - + + + @@ -138492,28 +139549,28 @@ - - - + + + - - - - - + + + + + - - + + - - - + + + - - + + @@ -138524,13 +139581,13 @@ - - - - - - - + + + + + + + @@ -138562,14 +139619,14 @@ - - + + - - - - + + + + @@ -138577,10 +139634,10 @@ - - - - + + + + @@ -138588,10 +139645,10 @@ - - - - + + + + @@ -138614,10 +139671,10 @@ - - - - + + + + @@ -138633,6 +139690,10 @@ + + + + @@ -138640,29 +139701,29 @@ - + - - + + - - + + - - - - + + + + - - - - - - + + + + + + @@ -138691,13 +139752,13 @@ - - - - - - - + + + + + + + @@ -138716,10 +139777,10 @@ - - - - + + + + @@ -138736,10 +139797,10 @@ - - - - + + + + @@ -138749,30 +139810,30 @@ - - - + + + - - - - - - + + + + + + - - + + - - - - - - - + + + + + + + @@ -138789,13 +139850,13 @@ - - + + - - - + + + @@ -138814,39 +139875,39 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + - - + + @@ -138869,55 +139930,55 @@ - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -138927,9 +139988,9 @@ - - - + + + @@ -138939,16 +140000,16 @@ - - - - - + + + + + - - - + + + @@ -138983,15 +140044,15 @@ - - - + + + - - - - + + + + @@ -139004,11 +140065,11 @@ - - - - - + + + + + @@ -139018,16 +140079,16 @@ - - - - - + + + + + - - - + + + @@ -139037,8 +140098,8 @@ - - + + @@ -139050,25 +140111,25 @@ - - + + - - + + - - - - - - - + + + + + + + - - + + @@ -139079,9 +140140,9 @@ - - - + + + @@ -139133,9 +140194,9 @@ - - - + + + @@ -139169,22 +140230,22 @@ - - - - - + + + + + - - - - - + + + + + - - + + @@ -139263,8 +140324,8 @@ - - + + @@ -139274,9 +140335,9 @@ - - - + + + @@ -139328,8 +140389,8 @@ - - + + @@ -139345,9 +140406,9 @@ - - - + + + @@ -139357,17 +140418,17 @@ - - - - + + + + - - - - - + + + + + @@ -139391,17 +140452,17 @@ - - + + - - - - - - - + + + + + + + @@ -139410,23 +140471,23 @@ - - + + - - - - + + + + - - - + + + - - + + @@ -139437,12 +140498,12 @@ - - - - - - + + + + + + @@ -139454,8 +140515,8 @@ - - + + @@ -139467,18 +140528,18 @@ - - - - - - + + + + + + - - - - + + + + @@ -139486,12 +140547,12 @@ - - + + - - + + @@ -139529,9 +140590,9 @@ - - - + + + @@ -139543,8 +140604,8 @@ - - + + @@ -139576,45 +140637,45 @@ - - - + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - - + + + + + @@ -139659,9 +140720,9 @@ - - - + + + @@ -139692,10 +140753,10 @@ - - - - + + + + @@ -139787,14 +140848,14 @@ - - - - + + + + - - + + @@ -139812,8 +140873,8 @@ - - + + @@ -140064,9 +141125,9 @@ - - - + + + @@ -140155,9 +141216,9 @@ - - - + + + @@ -140178,13 +141239,13 @@ - - - - - - - + + + + + + + @@ -140196,44 +141257,44 @@ - - - + + + - - - - + + + + - - + + - - + + - - - - - - + + + + + + - - + + - - - + + + @@ -140343,9 +141404,9 @@ - - - + + + @@ -140357,17 +141418,17 @@ - - - - - - + + + + + + - - - + + + @@ -140491,9 +141552,9 @@ - - - + + + @@ -140526,34 +141587,34 @@ - - + + - - + + - - + + - - + + - - - - + + + + - - - - - - + + + + + + @@ -140576,9 +141637,9 @@ - - - + + + @@ -140596,8 +141657,8 @@ - - + + @@ -140632,10 +141693,10 @@ - - - - + + + + @@ -140644,21 +141705,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -140666,21 +141727,21 @@ - - - - + + + + - - - - - + + + + + - - - + + + @@ -140699,16 +141760,16 @@ - - - + + + - - - + + + - - + + @@ -140726,43 +141787,47 @@ - - + + - - + + - - + + - - + + - - + + - - - + + + + + + + - + - + @@ -140774,12 +141839,12 @@ - - + + - - + + @@ -140794,23 +141859,23 @@ - - + + - - + + - - - + + + - + - + @@ -140829,15 +141894,15 @@ - - - + + + - - - - + + + + @@ -140850,27 +141915,27 @@ - - - + + + - - - - + + + + - - - + + + - - - - + + + + @@ -140892,9 +141957,9 @@ - - - + + + @@ -140903,10 +141968,10 @@ - - - - + + + + @@ -140914,9 +141979,9 @@ - - - + + + @@ -140924,9 +141989,9 @@ - - - + + + @@ -140935,17 +142000,17 @@ - - - - + + + + - + @@ -140963,8 +142028,8 @@ - - + + @@ -141007,8 +142072,8 @@ - - + + @@ -141101,8 +142166,16 @@ - - + + + + + + + + + + @@ -141111,8 +142184,8 @@ - - + + @@ -141144,19 +142217,19 @@ - - - - - - - - - + + + + + + + + + - - + + @@ -141168,9 +142241,9 @@ - - - + + + @@ -141189,14 +142262,14 @@ - - - - + + + + - - + + @@ -141224,9 +142297,9 @@ - - - + + + @@ -141249,11 +142322,12 @@ + - - + + @@ -141288,19 +142362,19 @@ - - + + - - - - + + + + - - - + + + @@ -141318,8 +142392,8 @@ - - + + @@ -141329,24 +142403,24 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -141356,26 +142430,26 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + @@ -141384,10 +142458,10 @@ - - - - + + + + @@ -141404,7 +142478,7 @@ - + @@ -141438,9 +142512,9 @@ - - - + + + @@ -141578,10 +142652,10 @@ - - - - + + + + @@ -141589,10 +142663,10 @@ - - - - + + + + @@ -141671,19 +142745,19 @@ - - - + + + - - - + + + - - - + + + @@ -141713,8 +142787,8 @@ - - + + @@ -141732,16 +142806,16 @@ - - + + - - + + @@ -141794,9 +142868,9 @@ - - - + + + @@ -141810,9 +142884,9 @@ - - - + + + @@ -141851,8 +142925,8 @@ - - + + @@ -141898,9 +142972,9 @@ - - - + + + @@ -141991,11 +143065,11 @@ - - - - - + + + + + @@ -142230,9 +143304,9 @@ - - - + + + @@ -142350,27 +143424,27 @@ - - + + - - + + - - + + - - - - - + + + + + @@ -142404,8 +143478,8 @@ - - + + @@ -142444,17 +143518,17 @@ - - + + - - + + - - - + + + @@ -142465,8 +143539,8 @@ - - + + @@ -142497,12 +143571,12 @@ - - + + - - + + @@ -142525,8 +143599,8 @@ - - + + @@ -142548,8 +143622,8 @@ - - + + @@ -142585,9 +143659,9 @@ - - - + + + @@ -142610,9 +143684,9 @@ - - - + + + @@ -142627,16 +143701,16 @@ - - + + - - + + - - + + @@ -142655,8 +143729,8 @@ - - + + @@ -142671,16 +143745,16 @@ - - - - - - - - - - + + + + + + + + + + @@ -142714,8 +143788,8 @@ - - + + @@ -142730,15 +143804,15 @@ - - - - + + + + - - - + + + @@ -142749,9 +143823,9 @@ - - - + + + @@ -142763,9 +143837,9 @@ - - - + + + @@ -142784,24 +143858,24 @@ - - - + + + - - - - + + + + - - - + + + @@ -142828,8 +143902,8 @@ - - + + @@ -142848,9 +143922,9 @@ - - - + + + @@ -142859,28 +143933,28 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + @@ -142889,14 +143963,14 @@ - - - - + + + + - - + + @@ -142907,9 +143981,9 @@ - - - + + + @@ -142924,8 +143998,8 @@ - - + + @@ -142947,8 +144021,8 @@ - - + + @@ -143051,15 +144125,15 @@ - - - + + + - - - - + + + + @@ -143071,8 +144145,8 @@ - - + + @@ -143138,16 +144212,16 @@ - - + + - - + + @@ -143155,10 +144229,10 @@ - - - - + + + + @@ -143182,38 +144256,38 @@ - - + + - - + + - - - + + + - - - + + + - - + + - - + + @@ -143221,10 +144295,10 @@ - - - - + + + + @@ -143248,9 +144322,9 @@ - - - + + + @@ -143258,8 +144332,8 @@ - - + + @@ -143308,9 +144382,9 @@ - - - + + + @@ -143331,10 +144405,10 @@ - - - - + + + + @@ -143384,9 +144458,9 @@ - - - + + + @@ -143446,50 +144520,50 @@ - - - + + + - - - - - - + + + + + + - - - - - + + + + + - - + + - - - - + + + + - - - - - - + + + + + + - - + + @@ -143665,9 +144739,9 @@ - - - + + + @@ -143681,27 +144755,27 @@ - - + + - - - - + + + + - - - + + + @@ -143715,33 +144789,33 @@ - - - - - + + + + + - - - + + + - - + + - - + + - - + + - - - + + + @@ -143987,15 +145061,15 @@ - - - + + + - - - - + + + + @@ -144046,15 +145120,15 @@ - - - + + + - - - + + + @@ -144067,15 +145141,15 @@ - - - - + + + + - - - + + + @@ -144153,9 +145227,9 @@ - - - + + + @@ -144171,36 +145245,36 @@ - - - + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + @@ -144215,9 +145289,9 @@ - - - + + + @@ -144232,14 +145306,14 @@ - - - + + + - - - + + + @@ -144281,15 +145355,15 @@ - - + + - - - - - + + + + + @@ -144297,11 +145371,11 @@ - - - - - + + + + + @@ -144311,14 +145385,32 @@ + + + + + + + + + + + + - - + + + + + + + + @@ -144338,8 +145430,8 @@ - - + + @@ -144382,8 +145474,8 @@ - - + + @@ -144419,8 +145511,8 @@ - - + + @@ -144487,9 +145579,9 @@ - - - + + + @@ -144560,26 +145652,26 @@ - - - - - - - + + + + + + + - - + + - - - + + + @@ -144616,21 +145708,21 @@ - - - + + + - - - + + + - - - + + + @@ -144697,15 +145789,15 @@ - - + + - - + + @@ -144716,16 +145808,16 @@ - - - - - + + + + + - - - + + + @@ -144734,11 +145826,11 @@ - - - - - + + + + + @@ -144747,9 +145839,9 @@ - - - + + + @@ -144799,11 +145891,11 @@ - - - - - + + + + + @@ -144813,23 +145905,23 @@ - - - - + + + + - - - + + + - - + + - - + + @@ -144848,12 +145940,12 @@ - - + + - - + + @@ -144861,8 +145953,8 @@ - - + + @@ -144883,8 +145975,8 @@ - - + + @@ -144892,16 +145984,16 @@ - - - + + + - + - - + + @@ -144909,8 +146001,8 @@ - - + + @@ -144926,9 +146018,9 @@ - - - + + + @@ -144951,17 +146043,17 @@ - - - + + + - - + + @@ -144978,12 +146070,12 @@ - - - - - - + + + + + + @@ -145005,6 +146097,12 @@ + + + + + + @@ -145038,9 +146136,9 @@ - - - + + + @@ -145065,34 +146163,34 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + @@ -145102,17 +146200,17 @@ - - + + - - - - + + + + @@ -145123,26 +146221,26 @@ - - - - + + + + - - - - + + + + - - + + - - - - + + + + @@ -145153,9 +146251,9 @@ - - - + + + @@ -145187,10 +146285,10 @@ - - - - + + + + @@ -145223,9 +146321,9 @@ - - - + + + @@ -145242,15 +146340,15 @@ - - - - + + + + - - - + + + @@ -145274,12 +146372,12 @@ - - - - - - + + + + + + @@ -145301,13 +146399,13 @@ - - + + - - - + + + @@ -145324,10 +146422,10 @@ - - - - + + + + @@ -145338,47 +146436,47 @@ - - - + + + - - + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index 7664c86d1179..72015e048a60 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -63,6 +63,7 @@ bdput bio_add_pc_page bio_alloc_bioset + bio_associate_blkg_from_css bio_endio bio_put bio_reset @@ -83,6 +84,7 @@ bitmap_zalloc blk_alloc_queue blk_cleanup_queue + blkdev_issue_zeroout blk_execute_rq blk_execute_rq_nowait blk_get_request @@ -90,6 +92,7 @@ blk_mq_rq_cpu blk_mq_sched_mark_restart_hctx blk_mq_start_request + blk_op_str blk_put_request blk_queue_flag_clear blk_queue_flag_set @@ -152,6 +155,7 @@ class_unregister __cleancache_get_page cleanup_srcu_struct + clear_nlink clear_page __ClearPageMovable clk_bulk_disable @@ -372,6 +376,7 @@ del_gendisk del_timer del_timer_sync + dentry_path_raw desc_to_gpio destroy_workqueue dev_alloc_name @@ -590,6 +595,9 @@ devres_release dev_set_name _dev_warn + d_find_alias + d_instantiate_new + d_invalidate disable_irq disable_irq_nosync disable_percpu_irq @@ -681,6 +689,39 @@ down_write d_path dput + dqget + dqput + dquot_acquire + dquot_alloc + dquot_alloc_inode + __dquot_alloc_space + dquot_claim_space_nodirty + dquot_commit + dquot_commit_info + dquot_destroy + dquot_disable + dquot_drop + dquot_file_open + dquot_free_inode + __dquot_free_space + dquot_get_dqblk + dquot_get_next_dqblk + dquot_get_next_id + dquot_get_state + dquot_initialize + dquot_initialize_needed + dquot_load_quota_inode + dquot_mark_dquot_dirty + dquot_quota_off + dquot_quota_on + dquot_quota_on_mount + dquot_release + dquot_resume + dquot_set_dqblk + dquot_set_dqinfo + __dquot_transfer + dquot_transfer + dquot_writeback_dquots drain_workqueue driver_create_file driver_find_device @@ -820,6 +861,7 @@ drm_writeback_connector_init drm_writeback_queue_job drm_writeback_signal_completion + d_tmpfile dump_stack __dynamic_pr_debug edac_device_add_device @@ -850,6 +892,7 @@ eventfd_ctx_remove_wait_queue eventfd_signal event_triggers_call + evict_inodes extcon_get_edev_by_phandle extcon_get_edev_name extcon_get_property @@ -867,6 +910,7 @@ filp_close filp_open_block find_get_pid + find_inode_nowait find_last_bit find_next_bit find_next_zero_bit @@ -897,6 +941,8 @@ free_pages_exact free_percpu free_percpu_irq + freeze_bdev + freeze_super freezing_slow_path freq_qos_add_notifier freq_qos_add_request @@ -905,11 +951,60 @@ freq_qos_update_request freq_scale fs_bio_set + fscrypt_decrypt_bio + fscrypt_dio_supported + fscrypt_drop_inode + fscrypt_encrypt_pagecache_blocks + __fscrypt_encrypt_symlink + fscrypt_file_open + fscrypt_fname_alloc_buffer + fscrypt_fname_disk_to_usr + fscrypt_fname_free_buffer + fscrypt_fname_siphash + fscrypt_free_bounce_page + fscrypt_free_inode + fscrypt_get_symlink + fscrypt_has_permitted_context + __fscrypt_inode_uses_inline_crypto + fscrypt_ioctl_add_key + fscrypt_ioctl_get_key_status + fscrypt_ioctl_get_nonce + fscrypt_ioctl_get_policy + fscrypt_ioctl_get_policy_ex + fscrypt_ioctl_remove_key + fscrypt_ioctl_remove_key_all_users + fscrypt_ioctl_set_policy + fscrypt_match_name + fscrypt_mergeable_bio + __fscrypt_prepare_link + __fscrypt_prepare_lookup + fscrypt_prepare_new_inode + __fscrypt_prepare_readdir + __fscrypt_prepare_rename + __fscrypt_prepare_setattr + fscrypt_prepare_symlink + fscrypt_put_encryption_info + fscrypt_set_bio_crypt_ctx + fscrypt_set_context + fscrypt_set_test_dummy_encryption + fscrypt_setup_filename + fscrypt_show_test_dummy_encryption + fscrypt_symlink_getattr + fscrypt_zeroout_range fsg_common_create_luns fsg_common_set_cdev fsg_common_set_inquiry_string fsg_common_set_sysfs fsg_config_from_params + fsverity_cleanup_inode + fsverity_enqueue_verify_work + fsverity_file_open + fsverity_ioctl_enable + fsverity_ioctl_measure + fsverity_ioctl_read_metadata + fsverity_prepare_setattr + fsverity_verify_bio + fsverity_verify_page fsync_bdev fwnode_device_is_available fwnode_find_reference @@ -930,10 +1025,15 @@ gcd generate_random_uuid generic_device_group + generic_fh_to_dentry + generic_fh_to_parent generic_file_llseek + generic_file_llseek_size + __generic_file_write_iter generic_handle_irq generic_iommu_put_resv_regions generic_mii_ioctl + generic_set_encrypted_ci_d_ops genlmsg_multicast_allns genlmsg_put genl_register_family @@ -1089,6 +1189,8 @@ idr_preload idr_remove idr_replace + iget_failed + iget_locked iio_alloc_pollfunc iio_buffer_init iio_buffer_put @@ -1106,6 +1208,7 @@ iio_read_channel_processed iio_read_channel_raw iio_trigger_notify_done + ilookup import_iovec inc_node_page_state inc_zone_page_state @@ -1124,6 +1227,7 @@ init_uts_ns init_wait_entry __init_waitqueue_head + inode_nohighmem inode_owner_or_capable inode_permission input_alloc_absinfo @@ -1145,6 +1249,7 @@ input_unregister_device input_unregister_handle input_unregister_handler + insert_inode_locked interval_tree_insert interval_tree_iter_first interval_tree_iter_next @@ -1876,6 +1981,9 @@ pm_wakeup_dev_event pm_wakeup_ws_event pm_wq + posix_acl_alloc + posix_acl_chmod + posix_acl_equiv_mode power_supply_changed power_supply_get_by_name power_supply_get_drvdata @@ -2237,6 +2345,7 @@ send_sig send_sig_info seq_buf_printf + seq_escape seq_hex_dump seq_lseek seq_open @@ -2259,11 +2368,13 @@ serial8250_rpm_put serial8250_suspend_port serial8250_unregister_port + set_cached_acl set_cpus_allowed_ptr set_normalized_timespec64 set_page_dirty_lock __SetPageMovable set_task_cpu + set_task_ioprio set_user_nice sg_alloc_table sg_alloc_table_from_pages @@ -2279,6 +2390,7 @@ __sg_page_iter_start sg_scsi_ioctl show_regs + shrink_dcache_sb shrink_slab sigprocmask si_mem_available @@ -2471,6 +2583,7 @@ synchronize_net synchronize_rcu synchronize_srcu + sync_inodes_sb syscon_node_to_regmap syscon_regmap_lookup_by_compatible syscon_regmap_lookup_by_phandle @@ -2518,6 +2631,8 @@ task_rq_lock tcp_hashinfo tcp_parse_options + thaw_bdev + thaw_super thermal_cooling_device_register thermal_cooling_device_unregister thermal_of_cooling_device_register @@ -2633,10 +2748,10 @@ __traceiter_android_vh_binder_reply __traceiter_android_vh_binder_restore_priority __traceiter_android_vh_binder_set_priority - __traceiter_android_vh_binder_trans - __traceiter_android_vh_binder_transaction_init __traceiter_android_vh_binder_thread_read __traceiter_android_vh_binder_thread_release + __traceiter_android_vh_binder_trans + __traceiter_android_vh_binder_transaction_init __traceiter_android_vh_binder_wait_for_work __traceiter_android_vh_binder_wakeup_ilocked __traceiter_android_vh_build_sched_domains @@ -2648,6 +2763,7 @@ __traceiter_android_vh_check_uninterruptible_tasks_dn __traceiter_android_vh_clear_mask_adjust __traceiter_android_vh_clear_reserved_fmt_fields + __traceiter_android_vh_cma_drain_all_pages_bypass __traceiter_android_vh_commit_creds __traceiter_android_vh_cpufreq_acct_update_power __traceiter_android_vh_cpufreq_fast_switch @@ -2657,6 +2773,7 @@ __traceiter_android_vh_cpu_idle_exit __traceiter_android_vh_cpu_up __traceiter_android_vh_do_send_sig_info + __traceiter_android_vh_drain_all_pages_bypass __traceiter_android_vh_em_cpu_energy __traceiter_android_vh_exclude_reserved_zone __traceiter_android_vh_exit_creds @@ -2690,6 +2807,8 @@ __traceiter_android_vh_mutex_wait_finish __traceiter_android_vh_mutex_wait_start __traceiter_android_vh_override_creds + __traceiter_android_vh_page_referenced_check_bypass + __traceiter_android_vh_pcplist_add_cma_pages_bypass __traceiter_android_vh_prepare_update_load_avg_se __traceiter_android_vh_printk_hotplug __traceiter_android_vh_process_killed @@ -2727,10 +2846,6 @@ __traceiter_android_vh_tune_inactive_ratio __traceiter_android_vh_tune_scan_type __traceiter_android_vh_tune_swappiness - __traceiter_android_vh_page_referenced_check_bypass - __traceiter_android_vh_drain_all_pages_bypass - __traceiter_android_vh_cma_drain_all_pages_bypass - __traceiter_android_vh_pcplist_add_cma_pages_bypass __traceiter_android_vh_ufs_compl_command __traceiter_android_vh_ufs_send_command __traceiter_android_vh_ufs_send_tm_command @@ -2855,6 +2970,7 @@ __tracepoint_android_vh_check_uninterruptible_tasks_dn __tracepoint_android_vh_clear_mask_adjust __tracepoint_android_vh_clear_reserved_fmt_fields + __tracepoint_android_vh_cma_drain_all_pages_bypass __tracepoint_android_vh_commit_creds __tracepoint_android_vh_cpufreq_acct_update_power __tracepoint_android_vh_cpufreq_fast_switch @@ -2864,6 +2980,7 @@ __tracepoint_android_vh_cpu_idle_exit __tracepoint_android_vh_cpu_up __tracepoint_android_vh_do_send_sig_info + __tracepoint_android_vh_drain_all_pages_bypass __tracepoint_android_vh_em_cpu_energy __tracepoint_android_vh_exclude_reserved_zone __tracepoint_android_vh_exit_creds @@ -2897,6 +3014,8 @@ __tracepoint_android_vh_mutex_wait_finish __tracepoint_android_vh_mutex_wait_start __tracepoint_android_vh_override_creds + __tracepoint_android_vh_page_referenced_check_bypass + __tracepoint_android_vh_pcplist_add_cma_pages_bypass __tracepoint_android_vh_prepare_update_load_avg_se __tracepoint_android_vh_printk_hotplug __tracepoint_android_vh_process_killed @@ -2934,10 +3053,6 @@ __tracepoint_android_vh_tune_inactive_ratio __tracepoint_android_vh_tune_scan_type __tracepoint_android_vh_tune_swappiness - __tracepoint_android_vh_page_referenced_check_bypass - __tracepoint_android_vh_drain_all_pages_bypass - __tracepoint_android_vh_cma_drain_all_pages_bypass - __tracepoint_android_vh_pcplist_add_cma_pages_bypass __tracepoint_android_vh_ufs_compl_command __tracepoint_android_vh_ufs_send_command __tracepoint_android_vh_ufs_send_tm_command @@ -3328,6 +3443,9 @@ verify_pkcs7_signature vfree vfs_fsync + vfs_ioc_fssetxattr_check + vfs_ioc_setflags_prepare + vfs_setpos video_devdata video_device_alloc video_device_release @@ -3394,6 +3512,7 @@ __warn_printk watchdog_init_timeout watchdog_set_restart_priority + wbc_account_cgroup_owner wireless_nlevent_flush woken_wake_function work_busy From b07625698eb56d47bc055e45197f9f5ace1875a3 Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Thu, 3 Mar 2022 09:38:47 -0500 Subject: [PATCH 035/156] BACKPORT: ext4: don't BUG if someone dirty pages without asking ext4 first [ Upstream commit cc5095747edfb054ca2068d01af20be3fcc3634f ] [un]pin_user_pages_remote is dirtying pages without properly warning the file system in advance. A related race was noted by Jan Kara in 2018[1]; however, more recently instead of it being a very hard-to-hit race, it could be reliably triggered by process_vm_writev(2) which was discovered by Syzbot[2]. This is technically a bug in mm/gup.c, but arguably ext4 is fragile in that if some other kernel subsystem dirty pages without properly notifying the file system using page_mkwrite(), ext4 will BUG, while other file systems will not BUG (although data will still be lost). So instead of crashing with a BUG, issue a warning (since there may be potential data loss) and just mark the page as clean to avoid unprivileged denial of service attacks until the problem can be properly fixed. More discussion and background can be found in the thread starting at [2]. [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz [2] https://lore.kernel.org/r/Yg0m6IjcNmfaSokM@google.com Reported-by: syzbot+d59332e2db681cf18f0318a06e994ebbb529a8db@syzkaller.appspotmail.com Reported-by: Lee Jones Signed-off-by: Theodore Ts'o Link: https://lore.kernel.org/r/YiDS9wVfq4mM2jGK@mit.edu Signed-off-by: Sasha Levin Signed-off-by: Lee Jones Change-Id: I974915cfe58a2f773b025db8864f4b7927de2153 --- fs/ext4/inode.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index c0b679216727..a6b3a5553fde 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -2036,6 +2036,15 @@ static int ext4_writepage(struct page *page, else len = PAGE_SIZE; + /* Should never happen but for bugs in other kernel subsystems */ + if (!page_has_buffers(page)) { + ext4_warning_inode(inode, + "page %lu does not have buffers attached", page->index); + ClearPageDirty(page); + unlock_page(page); + return 0; + } + page_bufs = page_buffers(page); /* * We cannot do block allocation or other extent handling in this @@ -2639,6 +2648,22 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) wait_on_page_writeback(page); BUG_ON(PageWriteback(page)); + /* + * Should never happen but for buggy code in + * other subsystems that call + * set_page_dirty() without properly warning + * the file system first. See [1] for more + * information. + * + * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz + */ + if (!page_has_buffers(page)) { + ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", page->index); + ClearPageDirty(page); + unlock_page(page); + continue; + } + if (mpd->map.m_len == 0) mpd->first_page = page->index; mpd->next_page = page->index + 1; From 31beefbf149b75b854a4d22b4deb3a648f4ad6f9 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Mon, 11 Apr 2022 11:42:03 +0200 Subject: [PATCH 036/156] BACKPORT: nl80211: correctly check NL80211_ATTR_REG_ALPHA2 size commit 6624bb34b4eb19f715db9908cca00122748765d7 upstream. We need this to be at least two bytes, so we can access alpha2[0] and alpha2[1]. It may be three in case some userspace used NUL-termination since it was NLA_STRING (and we also push it out with NUL-termination). Cc: stable@vger.kernel.org Reported-by: Lee Jones Link: https://lore.kernel.org/r/20220411114201.fd4a31f06541.Ie7ff4be2cf348d8cc28ed0d626fc54becf7ea799@changeid Signed-off-by: Johannes Berg Signed-off-by: Greg Kroah-Hartman Signed-off-by: Lee Jones Change-Id: Ib76876c2aa89aacf4c31d95b751f8b2d27788559 --- net/wireless/nl80211.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 0890b42153f5..9baca999e533 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -475,7 +475,8 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { .len = IEEE80211_MAX_MESH_ID_LEN }, [NL80211_ATTR_MPATH_NEXT_HOP] = NLA_POLICY_ETH_ADDR_COMPAT, - [NL80211_ATTR_REG_ALPHA2] = { .type = NLA_STRING, .len = 2 }, + /* allow 3 for NUL-termination, we used to declare this NLA_STRING */ + [NL80211_ATTR_REG_ALPHA2] = NLA_POLICY_RANGE(NLA_BINARY, 2, 3), [NL80211_ATTR_REG_RULES] = { .type = NLA_NESTED }, [NL80211_ATTR_BSS_CTS_PROT] = { .type = NLA_U8 }, From 6def3a5ed8ee24465edf6b30cdbe4b27c6d4e1df Mon Sep 17 00:00:00 2001 From: Chen-Yu Tsai Date: Thu, 9 Dec 2021 17:38:03 +0100 Subject: [PATCH 037/156] BACKPORT: media: v4l2-mem2mem: Apply DST_QUEUE_OFF_BASE on MMAP buffers across ioctls [ Upstream commit 8310ca94075e784bbb06593cd6c068ee6b6e4ca6 ] DST_QUEUE_OFF_BASE is applied to offset/mem_offset on MMAP capture buffers only for the VIDIOC_QUERYBUF ioctl, while the userspace fields (including offset/mem_offset) are filled in for VIDIOC_{QUERY,PREPARE,Q,DQ}BUF ioctls. This leads to differences in the values presented to userspace. If userspace attempts to mmap the capture buffer directly using values from DQBUF, it will fail. Move the code that applies the magic offset into a helper, and call that helper from all four ioctl entry points. [hverkuil: drop unnecessary '= 0' in v4l2_m2m_querybuf() for ret] Bug: 223375145 Fixes: 7f98639def42 ("V4L/DVB: add memory-to-memory device helper framework for videobuf") Fixes: 908a0d7c588e ("[media] v4l: mem2mem: port to videobuf2") Signed-off-by: Chen-Yu Tsai Signed-off-by: Hans Verkuil Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Sasha Levin Signed-off-by: Lee Jones Change-Id: Ifab9933f85f4ba7e0fadddf52debaf837830a06d --- drivers/media/v4l2-core/v4l2-mem2mem.c | 53 ++++++++++++++++++++------ 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/drivers/media/v4l2-core/v4l2-mem2mem.c b/drivers/media/v4l2-core/v4l2-mem2mem.c index b221b4e438a1..73190652c267 100644 --- a/drivers/media/v4l2-core/v4l2-mem2mem.c +++ b/drivers/media/v4l2-core/v4l2-mem2mem.c @@ -585,19 +585,14 @@ int v4l2_m2m_reqbufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, } EXPORT_SYMBOL_GPL(v4l2_m2m_reqbufs); -int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, - struct v4l2_buffer *buf) +static void v4l2_m2m_adjust_mem_offset(struct vb2_queue *vq, + struct v4l2_buffer *buf) { - struct vb2_queue *vq; - int ret = 0; - unsigned int i; - - vq = v4l2_m2m_get_vq(m2m_ctx, buf->type); - ret = vb2_querybuf(vq, buf); - /* Adjust MMAP memory offsets for the CAPTURE queue */ if (buf->memory == V4L2_MEMORY_MMAP && V4L2_TYPE_IS_CAPTURE(vq->type)) { if (V4L2_TYPE_IS_MULTIPLANAR(vq->type)) { + unsigned int i; + for (i = 0; i < buf->length; ++i) buf->m.planes[i].m.mem_offset += DST_QUEUE_OFF_BASE; @@ -605,8 +600,23 @@ int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, buf->m.offset += DST_QUEUE_OFF_BASE; } } +} - return ret; +int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, + struct v4l2_buffer *buf) +{ + struct vb2_queue *vq; + int ret; + + vq = v4l2_m2m_get_vq(m2m_ctx, buf->type); + ret = vb2_querybuf(vq, buf); + if (ret) + return ret; + + /* Adjust MMAP memory offsets for the CAPTURE queue */ + v4l2_m2m_adjust_mem_offset(vq, buf); + + return 0; } EXPORT_SYMBOL_GPL(v4l2_m2m_querybuf); @@ -763,6 +773,9 @@ int v4l2_m2m_qbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, if (ret) return ret; + /* Adjust MMAP memory offsets for the CAPTURE queue */ + v4l2_m2m_adjust_mem_offset(vq, buf); + /* * If the capture queue is streaming, but streaming hasn't started * on the device, but was asked to stop, mark the previously queued @@ -784,9 +797,17 @@ int v4l2_m2m_dqbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, struct v4l2_buffer *buf) { struct vb2_queue *vq; + int ret; vq = v4l2_m2m_get_vq(m2m_ctx, buf->type); - return vb2_dqbuf(vq, buf, file->f_flags & O_NONBLOCK); + ret = vb2_dqbuf(vq, buf, file->f_flags & O_NONBLOCK); + if (ret) + return ret; + + /* Adjust MMAP memory offsets for the CAPTURE queue */ + v4l2_m2m_adjust_mem_offset(vq, buf); + + return 0; } EXPORT_SYMBOL_GPL(v4l2_m2m_dqbuf); @@ -795,9 +816,17 @@ int v4l2_m2m_prepare_buf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, { struct video_device *vdev = video_devdata(file); struct vb2_queue *vq; + int ret; vq = v4l2_m2m_get_vq(m2m_ctx, buf->type); - return vb2_prepare_buf(vq, vdev->v4l2_dev->mdev, buf); + ret = vb2_prepare_buf(vq, vdev->v4l2_dev->mdev, buf); + if (ret) + return ret; + + /* Adjust MMAP memory offsets for the CAPTURE queue */ + v4l2_m2m_adjust_mem_offset(vq, buf); + + return 0; } EXPORT_SYMBOL_GPL(v4l2_m2m_prepare_buf); From f06daa5a0bc6c8b524b53671fce40e0a14c4f9af Mon Sep 17 00:00:00 2001 From: liang zhang Date: Thu, 21 Apr 2022 09:43:37 +0000 Subject: [PATCH 038/156] ANDROID: add for tuning readahead size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tune ReadAhead size for better memory usage and performance. accordding to Read-Ahead Efficiency on Mobile Devices: Observation, Characterization, and Optimization form IEEE Bug: 229839032 Change-Id: I91656bde5e616e181fd7557554d55e7ce1858136 Signed-off-by: liang zhang --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/mm.h | 4 ++++ mm/readahead.c | 2 ++ 3 files changed, 7 insertions(+) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 2ae0b6eecc37..7b5148376de2 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -397,3 +397,4 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_has_work_ilocked); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_read_done); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_handle_tlb_conf); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_node_memcgs); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ra_tuning_max_page); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index bde2f3556c2a..4af456345aa4 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -132,6 +132,10 @@ struct device; DECLARE_HOOK(android_vh_subpage_dma_contig_alloc, TP_PROTO(bool *allow_subpage_alloc, struct device *dev, size_t *size), TP_ARGS(allow_subpage_alloc, dev, size)); +struct readahead_control; +DECLARE_HOOK(android_vh_ra_tuning_max_page, + TP_PROTO(struct readahead_control *ractl, unsigned long *max_page), + TP_ARGS(ractl, max_page)); /* macro versions of hooks are no longer required */ #endif /* _TRACE_HOOK_MM_H */ diff --git a/mm/readahead.c b/mm/readahead.c index a6bfa987a04a..a95364c99487 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -459,6 +459,8 @@ static void ondemand_readahead(struct readahead_control *ractl, if (req_size > max_pages && bdi->io_pages > max_pages) max_pages = min(req_size, bdi->io_pages); + trace_android_vh_ra_tuning_max_page(ractl, &max_pages); + /* * start of file */ From 46f414b1c2431bda695f58aa16285bbd80f6737d Mon Sep 17 00:00:00 2001 From: Xie Yongji Date: Tue, 26 Oct 2021 22:40:15 +0800 Subject: [PATCH 039/156] BACKPORT: virtio-blk: Use blk_validate_block_size() to validate block size The block layer can't support a block size larger than page size yet. And a block size that's too small or not a power of two won't work either. If a misconfigured device presents an invalid block size in configuration space, it will result in the kernel crash something like below: [ 506.154324] BUG: kernel NULL pointer dereference, address: 0000000000000008 [ 506.160416] RIP: 0010:create_empty_buffers+0x24/0x100 [ 506.174302] Call Trace: [ 506.174651] create_page_buffers+0x4d/0x60 [ 506.175207] block_read_full_page+0x50/0x380 [ 506.175798] ? __mod_lruvec_page_state+0x60/0xa0 [ 506.176412] ? __add_to_page_cache_locked+0x1b2/0x390 [ 506.177085] ? blkdev_direct_IO+0x4a0/0x4a0 [ 506.177644] ? scan_shadow_nodes+0x30/0x30 [ 506.178206] ? lru_cache_add+0x42/0x60 [ 506.178716] do_read_cache_page+0x695/0x740 [ 506.179278] ? read_part_sector+0xe0/0xe0 [ 506.179821] read_part_sector+0x36/0xe0 [ 506.180337] adfspart_check_ICS+0x32/0x320 [ 506.180890] ? snprintf+0x45/0x70 [ 506.181350] ? read_part_sector+0xe0/0xe0 [ 506.181906] bdev_disk_changed+0x229/0x5c0 [ 506.182483] blkdev_get_whole+0x6d/0x90 [ 506.183013] blkdev_get_by_dev+0x122/0x2d0 [ 506.183562] device_add_disk+0x39e/0x3c0 [ 506.184472] virtblk_probe+0x3f8/0x79b [virtio_blk] [ 506.185461] virtio_dev_probe+0x15e/0x1d0 [virtio] So let's use a block layer helper to validate the block size. Signed-off-by: Xie Yongji Acked-by: Michael S. Tsirkin Link: https://lore.kernel.org/r/20211026144015.188-5-xieyongji@bytedance.com Signed-off-by: Jens Axboe (cherry picked from commit 57a13a5b8157d9a8606490aaa1b805bafe6c37e1) [keirf@: Implement missing error path] Bug: 226679849 Signed-off-by: Keir Fraser Change-Id: I78cde1101baf8da2f68d0b9f942a0f1ec89fb30e (cherry picked from commit 588affc843da96cda1747b4caa8fcd9bd8796d3c) --- drivers/block/virtio_blk.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 018fb172a098..132b019695f3 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -827,9 +827,17 @@ static int virtblk_probe(struct virtio_device *vdev) err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE, struct virtio_blk_config, blk_size, &blk_size); - if (!err) + if (!err) { + err = blk_validate_block_size(blk_size); + if (err) { + dev_err(&vdev->dev, + "virtio_blk: invalid block size: 0x%x\n", + blk_size); + goto out_cleanup_disk; + } + blk_queue_logical_block_size(q, blk_size); - else + } else blk_size = queue_logical_block_size(q); /* Use topology information if available */ @@ -890,6 +898,8 @@ static int virtblk_probe(struct virtio_device *vdev) device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups); return 0; +out_cleanup_disk: + blk_cleanup_queue(vblk->disk->queue); out_free_tags: blk_mq_free_tag_set(&vblk->tag_set); out_put_disk: From 6ed058a9bf4fe4ef402812da49e30d7777c40a49 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Wed, 27 Apr 2022 12:17:03 +0200 Subject: [PATCH 040/156] ANDROID: abi_gki_aarch64.xml: update based on proper LTO=full setting Commit 3628acf6b8d7 ("ANDROID: GKI: Update symbols to abi_gki_aarch64_oplus") updated the .xml file, but did not do so with LTO=full so future changes to the symbol list will generate huge churn in the .xml file masking out any new symbols from the diffstat. Regenerate the .xml file in order to help clean this up. Bug: 220957464 Cc: wuzhe Fixes: 3628acf6b8d7 ("ANDROID: GKI: Update symbols to abi_gki_aarch64_oplus") Signed-off-by: Greg Kroah-Hartman Change-Id: I6784f0414df7af0e6d00d454e47c44c931d529f4 --- android/abi_gki_aarch64.xml | 11642 +++++++++++++++++----------------- 1 file changed, 5795 insertions(+), 5847 deletions(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 952ed23ff18d..5ffa2b016bcb 100644 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -7262,7 +7262,6 @@ - @@ -9038,9 +9037,9 @@ - - - + + + @@ -9061,7 +9060,6 @@ - @@ -9143,7 +9141,6 @@ - @@ -12055,6 +12052,7 @@ + @@ -12461,8 +12459,8 @@ - - + + @@ -13103,7 +13101,23 @@ - + + + + + + + + + + + + + + + + + @@ -13751,6 +13765,7 @@ + @@ -15302,12 +15317,6 @@ - - - - - - @@ -15802,6 +15811,9 @@ + + + @@ -16021,7 +16033,29 @@ - + + + + + + + + + + + + + + + + + + + + + + + @@ -18911,7 +18945,6 @@ - @@ -19302,11 +19335,7 @@ - - - - - + @@ -20261,7 +20290,6 @@ - @@ -20459,7 +20487,6 @@ - @@ -20900,14 +20927,6 @@ - - - - - - - - @@ -21991,6 +22010,7 @@ + @@ -22731,6 +22751,11 @@ + + + + + @@ -24914,7 +24939,7 @@ - + @@ -24953,7 +24978,6 @@ - @@ -25140,6 +25164,7 @@ + @@ -25444,7 +25469,23 @@ - + + + + + + + + + + + + + + + + + @@ -26107,7 +26148,7 @@ - + @@ -26263,7 +26304,7 @@ - + @@ -26484,7 +26525,6 @@ - @@ -26562,7 +26602,7 @@ - + @@ -26619,6 +26659,7 @@ + @@ -26975,6 +27016,7 @@ + @@ -27613,7 +27655,7 @@ - + @@ -28555,7 +28597,6 @@ - @@ -31419,9 +31460,6 @@ - - - @@ -31535,6 +31573,7 @@ + @@ -32429,9 +32468,9 @@ - - - + + + @@ -33319,7 +33358,6 @@ - @@ -33574,7 +33612,68 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -34089,10 +34188,9 @@ - - + @@ -34504,6 +34602,7 @@ + @@ -34702,7 +34801,7 @@ - + @@ -36339,10 +36438,6 @@ - - - - @@ -38268,7 +38363,6 @@ - @@ -39059,8 +39153,8 @@ - - + + @@ -40033,17 +40127,6 @@ - - - - - - - - - - - @@ -40324,9 +40407,9 @@ - - - + + + @@ -40833,6 +40916,7 @@ + @@ -41816,7 +41900,6 @@ - @@ -44289,13 +44372,6 @@ - - - - - - - @@ -44325,11 +44401,6 @@ - - - - - @@ -44777,12 +44848,12 @@ - - - - - - + + + + + + @@ -44887,7 +44958,6 @@ - @@ -45669,21 +45739,21 @@ - + - + - + - + - + - + @@ -47788,8 +47858,8 @@ - - + + @@ -48135,7 +48205,35 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -48317,6 +48415,7 @@ + @@ -48350,7 +48449,7 @@ - + @@ -48701,7 +48800,14 @@ - + + + + + + + + @@ -49347,11 +49453,6 @@ - - - - - @@ -49763,7 +49864,7 @@ - + @@ -49889,6 +49990,11 @@ + + + + + @@ -50406,56 +50512,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -50796,6 +50853,7 @@ + @@ -51743,7 +51801,7 @@ - + @@ -52588,7 +52646,6 @@ - @@ -53023,7 +53080,6 @@ - @@ -53074,6 +53130,7 @@ + @@ -53684,7 +53741,23 @@ - + + + + + + + + + + + + + + + + + @@ -54046,7 +54119,6 @@ - @@ -54977,9 +55049,9 @@ - - - + + + @@ -55699,6 +55771,7 @@ + @@ -56283,7 +56356,7 @@ - + @@ -56749,7 +56822,7 @@ - + @@ -57852,6 +57925,7 @@ + @@ -58988,7 +59062,6 @@ - @@ -59451,11 +59524,6 @@ - - - - - @@ -60680,44 +60748,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -61943,65 +61974,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -63420,7 +63393,6 @@ - @@ -63725,7 +63697,17 @@ - + + + + + + + + + + + @@ -64805,7 +64787,7 @@ - + @@ -65299,10 +65281,10 @@ - - - - + + + + @@ -65353,6 +65335,11 @@ + + + + + @@ -66572,11 +66559,6 @@ - - - - - @@ -67612,11 +67594,6 @@ - - - - - @@ -68078,7 +68055,7 @@ - + @@ -70141,6 +70118,7 @@ + @@ -70851,6 +70829,7 @@ + @@ -72587,9 +72566,10 @@ + - - + + @@ -72708,7 +72688,6 @@ - @@ -73033,7 +73012,6 @@ - @@ -73398,8 +73376,8 @@ - - + + @@ -73471,6 +73449,7 @@ + @@ -73938,38 +73917,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -74080,7 +74028,7 @@ - + @@ -75635,6 +75583,7 @@ + @@ -76771,7 +76720,6 @@ - @@ -78179,12 +78127,12 @@ - + - + @@ -79240,12 +79188,12 @@ - - - - - - + + + + + + @@ -80542,6 +80490,7 @@ + @@ -80781,7 +80730,6 @@ - @@ -81507,6 +81455,7 @@ + @@ -82111,6 +82060,7 @@ + @@ -82183,6 +82133,9 @@ + + + @@ -82199,11 +82152,6 @@ - - - - - @@ -82226,7 +82174,6 @@ - @@ -82687,7 +82634,7 @@ - + @@ -82763,14 +82710,6 @@ - - - - - - - - @@ -85878,6 +85817,7 @@ + @@ -86496,7 +86436,7 @@ - + @@ -87178,9 +87118,9 @@ - - - + + + @@ -87649,26 +87589,7 @@ - - - - - - - - - - - - - - - - - - - - + @@ -87693,7 +87614,7 @@ - + @@ -87718,6 +87639,7 @@ + @@ -87756,6 +87678,7 @@ + @@ -89170,7 +89093,7 @@ - + @@ -89817,8 +89740,8 @@ - - + + @@ -91054,6 +90977,14 @@ + + + + + + + + @@ -91488,7 +91419,20 @@ - + + + + + + + + + + + + + + @@ -92045,7 +91989,7 @@ - + @@ -92220,7 +92164,7 @@ - + @@ -92515,6 +92459,7 @@ + @@ -92655,12 +92600,12 @@ - + - + @@ -92752,7 +92697,6 @@ - @@ -94227,7 +94171,6 @@ - @@ -95513,7 +95456,7 @@ - + @@ -96025,7 +95968,7 @@ - + @@ -96071,7 +96014,7 @@ - + @@ -96509,7 +96452,7 @@ - + @@ -97217,9 +97160,6 @@ - - - @@ -98459,6 +98399,7 @@ + @@ -98734,8 +98675,8 @@ - - + + @@ -98945,8 +98886,8 @@ - - + + @@ -100103,6 +100044,7 @@ + @@ -100968,6 +100910,7 @@ + @@ -101080,7 +101023,14 @@ - + + + + + + + + @@ -102565,9 +102515,6 @@ - - - @@ -102580,6 +102527,9 @@ + + + @@ -102603,7 +102553,7 @@ - + @@ -102744,8 +102694,8 @@ - - + + @@ -103346,7 +103296,6 @@ - @@ -103615,6 +103564,7 @@ + @@ -104149,7 +104099,20 @@ - + + + + + + + + + + + + + + @@ -104208,6 +104171,9 @@ + + + @@ -107652,6 +107618,7 @@ + @@ -107666,20 +107633,6 @@ - - - - - - - - - - - - - - @@ -108014,6 +107967,7 @@ + @@ -111950,11 +111904,6 @@ - - - - - @@ -112281,6 +112230,7 @@ + @@ -114068,7 +114018,6 @@ - @@ -114153,7 +114102,6 @@ - @@ -114277,8 +114225,8 @@ - - + + @@ -114876,6 +114824,7 @@ + @@ -118782,9 +118731,9 @@ - - - + + + @@ -118795,33 +118744,33 @@ - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + @@ -118831,27 +118780,27 @@ - - - + + + - - - + + + - - - + + + - - - + + + @@ -118861,8 +118810,8 @@ - - + + @@ -118881,8 +118830,8 @@ - - + + @@ -118898,57 +118847,57 @@ - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + @@ -118959,16 +118908,16 @@ - - + + - - + + @@ -118992,12 +118941,12 @@ - - - - - - + + + + + + @@ -119014,9 +118963,9 @@ - - - + + + @@ -119032,29 +118981,29 @@ - - + + - - - - - + + + + + - - - + + + - - + + - - - + + + @@ -119071,15 +119020,15 @@ - - - + + + - - - + + + @@ -119092,22 +119041,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -119186,13 +119135,13 @@ - - - - - - - + + + + + + + @@ -119201,29 +119150,29 @@ - - - + + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -119265,11 +119214,11 @@ - - - - - + + + + + @@ -119289,14 +119238,14 @@ - - + + - - - - + + + + @@ -119319,15 +119268,15 @@ - - - - + + + + - - - + + + @@ -119335,11 +119284,11 @@ - - - - - + + + + + @@ -119390,21 +119339,21 @@ - - - - - + + + + + - - - - - - + + + + + + - - + + @@ -119445,18 +119394,18 @@ - - + + - - - + + + - - - + + + @@ -119489,17 +119438,17 @@ - - - - + + + + - - - - - + + + + + @@ -119510,47 +119459,47 @@ - - - - + + + + - - + + - - - + + + - - - + + + - - - + + + - - + + - - - - + + + + - - + + - - + + @@ -119560,15 +119509,15 @@ - - + + - - - - - + + + + + @@ -119581,9 +119530,9 @@ - - - + + + @@ -119592,49 +119541,49 @@ - - - - + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -119644,10 +119593,10 @@ - - - - + + + + @@ -119697,17 +119646,17 @@ - - + + - - + + - - - + + + @@ -119742,10 +119691,10 @@ - - - - + + + + @@ -119777,13 +119726,13 @@ - - - + + + - - + + @@ -119814,8 +119763,8 @@ - - + + @@ -119824,8 +119773,8 @@ - - + + @@ -119847,8 +119796,8 @@ - - + + @@ -119895,8 +119844,8 @@ - - + + @@ -119918,14 +119867,14 @@ - - + + - - - - + + + + @@ -119961,9 +119910,9 @@ - - - + + + @@ -120112,16 +120061,16 @@ - - + + - - + + @@ -120170,42 +120119,42 @@ - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - + + + - - - - - - + + + + + + - - - + + + @@ -120226,31 +120175,31 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -120268,9 +120217,9 @@ - - - + + + @@ -120318,67 +120267,67 @@ - - - - + + + + - - - - - + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - + + + + + + + + + + @@ -120394,16 +120343,16 @@ - - + + - - - - - - + + + + + + @@ -120424,22 +120373,22 @@ - - + + - - + + - - + + - - + + @@ -120463,23 +120412,23 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + @@ -120489,8 +120438,8 @@ - - + + @@ -120498,13 +120447,13 @@ - - - + + + - - + + @@ -120574,9 +120523,9 @@ - - - + + + @@ -120620,11 +120569,11 @@ - - - - - + + + + + @@ -120640,19 +120589,19 @@ - - - + + + - - - + + + - - - + + + @@ -120684,9 +120633,9 @@ - - - + + + @@ -120775,14 +120724,14 @@ - - - + + + - - - + + + @@ -120791,47 +120740,47 @@ - - - + + + - - - - + + + + - - + + - - - - - + + + + + - - - - - + + + + + - - + + - - - - + + + + - - + + @@ -120844,30 +120793,30 @@ - - + + - - + + - - + + - - - + + + - - - + + + @@ -120876,48 +120825,48 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -120928,8 +120877,8 @@ - - + + @@ -121047,8 +120996,8 @@ - - + + @@ -121156,28 +121105,28 @@ - - - + + + - - + + - - - - - + + + + + @@ -121188,13 +121137,13 @@ - - - - - - - + + + + + + + @@ -121205,11 +121154,11 @@ - - - - - + + + + + @@ -121221,11 +121170,11 @@ - - - - - + + + + + @@ -121244,8 +121193,8 @@ - - + + @@ -121282,10 +121231,10 @@ - - - - + + + + @@ -121323,22 +121272,22 @@ - - + + - - - - + + + + - - + + - - + + @@ -121349,9 +121298,9 @@ - - - + + + @@ -121370,22 +121319,22 @@ - - + + - - - - + + + + - - - + + + @@ -121446,8 +121395,8 @@ - - + + @@ -121478,18 +121427,18 @@ - - + + - - - + + + - - - + + + @@ -121501,8 +121450,8 @@ - - + + @@ -121518,9 +121467,9 @@ - - - + + + @@ -121553,12 +121502,12 @@ - - + + - - + + @@ -121569,23 +121518,23 @@ - - + + - - - + + + - - - - + + + + @@ -121600,9 +121549,9 @@ - - - + + + @@ -121621,8 +121570,8 @@ - - + + @@ -121642,58 +121591,58 @@ - - - + + + - - - + + + - - - - + + + + - - - - - + + + + + - - - - - + + + + + - + - + - - - + + + - - - - - + + + + + - - - - - + + + + + @@ -121742,12 +121691,12 @@ - - + + - - + + @@ -121776,10 +121725,10 @@ - - - - + + + + @@ -121834,42 +121783,42 @@ - - - - - - + + + + + + - - - - - - + + + + + + - - + + - - - + + + - - - + + + - + @@ -121912,10 +121861,10 @@ - - - - + + + + @@ -121923,9 +121872,9 @@ - - - + + + @@ -121935,7 +121884,7 @@ - + @@ -121956,8 +121905,8 @@ - - + + @@ -121978,22 +121927,22 @@ - - + + - - - + + + - - - + + + @@ -122043,11 +121992,11 @@ - - - - - + + + + + @@ -122055,44 +122004,44 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - - + + + + @@ -122113,8 +122062,8 @@ - - + + @@ -122139,18 +122088,18 @@ - - - + + + - - - + + + @@ -122167,9 +122116,9 @@ - - - + + + @@ -122180,11 +122129,11 @@ - - - - - + + + + + @@ -122194,11 +122143,11 @@ - - + + - + @@ -122220,17 +122169,17 @@ - - + + - - + + - - - + + + @@ -122238,22 +122187,22 @@ - - + + - - + + - - - - + + + + @@ -122261,9 +122210,9 @@ - - - + + + @@ -122294,12 +122243,12 @@ - - + + - - + + @@ -122308,7 +122257,7 @@ - + @@ -122339,11 +122288,11 @@ - - - - - + + + + + @@ -122353,25 +122302,25 @@ - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -122408,11 +122357,11 @@ - - - - - + + + + + @@ -122422,11 +122371,11 @@ - - - - - + + + + + @@ -122458,9 +122407,9 @@ - - - + + + @@ -122471,8 +122420,8 @@ - - + + @@ -122482,9 +122431,9 @@ - - - + + + @@ -122493,11 +122442,11 @@ - - - - - + + + + + @@ -122510,16 +122459,16 @@ - - + + - - + + - - + + @@ -122540,8 +122489,8 @@ - - + + @@ -122583,49 +122532,49 @@ - - + + - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -122647,9 +122596,9 @@ - - - + + + @@ -122662,9 +122611,9 @@ - - - + + + @@ -122686,10 +122635,10 @@ - - - - + + + + @@ -122720,15 +122669,15 @@ - - - - + + + + - - - + + + @@ -122748,8 +122697,8 @@ - - + + @@ -122765,8 +122714,8 @@ - - + + @@ -122774,13 +122723,13 @@ - - + + - - - + + + @@ -122791,9 +122740,9 @@ - - - + + + @@ -122810,16 +122759,16 @@ - - + + - - + + @@ -122844,9 +122793,9 @@ - - - + + + @@ -122936,9 +122885,9 @@ - - - + + + @@ -122951,17 +122900,17 @@ - - - - + + + + - - - - + + + + @@ -122969,10 +122918,10 @@ - - - - + + + + @@ -122980,9 +122929,9 @@ - - - + + + @@ -123007,11 +122956,11 @@ - - - - - + + + + + @@ -123083,20 +123032,20 @@ - - + + - - + + - - + + @@ -123109,8 +123058,8 @@ - - + + @@ -123132,12 +123081,12 @@ - - - - - - + + + + + + @@ -123146,40 +123095,40 @@ - - - + + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -123214,28 +123163,28 @@ - - - + + + - - + + - - - - + + + + - - - + + + @@ -123257,10 +123206,10 @@ - - - - + + + + @@ -123282,11 +123231,11 @@ - - - - - + + + + + @@ -123309,15 +123258,15 @@ - - - + + + - - - - + + + + @@ -123334,14 +123283,14 @@ - - - + + + - - - + + + @@ -123370,22 +123319,22 @@ - - + + - - + + - - - - + + + + @@ -123408,10 +123357,10 @@ - - - - + + + + @@ -123419,15 +123368,15 @@ - - - - + + + + - - - + + + @@ -123534,11 +123483,11 @@ - - - - - + + + + + @@ -123736,29 +123685,29 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -123768,10 +123717,10 @@ - - - - + + + + @@ -123810,14 +123759,14 @@ - - - - - - - - + + + + + + + + @@ -123830,9 +123779,9 @@ - - - + + + @@ -123865,8 +123814,8 @@ - - + + @@ -123933,14 +123882,14 @@ - - - + + + - - - + + + @@ -123948,10 +123897,10 @@ - - - - + + + + @@ -123996,15 +123945,15 @@ - - - - + + + + - - - + + + @@ -124103,16 +124052,16 @@ - - - + + + - - - - - + + + + + @@ -124168,46 +124117,46 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - + + - - - - - + + + + + - - + + - - + + @@ -124316,9 +124265,9 @@ - - - + + + @@ -124340,17 +124289,17 @@ - - + + - - - + + + - - + + @@ -124358,15 +124307,15 @@ - - - + + + - - - - + + + + @@ -124377,34 +124326,34 @@ - - + + - - - - + + + + - - + + - - - + + + - - - - + + + + @@ -124416,9 +124365,9 @@ - - - + + + @@ -124426,8 +124375,8 @@ - - + + @@ -124438,15 +124387,15 @@ - + - - - - - - + + + + + + @@ -124458,13 +124407,13 @@ - - - - + + + + - - + + @@ -124484,10 +124433,10 @@ - - - - + + + + @@ -124645,46 +124594,46 @@ - - - + + + - - - + + + - - + + - - - - - + + + + + - - + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -124817,8 +124766,8 @@ - - + + @@ -124851,16 +124800,16 @@ - - + + - - + + - - + + @@ -124872,8 +124821,8 @@ - - + + @@ -124884,8 +124833,8 @@ - - + + @@ -125055,9 +125004,9 @@ - - - + + + @@ -125067,62 +125016,62 @@ - - + + - - - + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -125140,14 +125089,14 @@ - - - + + + - - - + + + @@ -125182,13 +125131,13 @@ - - - - - - - + + + + + + + @@ -125412,62 +125361,62 @@ - - + + - - - + + + - - + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + @@ -125483,15 +125432,15 @@ - - + + - - - - - + + + + + @@ -125502,10 +125451,10 @@ - - - - + + + + @@ -125534,8 +125483,8 @@ - - + + @@ -125543,9 +125492,9 @@ - - - + + + @@ -125582,9 +125531,9 @@ - - - + + + @@ -125593,8 +125542,8 @@ - - + + @@ -125602,19 +125551,19 @@ - - - - - + + + + + - - - - - - + + + + + + @@ -125622,20 +125571,20 @@ - - + + - - + + - - + + - - + + @@ -125648,17 +125597,17 @@ - - + + - - + + - - - + + + @@ -125705,19 +125654,19 @@ - - - - - - - + + + + + + + - - - + + + @@ -125725,17 +125674,17 @@ - - - + + + - - + + @@ -125753,20 +125702,20 @@ - - + + - - + + - - + + @@ -125779,11 +125728,11 @@ - - - - - + + + + + @@ -125794,33 +125743,33 @@ - - - + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -126104,23 +126053,23 @@ - - + + - - - - + + + + - - - + + + @@ -126136,12 +126085,12 @@ - - - - - - + + + + + + @@ -126193,20 +126142,20 @@ - - + + - - - - + + + + - - - - + + + + @@ -126361,10 +126310,10 @@ - - - - + + + + @@ -126414,9 +126363,9 @@ - - - + + + @@ -126545,19 +126494,19 @@ - - - + + + - - - + + + - - - + + + @@ -126658,8 +126607,8 @@ - - + + @@ -126681,9 +126630,9 @@ - - - + + + @@ -126711,8 +126660,8 @@ - - + + @@ -126727,24 +126676,24 @@ - - + + - - - + + + - - - - + + + + - - - + + + @@ -126763,9 +126712,9 @@ - - - + + + @@ -126779,18 +126728,18 @@ - - + + - - - + + + - - - + + + @@ -126811,9 +126760,9 @@ - - - + + + @@ -126844,28 +126793,28 @@ - - + + - - + + - - - + + + - - - - + + + + - - - + + + @@ -126876,83 +126825,83 @@ - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - - + + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + @@ -127008,17 +126957,17 @@ - - + + - - + + - - - + + + @@ -127032,20 +126981,20 @@ - - + + - - + + - - + + @@ -127114,9 +127063,9 @@ - - - + + + @@ -127124,44 +127073,44 @@ - - + + - - - - - + + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + @@ -127171,12 +127120,12 @@ - - - - - - + + + + + + @@ -127187,9 +127136,9 @@ - - - + + + @@ -127197,18 +127146,18 @@ - - - - + + + + - - - - - - - + + + + + + + @@ -127223,30 +127172,30 @@ - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + @@ -127282,8 +127231,8 @@ - - + + @@ -127359,20 +127308,20 @@ - - + + - - - - - - - - - - + + + + + + + + + + @@ -127381,19 +127330,19 @@ - - - + + + - - - - + + + + - - - + + + @@ -127420,40 +127369,40 @@ - - - - + + + + - - + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -127692,8 +127641,8 @@ - - + + @@ -127707,13 +127656,13 @@ - - + + - - - + + + @@ -127730,10 +127679,10 @@ - - - - + + + + @@ -127766,14 +127715,14 @@ - - - + + + - - - + + + @@ -127781,17 +127730,17 @@ - - - - + + + + - - + + @@ -127806,9 +127755,9 @@ - - - + + + @@ -127822,54 +127771,54 @@ - - - - + + + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + - - + + - - + + - - - + + + - - + + @@ -127879,15 +127828,15 @@ - - - + + + - - - - + + + + @@ -127920,18 +127869,18 @@ - - - - + + + + - - - - - + + + + + @@ -127939,13 +127888,13 @@ - - - + + + - - + + @@ -127975,12 +127924,12 @@ - - + + - - + + @@ -128026,15 +127975,15 @@ - - - + + + - - - - + + + + @@ -128043,13 +127992,13 @@ - - - + + + - - + + @@ -128059,21 +128008,21 @@ - - - + + + - - - - + + + + - - - - + + + + @@ -128089,26 +128038,26 @@ - - + + - - - + + + - - + + - - - + + + @@ -128125,11 +128074,11 @@ - - + + - + @@ -128144,8 +128093,8 @@ - - + + @@ -128193,18 +128142,18 @@ - - - + + + - - + + - - - + + + @@ -128212,13 +128161,13 @@ - - + + - - - + + + @@ -128234,9 +128183,9 @@ - - - + + + @@ -128330,9 +128279,9 @@ - - - + + + @@ -128381,11 +128330,11 @@ - - - - - + + + + + @@ -128405,11 +128354,11 @@ - - - - - + + + + + @@ -128521,8 +128470,8 @@ - - + + @@ -128531,9 +128480,9 @@ - - - + + + @@ -128546,8 +128495,8 @@ - - + + @@ -128590,12 +128539,12 @@ - - + + - - + + @@ -128632,35 +128581,35 @@ - - + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - + + + @@ -128672,13 +128621,13 @@ - - - + + + - - + + @@ -128721,44 +128670,44 @@ - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - + + + - - - - + + + + - - + + - - + + @@ -128793,13 +128742,13 @@ - - - - - - - + + + + + + + @@ -128824,18 +128773,18 @@ - - - + + + - - + + - - - + + + @@ -128866,10 +128815,10 @@ - - - - + + + + @@ -128879,12 +128828,12 @@ - - + + - - + + @@ -128895,17 +128844,17 @@ - - - - - - - + + + + + + + - - + + @@ -128922,8 +128871,8 @@ - - + + @@ -128953,15 +128902,15 @@ - - + + - - - - - + + + + + @@ -128973,8 +128922,8 @@ - - + + @@ -129000,8 +128949,8 @@ - - + + @@ -129011,20 +128960,20 @@ - - + + - + - - - + + + @@ -129038,41 +128987,41 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -129082,13 +129031,13 @@ - - + + - - - + + + @@ -129102,41 +129051,41 @@ - - - - - - + + + + + + - - + + - - - - - - + + + + + + - - - - - + + + + + - - - - - - - - + + + + + + + + @@ -129150,8 +129099,8 @@ - - + + @@ -129163,18 +129112,18 @@ - - + + - - - - + + + + @@ -129198,23 +129147,23 @@ - - - + + + - - - + + + - - - + + + - - + + @@ -129227,9 +129176,9 @@ - - - + + + @@ -129237,29 +129186,29 @@ - - - - + + + + - - + + - - - - - + + + + + - - - - - - + + + + + + @@ -129292,9 +129241,9 @@ - - - + + + @@ -129303,13 +129252,13 @@ - - + + - - - + + + @@ -129406,8 +129355,8 @@ - - + + @@ -129438,13 +129387,13 @@ - - - + + + - - + + @@ -129489,10 +129438,10 @@ - - + + - + @@ -129507,22 +129456,22 @@ - - - + + + - - - + + + - - + + - - + + @@ -129573,38 +129522,38 @@ - - - - + + + + - - - - - - - - + + + + + + + + - - + + - - + + - + - - + + @@ -129615,9 +129564,9 @@ - - - + + + @@ -129632,26 +129581,26 @@ - - + + - - - - + + + + - - - - - - + + + + + + - - + + @@ -129666,28 +129615,28 @@ - - + + - - - + + + - - + + - - - - - + + + + + - - - - + + + + @@ -129702,28 +129651,28 @@ - - - - - + + + + + - - + + - - - - - + + + + + - - - - + + + + @@ -129745,18 +129694,18 @@ - - + + - - + + - - + + @@ -129779,12 +129728,12 @@ - - + + - - + + @@ -129821,9 +129770,9 @@ - - - + + + @@ -129858,81 +129807,81 @@ - - - + + + - - - + + + - - - - + + + + - - - - - - + + + + + + - - - - + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -130062,12 +130011,12 @@ - - + + - - + + @@ -130110,14 +130059,14 @@ - - + + - - - - + + + + @@ -130138,36 +130087,36 @@ - - - - - + + + + + - - + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -130178,39 +130127,39 @@ - - + + - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - - - + + + + @@ -130236,14 +130185,14 @@ - - + + - - - - + + + + @@ -130264,12 +130213,12 @@ - - + + - - + + @@ -130285,8 +130234,8 @@ - - + + @@ -130413,21 +130362,21 @@ - - - - - + + + + + - - - - - - - - + + + + + + + + @@ -130437,8 +130386,8 @@ - - + + @@ -130461,8 +130410,8 @@ - - + + @@ -130470,9 +130419,9 @@ - - - + + + @@ -130492,24 +130441,24 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -130517,50 +130466,50 @@ - - - - - - + + + + + + - - - + + + - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - - + + + @@ -130576,8 +130525,8 @@ - - + + @@ -130598,8 +130547,8 @@ - - + + @@ -130614,8 +130563,8 @@ - - + + @@ -130648,20 +130597,20 @@ - - + + - - + + - - + + @@ -130698,12 +130647,12 @@ - - + + - - + + @@ -130715,34 +130664,34 @@ - - + + - - - + + + - - - - + + + + - - - - + + + + - - - + + + - - + + @@ -130759,56 +130708,56 @@ - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -130827,9 +130776,9 @@ - - - + + + @@ -130851,42 +130800,42 @@ - - + + - - + + - - - - + + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -130894,170 +130843,170 @@ - + - - - - + + + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + - - + + - - - - + + + + - - - + + + - - - - + + + + - - - - + + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - - - - + + + + + + - - - - - - + + + + + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - + + + - - - - + + + + - - - - + + + + @@ -131065,53 +131014,53 @@ - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - - - - + + + + + + - - - - - + + + + + @@ -131146,8 +131095,8 @@ - - + + @@ -131247,13 +131196,13 @@ - - - + + + - - + + @@ -131264,15 +131213,15 @@ - - - - - - - - - + + + + + + + + + @@ -131308,9 +131257,9 @@ - - - + + + @@ -131336,30 +131285,30 @@ - - - + + + - - - - + + + + - - - - + + + + - - - + + + @@ -131372,10 +131321,10 @@ - - - - + + + + @@ -131383,11 +131332,11 @@ - - - - - + + + + + @@ -131410,10 +131359,10 @@ - - - - + + + + @@ -131426,20 +131375,20 @@ - - - + + + - - - - + + + + - + - - + + @@ -131453,9 +131402,9 @@ - - - + + + @@ -131557,14 +131506,14 @@ - - - + + + - - + + @@ -131651,8 +131600,8 @@ - - + + @@ -131662,56 +131611,56 @@ - - + + - - - - - - - - - + + + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - + + - - + + - - + + - - + + - + @@ -131722,10 +131671,10 @@ - - - - + + + + @@ -131751,21 +131700,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -131776,35 +131725,35 @@ - - - - - - - - + + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -131836,46 +131785,46 @@ - - - - + + + + - - - + + + - - + + - - + + - - - + + + - - - - - + + + + + - - + + - - - + + + @@ -131884,35 +131833,35 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + - - + + @@ -131921,23 +131870,23 @@ - - - + + + - - - + + + - - - + + + @@ -131945,34 +131894,34 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - + + + + - - - - - + + + + + @@ -131982,16 +131931,16 @@ - - - - + + + + - - - - + + + + @@ -132007,10 +131956,10 @@ - - - - + + + + @@ -132020,10 +131969,10 @@ - - - - + + + + @@ -132033,37 +131982,37 @@ - - - - + + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -132074,9 +132023,9 @@ - - - + + + @@ -132090,11 +132039,11 @@ - - - - - + + + + + @@ -132130,11 +132079,11 @@ - - + + - + @@ -132151,15 +132100,15 @@ - - + + - - + + - - + + @@ -132179,8 +132128,8 @@ - - + + @@ -132190,8 +132139,8 @@ - - + + @@ -132203,8 +132152,8 @@ - - + + @@ -132218,18 +132167,18 @@ - - - + + + - - - - + + + + - - + + @@ -132237,10 +132186,10 @@ - - - - + + + + @@ -132253,14 +132202,14 @@ - - - - + + + + - - + + @@ -132296,18 +132245,18 @@ - - - + + + - - - + + + @@ -132318,8 +132267,8 @@ - - + + @@ -132330,10 +132279,10 @@ - - - - + + + + @@ -132343,30 +132292,30 @@ - - - - - + + + + + - - + + - - + + - - - + + + - - + + @@ -132385,11 +132334,11 @@ - - - - - + + + + + @@ -132397,13 +132346,13 @@ - - + + - - + + @@ -132420,11 +132369,11 @@ - - - - - + + + + + @@ -132463,8 +132412,8 @@ - - + + @@ -132472,25 +132421,25 @@ - - - + + + - - + + - - - - + + + + - - - - + + + + @@ -132503,9 +132452,9 @@ - - - + + + @@ -132516,13 +132465,13 @@ - - + + - - - + + + @@ -132530,50 +132479,50 @@ - - - + + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -132587,8 +132536,8 @@ - - + + @@ -132664,9 +132613,9 @@ - - - + + + @@ -132704,32 +132653,32 @@ - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - + @@ -132742,9 +132691,9 @@ - - - + + + @@ -132824,10 +132773,10 @@ - - - - + + + + @@ -132872,13 +132821,13 @@ - - - - - - - + + + + + + + @@ -133065,12 +133014,12 @@ - - + + - - + + @@ -133082,11 +133031,11 @@ - - - - - + + + + + @@ -133127,8 +133076,8 @@ - - + + @@ -133181,8 +133130,8 @@ - - + + @@ -133194,9 +133143,9 @@ - - - + + + @@ -133207,28 +133156,28 @@ - - - + + + - - - - - - + + + + + + - - + + - - - - - + + + + + @@ -133239,9 +133188,9 @@ - - - + + + @@ -133302,8 +133251,8 @@ - - + + @@ -133342,9 +133291,9 @@ - - - + + + @@ -133400,8 +133349,8 @@ - - + + @@ -133423,12 +133372,12 @@ - - + + - - + + @@ -133462,8 +133411,8 @@ - - + + @@ -133474,9 +133423,9 @@ - - - + + + @@ -133484,47 +133433,47 @@ - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - - - - + + + + + @@ -133532,18 +133481,18 @@ - - - - + + + + - - + + @@ -133551,14 +133500,14 @@ - - - - + + + + - - - + + + @@ -133594,17 +133543,17 @@ - - + + - - - + + + @@ -133614,9 +133563,9 @@ - - - + + + @@ -133658,12 +133607,12 @@ - - + + - - + + @@ -133677,67 +133626,67 @@ - - - + + + - - + + - - + + - - + + - - + + - - - - - + + + + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + @@ -133745,19 +133694,19 @@ - - + + - - + + - - - - - + + + + + @@ -133768,41 +133717,41 @@ - - - - + + + + - - - + + + - - + + - - - - - + + + + + - - + + - - + + @@ -133816,16 +133765,16 @@ - - - - + + + + - - - - + + + + @@ -133846,11 +133795,11 @@ - - - - - + + + + + @@ -133867,24 +133816,24 @@ - - - - + + + + - - - - - + + + + + - - - - - + + + + + @@ -133909,10 +133858,10 @@ - - - - + + + + @@ -133925,12 +133874,12 @@ - - + + - - + + @@ -133955,18 +133904,18 @@ - - - + + + - - + + - - - + + + @@ -133990,18 +133939,18 @@ - - - - - + + + + + - - - - - + + + + + @@ -134012,19 +133961,19 @@ - - - - + + + + - - + + - - - + + + @@ -134043,14 +133992,14 @@ - - - + + + - - - + + + @@ -134076,9 +134025,9 @@ - - - + + + @@ -134091,22 +134040,22 @@ - - - - + + + + - - + + - - + + @@ -134120,43 +134069,43 @@ - - + + - - - + + + - - - + + + - - - - - + + + + + - - - + + + - - - - - + + + + + @@ -134169,9 +134118,9 @@ - - - + + + @@ -134204,14 +134153,14 @@ - - - - + + + + - - + + @@ -134222,44 +134171,44 @@ - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -134293,16 +134242,16 @@ - - - - - - + + + + + + - - - + + + @@ -134310,9 +134259,9 @@ - - - + + + @@ -134356,33 +134305,33 @@ - - - + + + - - - + + + - - + + - - + + - - - + + + - - - - + + + + @@ -134407,9 +134356,9 @@ - - - + + + @@ -134425,32 +134374,32 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + - - - + + + @@ -134459,9 +134408,9 @@ - - - + + + @@ -134469,74 +134418,74 @@ - - - + + + - - + + - - - + + + - - - + + + - - - - + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + - - + + - - + + - - - - + + + + @@ -134548,42 +134497,42 @@ - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - + + @@ -134598,8 +134547,8 @@ - - + + @@ -134619,8 +134568,8 @@ - - + + @@ -134634,61 +134583,61 @@ - - - - - + + + + + - - - + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - + + + + @@ -134706,12 +134655,12 @@ - - - - - - + + + + + + @@ -134722,12 +134671,12 @@ - - - - - - + + + + + + @@ -134742,9 +134691,9 @@ - - - + + + @@ -134757,15 +134706,15 @@ - - + + - - - - - + + + + + @@ -134782,10 +134731,10 @@ - - - - + + + + @@ -134805,10 +134754,10 @@ - - - - + + + + @@ -134854,10 +134803,10 @@ - - - - + + + + @@ -134872,16 +134821,16 @@ - - - + + + - - + + - - + + @@ -134904,11 +134853,11 @@ - - - - - + + + + + @@ -134938,16 +134887,16 @@ - - - - - - + + + + + + - - + + @@ -135050,33 +134999,33 @@ - - + + - - + + - - - - - - + + + + + + - - - + + + - - + + @@ -135102,12 +135051,12 @@ - - + + - - + + @@ -135118,16 +135067,16 @@ - - + + - - + + - - + + @@ -135212,19 +135161,19 @@ - - - + + + - - - + + + - - - + + + @@ -135239,8 +135188,8 @@ - - + + @@ -135271,9 +135220,9 @@ - - - + + + @@ -135303,14 +135252,14 @@ - - - + + + - - - + + + @@ -135323,9 +135272,9 @@ - - - + + + @@ -135333,10 +135282,10 @@ - - - - + + + + @@ -135350,12 +135299,12 @@ - - + + - - + + @@ -135376,9 +135325,9 @@ - - - + + + @@ -135390,9 +135339,9 @@ - - - + + + @@ -135440,8 +135389,8 @@ - - + + @@ -135487,9 +135436,9 @@ - - - + + + @@ -135505,10 +135454,10 @@ - - - - + + + + @@ -135529,10 +135478,10 @@ - - - - + + + + @@ -135565,17 +135514,17 @@ - - + + - - - + + + @@ -135589,55 +135538,55 @@ - - - - + + + + - - + + - - - + + + - - + + - - + + - - - - - + + + + + - - - + + + - - - + + + - - + + - - + + - - + + @@ -135659,20 +135608,20 @@ - - - - - - + + + + + + - - + + - - + + @@ -135680,21 +135629,21 @@ - - - - - + + + + + - - - - - + + + + + - - + + @@ -135705,7 +135654,7 @@ - + @@ -135724,21 +135673,21 @@ - - - - + + + + - - - - - - - - - + + + + + + + + + @@ -135772,8 +135721,8 @@ - - + + @@ -135793,9 +135742,9 @@ - - - + + + @@ -135803,23 +135752,23 @@ - - - + + + - - - + + + - - - + + + @@ -135840,37 +135789,37 @@ - - + + - - - - + + + + - - - + + + - - + + - - - - + + + + @@ -135884,11 +135833,11 @@ - - - - - + + + + + @@ -135898,12 +135847,12 @@ - - - - - - + + + + + + @@ -135947,10 +135896,10 @@ - - - - + + + + @@ -136009,24 +135958,24 @@ - - + + - - + + - - + + @@ -136036,11 +135985,11 @@ - - - - - + + + + + @@ -136085,14 +136034,14 @@ - - - - + + + + - - - + + + @@ -136325,8 +136274,8 @@ - - + + @@ -136346,29 +136295,29 @@ - - - + + + - - + + - - + + - - + + - - + + - - + + @@ -136380,14 +136329,14 @@ - - - + + + - - - + + + @@ -136400,16 +136349,16 @@ - - - - + + + + - - - - + + + + @@ -136461,12 +136410,12 @@ - - + + - - + + @@ -136513,24 +136462,24 @@ - - + + - - + + - - + + - - + + @@ -136550,13 +136499,13 @@ - - + + - - - + + + @@ -136570,16 +136519,16 @@ - - - - + + + + - - - - + + + + @@ -136592,9 +136541,9 @@ - - - + + + @@ -136615,8 +136564,8 @@ - - + + @@ -136631,8 +136580,8 @@ - - + + @@ -136666,8 +136615,8 @@ - - + + @@ -136685,10 +136634,10 @@ - - - - + + + + @@ -136701,54 +136650,54 @@ - - + + - - - + + + - - + + - + - + - - - - + + + + - - - - + + + + - - - - - - - - - + + + + + + + + + - - + + - - + + @@ -136757,36 +136706,36 @@ - - - - - + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + @@ -136797,12 +136746,12 @@ - - - - - - + + + + + + @@ -136821,9 +136770,9 @@ - - - + + + @@ -136833,8 +136782,8 @@ - - + + @@ -136848,10 +136797,10 @@ - - - - + + + + @@ -136864,13 +136813,13 @@ - - - + + + - - + + @@ -136923,11 +136872,11 @@ - - - - - + + + + + @@ -136943,24 +136892,24 @@ - - + + - - + + - - + + - - + + @@ -136968,24 +136917,24 @@ - - + + - - + + - - + + @@ -137045,8 +136994,8 @@ - - + + @@ -137056,28 +137005,28 @@ - - - - - + + + + + - - - - - + + + + + - - - - + + + + @@ -137091,23 +137040,23 @@ - - - + + + - - + + - - - - + + + + - - + + @@ -137130,48 +137079,48 @@ - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + @@ -137180,7 +137129,7 @@ - + @@ -137220,17 +137169,17 @@ - + - + - - + + @@ -137249,8 +137198,8 @@ - - + + @@ -137287,8 +137236,8 @@ - - + + @@ -137300,23 +137249,23 @@ - - - - + + + + - - - + + + - - - + + + @@ -137376,8 +137325,8 @@ - - + + @@ -137418,16 +137367,16 @@ - - + + - - + + - - + + @@ -137438,8 +137387,8 @@ - - + + @@ -137462,12 +137411,12 @@ - - + + - - + + @@ -137483,8 +137432,8 @@ - - + + @@ -137603,10 +137552,10 @@ - - - - + + + + @@ -137615,20 +137564,20 @@ - - - - - - - - + + + + + + + + - - - - + + + + @@ -137636,14 +137585,14 @@ - - - + + + - - - + + + @@ -137661,8 +137610,8 @@ - - + + @@ -137674,8 +137623,8 @@ - - + + @@ -137712,8 +137661,8 @@ - - + + @@ -137764,22 +137713,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -137869,8 +137818,8 @@ - - + + @@ -137878,22 +137827,22 @@ - - + + - - - - - - + + + + + + - - - - + + + + @@ -137906,9 +137855,9 @@ - - - + + + @@ -137916,16 +137865,16 @@ - - - + + + - - - - - + + + + + @@ -137966,13 +137915,13 @@ - - - - - - - + + + + + + + @@ -137980,28 +137929,28 @@ - - + + - - + + - - + + - - + + - - + + @@ -138017,20 +137966,20 @@ - - - - - - + + + + + + - - - + + + - - + + @@ -138046,8 +137995,8 @@ - - + + @@ -138064,8 +138013,8 @@ - - + + @@ -138078,15 +138027,15 @@ - - - + + + - - - - + + + + @@ -138094,39 +138043,39 @@ - - - + + + - - - - + + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + @@ -138134,30 +138083,30 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -138284,8 +138233,8 @@ - - + + @@ -138409,12 +138358,12 @@ - - + + - - + + @@ -138431,9 +138380,9 @@ - - - + + + @@ -138441,9 +138390,9 @@ - - - + + + @@ -138451,9 +138400,9 @@ - - - + + + @@ -138478,12 +138427,12 @@ - - + + - - + + @@ -138501,10 +138450,10 @@ - - - - + + + + @@ -138526,67 +138475,67 @@ - - - + + + - - - - + + + + - - - + + + - - + + - - - - - - + + + + + + - - - - + + + + - - - - - - - + + + + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -138597,13 +138546,13 @@ - + - - - - + + + + @@ -138637,10 +138586,10 @@ - - - - + + + + @@ -138682,7 +138631,7 @@ - + @@ -138690,34 +138639,34 @@ - - + + - - + + - - + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -138728,10 +138677,10 @@ - - - - + + + + @@ -138741,8 +138690,8 @@ - - + + @@ -138753,8 +138702,8 @@ - - + + @@ -138792,8 +138741,8 @@ - - + + @@ -138831,14 +138780,14 @@ - - - - - + + + + + - - + + @@ -138876,11 +138825,11 @@ - - - - - + + + + + @@ -139024,21 +138973,21 @@ - - + + - - - + + + - - + + - - + + @@ -139072,11 +139021,11 @@ - - - - - + + + + + @@ -139091,21 +139040,21 @@ - - + + - - + + - - + + - - - + + + @@ -139121,11 +139070,11 @@ - - - - - + + + + + @@ -139149,12 +139098,12 @@ - - - - - - + + + + + + @@ -139174,28 +139123,28 @@ - - - - + + + + - - - - + + + + - - - + + + - - - - + + + + @@ -139215,26 +139164,26 @@ - - - - + + + + - - - + + + - - - - - + + + + + - - - + + + @@ -139243,20 +139192,20 @@ - - - + + + - - - + + + - - - + + + @@ -139266,26 +139215,26 @@ - - - + + + - - - + + + - - - - + + + + - - - - + + + + @@ -139431,9 +139380,9 @@ - - - + + + @@ -139448,9 +139397,9 @@ - - - + + + @@ -139458,26 +139407,26 @@ - - + + - - - - + + + + - - + + - - + + @@ -139485,25 +139434,25 @@ - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -139515,33 +139464,33 @@ - - - - - - + + + + + + - - - - - - + + + + + + - - - - + + + + - - - + + + @@ -139549,28 +139498,28 @@ - - - + + + - - - - - + + + + + - - + + - - - + + + - - + + @@ -139581,13 +139530,13 @@ - - - - - - - + + + + + + + @@ -139619,14 +139568,14 @@ - - + + - - - - + + + + @@ -139634,10 +139583,10 @@ - - - - + + + + @@ -139645,10 +139594,10 @@ - - - - + + + + @@ -139671,10 +139620,10 @@ - - - - + + + + @@ -139690,8 +139639,8 @@ - - + + @@ -139701,29 +139650,29 @@ - + - - + + - - + + - - - - + + + + - - - - - - + + + + + + @@ -139752,13 +139701,13 @@ - - - - - - - + + + + + + + @@ -139777,10 +139726,10 @@ - - - - + + + + @@ -139797,10 +139746,10 @@ - - - - + + + + @@ -139810,30 +139759,30 @@ - - - + + + - - - - - - + + + + + + - - + + - - - - - - - + + + + + + + @@ -139850,13 +139799,13 @@ - - + + - - - + + + @@ -139875,39 +139824,39 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + - - + + @@ -139930,55 +139879,55 @@ - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -139988,9 +139937,9 @@ - - - + + + @@ -140000,16 +139949,16 @@ - - - - - + + + + + - - - + + + @@ -140044,15 +139993,15 @@ - - - + + + - - - - + + + + @@ -140065,11 +140014,11 @@ - - - - - + + + + + @@ -140079,16 +140028,16 @@ - - - - - + + + + + - - - + + + @@ -140098,8 +140047,8 @@ - - + + @@ -140111,25 +140060,25 @@ - - + + - - + + - - - - - - - + + + + + + + - - + + @@ -140140,9 +140089,9 @@ - - - + + + @@ -140194,9 +140143,9 @@ - - - + + + @@ -140230,22 +140179,22 @@ - - - - - + + + + + - - - - - + + + + + - - + + @@ -140324,8 +140273,8 @@ - - + + @@ -140335,9 +140284,9 @@ - - - + + + @@ -140389,8 +140338,8 @@ - - + + @@ -140406,9 +140355,9 @@ - - - + + + @@ -140418,17 +140367,17 @@ - - - - + + + + - - - - - + + + + + @@ -140452,17 +140401,17 @@ - - + + - - - - - - - + + + + + + + @@ -140471,23 +140420,23 @@ - - + + - - - - + + + + - - - + + + - - + + @@ -140498,12 +140447,12 @@ - - - - - - + + + + + + @@ -140515,8 +140464,8 @@ - - + + @@ -140528,18 +140477,18 @@ - - - - - - + + + + + + - - - - + + + + @@ -140547,12 +140496,12 @@ - - + + - - + + @@ -140590,9 +140539,9 @@ - - - + + + @@ -140604,8 +140553,8 @@ - - + + @@ -140637,45 +140586,45 @@ - - - + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - - + + + + + @@ -140720,9 +140669,9 @@ - - - + + + @@ -140753,10 +140702,10 @@ - - - - + + + + @@ -140848,14 +140797,14 @@ - - - - + + + + - - + + @@ -140873,8 +140822,8 @@ - - + + @@ -141125,9 +141074,9 @@ - - - + + + @@ -141216,9 +141165,9 @@ - - - + + + @@ -141239,13 +141188,13 @@ - - - - - - - + + + + + + + @@ -141257,44 +141206,44 @@ - - - + + + - - - - + + + + - - + + - - + + - - - - - - + + + + + + - - + + - - - + + + @@ -141404,9 +141353,9 @@ - - - + + + @@ -141418,17 +141367,17 @@ - - - - - - + + + + + + - - - + + + @@ -141552,9 +141501,9 @@ - - - + + + @@ -141587,34 +141536,34 @@ - - + + - - + + - - + + - - + + - - - - + + + + - - - - - - + + + + + + @@ -141637,9 +141586,9 @@ - - - + + + @@ -141657,8 +141606,8 @@ - - + + @@ -141693,10 +141642,10 @@ - - - - + + + + @@ -141705,21 +141654,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -141727,21 +141676,21 @@ - - - - + + + + - - - - - + + + + + - - - + + + @@ -141760,16 +141709,16 @@ - - - + + + - - - + + + - - + + @@ -141787,33 +141736,33 @@ - - + + - - + + - - + + - - + + - - + + - - - + + + @@ -141824,10 +141773,10 @@ - + - + @@ -141839,12 +141788,12 @@ - - + + - - + + @@ -141859,23 +141808,23 @@ - - + + - - + + - - - + + + - + - + @@ -141894,15 +141843,15 @@ - - - + + + - - - - + + + + @@ -141915,27 +141864,27 @@ - - - + + + - - - - + + + + - - - + + + - - - - + + + + @@ -141957,9 +141906,9 @@ - - - + + + @@ -141968,10 +141917,10 @@ - - - - + + + + @@ -141979,9 +141928,9 @@ - - - + + + @@ -141989,9 +141938,9 @@ - - - + + + @@ -142000,17 +141949,17 @@ - - - - + + + + - + @@ -142028,8 +141977,8 @@ - - + + @@ -142072,8 +142021,8 @@ - - + + @@ -142166,16 +142115,16 @@ - - + + - - + + @@ -142184,8 +142133,8 @@ - - + + @@ -142217,19 +142166,19 @@ - - - - - - - - - + + + + + + + + + - - + + @@ -142241,9 +142190,9 @@ - - - + + + @@ -142262,14 +142211,14 @@ - - - - + + + + - - + + @@ -142297,9 +142246,9 @@ - - - + + + @@ -142322,12 +142271,11 @@ - - - + + @@ -142362,19 +142310,19 @@ - - + + - - - - + + + + - - - + + + @@ -142392,8 +142340,8 @@ - - + + @@ -142403,24 +142351,24 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -142430,26 +142378,26 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + @@ -142458,10 +142406,10 @@ - - - - + + + + @@ -142478,7 +142426,7 @@ - + @@ -142512,9 +142460,9 @@ - - - + + + @@ -142652,10 +142600,10 @@ - - - - + + + + @@ -142663,10 +142611,10 @@ - - - - + + + + @@ -142745,19 +142693,19 @@ - - - + + + - - - + + + - - - + + + @@ -142787,8 +142735,8 @@ - - + + @@ -142806,16 +142754,16 @@ - - + + - - + + @@ -142868,9 +142816,9 @@ - - - + + + @@ -142884,9 +142832,9 @@ - - - + + + @@ -142925,8 +142873,8 @@ - - + + @@ -142972,9 +142920,9 @@ - - - + + + @@ -143065,11 +143013,11 @@ - - - - - + + + + + @@ -143304,9 +143252,9 @@ - - - + + + @@ -143424,27 +143372,27 @@ - - + + - - + + - - + + - - - - - + + + + + @@ -143478,8 +143426,8 @@ - - + + @@ -143518,17 +143466,17 @@ - - + + - - + + - - - + + + @@ -143539,8 +143487,8 @@ - - + + @@ -143571,12 +143519,12 @@ - - + + - - + + @@ -143599,8 +143547,8 @@ - - + + @@ -143622,8 +143570,8 @@ - - + + @@ -143659,9 +143607,9 @@ - - - + + + @@ -143684,9 +143632,9 @@ - - - + + + @@ -143701,16 +143649,16 @@ - - + + - - + + - - + + @@ -143729,8 +143677,8 @@ - - + + @@ -143745,16 +143693,16 @@ - - - - - - - - - - + + + + + + + + + + @@ -143788,8 +143736,8 @@ - - + + @@ -143804,15 +143752,15 @@ - - - - + + + + - - - + + + @@ -143823,9 +143771,9 @@ - - - + + + @@ -143837,9 +143785,9 @@ - - - + + + @@ -143858,24 +143806,24 @@ - - - + + + - - - - + + + + - - - + + + @@ -143902,8 +143850,8 @@ - - + + @@ -143922,9 +143870,9 @@ - - - + + + @@ -143933,28 +143881,28 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + @@ -143963,14 +143911,14 @@ - - - - + + + + - - + + @@ -143981,9 +143929,9 @@ - - - + + + @@ -143998,8 +143946,8 @@ - - + + @@ -144021,8 +143969,8 @@ - - + + @@ -144125,15 +144073,15 @@ - - - + + + - - - - + + + + @@ -144145,8 +144093,8 @@ - - + + @@ -144212,16 +144160,16 @@ - - + + - - + + @@ -144229,10 +144177,10 @@ - - - - + + + + @@ -144256,38 +144204,38 @@ - - + + - - + + - - - + + + - - - + + + - - + + - - + + @@ -144295,10 +144243,10 @@ - - - - + + + + @@ -144322,9 +144270,9 @@ - - - + + + @@ -144332,8 +144280,8 @@ - - + + @@ -144382,9 +144330,9 @@ - - - + + + @@ -144405,10 +144353,10 @@ - - - - + + + + @@ -144458,9 +144406,9 @@ - - - + + + @@ -144520,50 +144468,50 @@ - - - + + + - - - - - - + + + + + + - - - - - + + + + + - - + + - - - - + + + + - - - - - - + + + + + + - - + + @@ -144739,9 +144687,9 @@ - - - + + + @@ -144755,27 +144703,27 @@ - - + + - - - - + + + + - - - + + + @@ -144789,33 +144737,33 @@ - - - - - + + + + + - - - + + + - - + + - - + + - - + + - - - + + + @@ -144846,15 +144794,15 @@ - - - - + + + + - - - + + + @@ -144866,30 +144814,30 @@ - - - - + + + + - - + + - - - - + + + + - - - + + + - - - + + + @@ -144901,62 +144849,62 @@ - - + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + @@ -144964,42 +144912,42 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - + + @@ -145008,24 +144956,24 @@ - - + + - - - - + + + + - - - - + + + + @@ -145036,8 +144984,8 @@ - - + + @@ -145061,15 +145009,15 @@ - - - + + + - - - - + + + + @@ -145120,15 +145068,15 @@ - - - + + + - - - + + + @@ -145141,15 +145089,15 @@ - - - - + + + + - - - + + + @@ -145227,9 +145175,9 @@ - - - + + + @@ -145245,36 +145193,36 @@ - - - + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + @@ -145289,9 +145237,9 @@ - - - + + + @@ -145306,14 +145254,14 @@ - - - + + + - - - + + + @@ -145355,15 +145303,15 @@ - - + + - - - - - + + + + + @@ -145371,11 +145319,11 @@ - - - - - + + + + + @@ -145403,14 +145351,14 @@ - - - - - + + + + + - - + + @@ -145430,8 +145378,8 @@ - - + + @@ -145474,8 +145422,8 @@ - - + + @@ -145511,8 +145459,8 @@ - - + + @@ -145579,9 +145527,9 @@ - - - + + + @@ -145652,26 +145600,26 @@ - - - - - - - + + + + + + + - - + + - - - + + + @@ -145708,21 +145656,21 @@ - - - + + + - - - + + + - - - + + + @@ -145789,15 +145737,15 @@ - - + + - - + + @@ -145808,16 +145756,16 @@ - - - - - + + + + + - - - + + + @@ -145826,11 +145774,11 @@ - - - - - + + + + + @@ -145839,9 +145787,9 @@ - - - + + + @@ -145891,11 +145839,11 @@ - - - - - + + + + + @@ -145905,23 +145853,23 @@ - - - - + + + + - - - + + + - - + + - - + + @@ -145940,12 +145888,12 @@ - - + + - - + + @@ -145953,8 +145901,8 @@ - - + + @@ -145975,8 +145923,8 @@ - - + + @@ -145984,16 +145932,16 @@ - - - + + + - + - - + + @@ -146001,8 +145949,8 @@ - - + + @@ -146018,9 +145966,9 @@ - - - + + + @@ -146043,17 +145991,17 @@ - - - + + + - - + + @@ -146070,12 +146018,12 @@ - - - - - - + + + + + + @@ -146136,9 +146084,9 @@ - - - + + + @@ -146163,34 +146111,34 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + @@ -146200,17 +146148,17 @@ - - + + - - - - + + + + @@ -146221,26 +146169,26 @@ - - - - + + + + - - - - + + + + - - + + - - - - + + + + @@ -146251,9 +146199,9 @@ - - - + + + @@ -146285,10 +146233,10 @@ - - - - + + + + @@ -146321,9 +146269,9 @@ - - - + + + @@ -146340,15 +146288,15 @@ - - - - + + + + - - - + + + @@ -146372,12 +146320,12 @@ - - - - - - + + + + + + @@ -146399,13 +146347,13 @@ - - + + - - - + + + @@ -146422,10 +146370,10 @@ - - - - + + + + @@ -146436,47 +146384,47 @@ - - - + + + - - + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + From e0513ed978569bbb8e9983c5208dcd9745f2815d Mon Sep 17 00:00:00 2001 From: Ryun Park Date: Wed, 27 Apr 2022 17:00:08 +0900 Subject: [PATCH 041/156] ANDROID: ABI: Update allowed list for galaxy Leaf changes summary: 7 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 7 Added functions Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 0 Added variable 7 Added functions: [A] 'function int gpiochip_irqchip_add_key(gpio_chip*, irq_chip*, unsigned int, irq_flow_handler_t, unsigned int, bool, lock_class_key*, lock_class_key*)' [A] 'function void gpiochip_set_nested_irqchip(gpio_chip*, irq_chip*, unsigned int)' [A] 'function usb_request* gs_alloc_req(usb_ep*, unsigned int, gfp_t)' [A] 'function void gs_free_req(usb_ep*, usb_request*)' [A] 'function void gserial_free_line(unsigned char)' [A] 'function void gserial_resume(gserial*)' [A] 'function void gserial_suspend(gserial*)' Bug: 230572486 Change-Id: Ie85627ba247c5bc7f5e9da90934064f9ce5d9ba9 Signed-off-by: Ryun Park Signed-off-by: Greg Kroah-Hartman --- android/abi_gki_aarch64.xml | 47 ++++++++++++++++++++++++++++++++++ android/abi_gki_aarch64_galaxy | 7 +++++ 2 files changed, 54 insertions(+) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 5ffa2b016bcb..7f3641720ed4 100644 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -2662,10 +2662,12 @@ + + @@ -2693,9 +2695,14 @@ + + + + + @@ -129236,6 +129243,17 @@ + + + + + + + + + + + @@ -129256,6 +129274,12 @@ + + + + + + @@ -129383,6 +129407,17 @@ + + + + + + + + + + + @@ -129396,6 +129431,18 @@ + + + + + + + + + + + + diff --git a/android/abi_gki_aarch64_galaxy b/android/abi_gki_aarch64_galaxy index f35385449132..d5358ae912e2 100644 --- a/android/abi_gki_aarch64_galaxy +++ b/android/abi_gki_aarch64_galaxy @@ -2027,10 +2027,12 @@ gpiochip_generic_free gpiochip_generic_request gpiochip_get_data + gpiochip_irqchip_add_key gpiochip_line_is_valid gpiochip_lock_as_irq gpiochip_populate_parent_fwspec_fourcell gpiochip_remove + gpiochip_set_nested_irqchip gpiochip_unlock_as_irq gpiod_cansleep gpiod_count @@ -2053,9 +2055,14 @@ gro_cells_destroy gro_cells_init gro_cells_receive + gs_alloc_req + gs_free_req gserial_alloc_line gserial_connect gserial_disconnect + gserial_free_line + gserial_resume + gserial_suspend guid_gen handle_bad_irq handle_edge_irq From aed2e27d511a8232df3ed175ce0663c3ff2f1a02 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 9 Apr 2021 13:27:44 -0700 Subject: [PATCH 042/156] UPSTREAM: kfence, x86: fix preemptible warning on KPTI-enabled systems On systems with KPTI enabled, we can currently observe the following warning: BUG: using smp_processor_id() in preemptible caller is invalidate_user_asid+0x13/0x50 CPU: 6 PID: 1075 Comm: dmesg Not tainted 5.12.0-rc4-gda4a2b1a5479-kfence_1+ #1 Hardware name: Hewlett-Packard HP Pro 3500 Series/2ABF, BIOS 8.11 10/24/2012 Call Trace: dump_stack+0x7f/0xad check_preemption_disabled+0xc8/0xd0 invalidate_user_asid+0x13/0x50 flush_tlb_one_kernel+0x5/0x20 kfence_protect+0x56/0x80 ... While it normally makes sense to require preemption to be off, so that the expected CPU's TLB is flushed and not another, in our case it really is best-effort (see comments in kfence_protect_page()). Avoid the warning by disabling preemption around flush_tlb_one_kernel(). Link: https://lore.kernel.org/lkml/YGIDBAboELGgMgXy@elver.google.com/ Link: https://lkml.kernel.org/r/20210330065737.652669-1-elver@google.com Signed-off-by: Marco Elver Reported-by: Tomi Sarvela Cc: Alexander Potapenko Cc: Dmitry Vyukov Cc: Andrey Konovalov Cc: Jann Horn Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 6a77d38efcda40f555a920909eab22ee0917fd0d) Bug: 229863099 Signed-off-by: Colin Downs-Razouk Change-Id: Ia917b052ffbb267254f281f55141c34ad193c78e --- arch/x86/include/asm/kfence.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/kfence.h b/arch/x86/include/asm/kfence.h index 97bbb4a9083a..05b48b33baf0 100644 --- a/arch/x86/include/asm/kfence.h +++ b/arch/x86/include/asm/kfence.h @@ -56,8 +56,13 @@ static inline bool kfence_protect_page(unsigned long addr, bool protect) else set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT)); - /* Flush this CPU's TLB. */ + /* + * Flush this CPU's TLB, assuming whoever did the allocation/free is + * likely to continue running on this CPU. + */ + preempt_disable(); flush_tlb_one_kernel(addr); + preempt_enable(); return true; } From 2e3c211e7e977e6198ea36b472d1799d0d34b362 Mon Sep 17 00:00:00 2001 From: Jiazi Li Date: Wed, 29 Sep 2021 19:59:28 +0800 Subject: [PATCH 043/156] BACKPORT: dm: fix NULL pointer issue when free bio dm_io_dec_pending call end_io_acct first, will dec md in-flight pending count. If a task is swapping table at same time. task1 task2 do_resume ->do_suspend ->dm_wait_for_completion bio_endio ->clone_endio ->dm_io_dec_pending ->end_io_acct ->wakeup task1 ->dm_swap_table ->__bind ->__bind_mempools ->bioset_exit ->mempool_exit ->free_io mempool->elements is NULL, and lead to following crash: [ 67.330330] Unable to handle kernel NULL pointer dereference at virtual address 0000000000000000 ...... [ 67.330494] pstate: 80400085 (Nzcv daIf +PAN -UAO) [ 67.330510] pc : mempool_free+0x70/0xa0 [ 67.330515] lr : mempool_free+0x4c/0xa0 [ 67.330520] sp : ffffff8008013b20 [ 67.330524] x29: ffffff8008013b20 x28: 0000000000000004 [ 67.330530] x27: ffffffa8c2ff40a0 x26: 00000000ffff1cc8 [ 67.330535] x25: 0000000000000000 x24: ffffffdada34c800 [ 67.330541] x23: 0000000000000000 x22: ffffffdada34c800 [ 67.330547] x21: 00000000ffff1cc8 x20: ffffffd9a1304d80 [ 67.330552] x19: ffffffdada34c970 x18: 000000b312625d9c [ 67.330558] x17: 00000000002dcfbf x16: 00000000000006dd [ 67.330563] x15: 000000000093b41e x14: 0000000000000010 [ 67.330569] x13: 0000000000007f7a x12: 0000000034155555 [ 67.330574] x11: 0000000000000001 x10: 0000000000000001 [ 67.330579] x9 : 0000000000000000 x8 : 0000000000000000 [ 67.330585] x7 : 0000000000000000 x6 : ffffff80148b5c1a [ 67.330590] x5 : ffffff8008013ae0 x4 : 0000000000000001 [ 67.330596] x3 : ffffff80080139c8 x2 : ffffff801083bab8 [ 67.330601] x1 : 0000000000000000 x0 : ffffffdada34c970 [ 67.330609] Call trace: [ 67.330616] mempool_free+0x70/0xa0 [ 67.330627] bio_put+0xf8/0x110 [ 67.330638] dec_pending+0x13c/0x230 [ 67.330644] clone_endio+0x90/0x180 [ 67.330649] bio_endio+0x198/0x1b8 [ 67.330655] dec_pending+0x190/0x230 [ 67.330660] clone_endio+0x90/0x180 [ 67.330665] bio_endio+0x198/0x1b8 [ 67.330673] blk_update_request+0x214/0x428 [ 67.330683] scsi_end_request+0x2c/0x300 [ 67.330688] scsi_io_completion+0xa0/0x710 [ 67.330695] scsi_finish_command+0xd8/0x110 [ 67.330700] scsi_softirq_done+0x114/0x148 [ 67.330708] blk_done_softirq+0x74/0xd0 [ 67.330716] __do_softirq+0x18c/0x374 [ 67.330724] irq_exit+0xb4/0xb8 [ 67.330732] __handle_domain_irq+0x84/0xc0 [ 67.330737] gic_handle_irq+0x148/0x1b0 [ 67.330744] el1_irq+0xe8/0x190 [ 67.330753] lpm_cpuidle_enter+0x4f8/0x538 [ 67.330759] cpuidle_enter_state+0x1fc/0x398 [ 67.330764] cpuidle_enter+0x18/0x20 [ 67.330772] do_idle+0x1b4/0x290 [ 67.330778] cpu_startup_entry+0x20/0x28 [ 67.330786] secondary_start_kernel+0x160/0x170 Move end_io_acct after free_io to fix this issue. Bug: 228982905 Link: https://lore.kernel.org/dm-devel/1632916768-22379-1-git-send-email-lijiazi@xiaomi.com/T/#u [Akilesh: Resolved merge conflict in drivers/md/dm.c] Signed-off-by: Jiazi Li Signed-off-by: Akilesh Kailash (cherry picked from commit d208b89401e073de986dc891037c5a668f5d5d95) Change-Id: I9f122cab2af3b961c472b8cf2087399c63c28de1 --- drivers/md/dm.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 10eb0544f1cb..9fd176e225af 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -608,18 +608,17 @@ static void start_io_acct(struct dm_io *io) false, 0, &io->stats_aux); } -static void end_io_acct(struct dm_io *io) +static void end_io_acct(struct mapped_device *md, struct bio *bio, + unsigned long start_time, struct dm_stats_aux *stats_aux) { - struct mapped_device *md = io->md; - struct bio *bio = io->orig_bio; - unsigned long duration = jiffies - io->start_time; + unsigned long duration = jiffies - start_time; - bio_end_io_acct(bio, io->start_time); + bio_end_io_acct(bio, start_time); if (unlikely(dm_stats_used(&md->stats))) dm_stats_account_io(&md->stats, bio_data_dir(bio), bio->bi_iter.bi_sector, bio_sectors(bio), - true, duration, &io->stats_aux); + true, duration, stats_aux); /* nudge anyone waiting on suspend queue */ if (unlikely(wq_has_sleeper(&md->wait))) @@ -904,6 +903,8 @@ static void dec_pending(struct dm_io *io, blk_status_t error) blk_status_t io_error; struct bio *bio; struct mapped_device *md = io->md; + unsigned long start_time = 0; + struct dm_stats_aux stats_aux; /* Push-back supersedes any I/O errors */ if (unlikely(error)) { @@ -930,8 +931,10 @@ static void dec_pending(struct dm_io *io, blk_status_t error) io_error = io->status; bio = io->orig_bio; - end_io_acct(io); + start_time = io->start_time; + stats_aux = io->stats_aux; free_io(md, io); + end_io_acct(md, bio, start_time, &stats_aux); if (io_error == BLK_STS_DM_REQUEUE) return; From 24d464d38beb08cda1e972f79827d2fed6193de7 Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Mon, 7 Mar 2022 16:30:44 +0100 Subject: [PATCH 044/156] BACKPORT: fuse: fix pipe buffer lifetime for direct_io commit 0c4bcfdecb1ac0967619ee7ff44871d93c08c909 upstream. In FOPEN_DIRECT_IO mode, fuse_file_write_iter() calls fuse_direct_write_iter(), which normally calls fuse_direct_io(), which then imports the write buffer with fuse_get_user_pages(), which uses iov_iter_get_pages() to grab references to userspace pages instead of actually copying memory. On the filesystem device side, these pages can then either be read to userspace (via fuse_dev_read()), or splice()d over into a pipe using fuse_dev_splice_read() as pipe buffers with &nosteal_pipe_buf_ops. This is wrong because after fuse_dev_do_read() unlocks the FUSE request, the userspace filesystem can mark the request as completed, causing write() to return. At that point, the userspace filesystem should no longer have access to the pipe buffer. Fix by copying pages coming from the user address space to new pipe buffers. Bug: 226679409 Reported-by: Jann Horn Fixes: c3021629a0d8 ("fuse: support splice() reading from fuse device") Cc: Signed-off-by: Miklos Szeredi Signed-off-by: Greg Kroah-Hartman Signed-off-by: Lee Jones Change-Id: I57a98e96e36bb97ce3e7b1ebf88917c6c8b0247d --- fs/fuse/dev.c | 12 +++++++++++- fs/fuse/file.c | 1 + fs/fuse/fuse_i.h | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index c93b0154dd51..e9a1543ba7cb 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -950,7 +950,17 @@ static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep, while (count) { if (cs->write && cs->pipebufs && page) { - return fuse_ref_page(cs, page, offset, count); + /* + * Can't control lifetime of pipe buffers, so always + * copy user pages. + */ + if (cs->req->args->user_pages) { + err = fuse_copy_fill(cs); + if (err) + return err; + } else { + return fuse_ref_page(cs, page, offset, count); + } } else if (!cs->len) { if (cs->move_pages && page && offset == 0 && count == PAGE_SIZE) { diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 97a40476d1bd..a68c8310942f 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1420,6 +1420,7 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, (PAGE_SIZE - ret) & (PAGE_SIZE - 1); } + ap->args.user_pages = true; if (write) ap->args.in_pages = true; else diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index fc5fcd8e0671..69a631c12b15 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -277,6 +277,7 @@ struct fuse_args { bool nocreds:1; bool in_pages:1; bool out_pages:1; + bool user_pages:1; bool out_argvar:1; bool page_zeroing:1; bool page_replace:1; From 35a697cab43521c0292e076f8bcccec141e358a3 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sat, 12 Mar 2022 15:29:58 -0800 Subject: [PATCH 045/156] BACKPORT: net/packet: fix slab-out-of-bounds access in packet_recvmsg() [ Upstream commit c700525fcc06b05adfea78039de02628af79e07a ] syzbot found that when an AF_PACKET socket is using PACKET_COPY_THRESH and mmap operations, tpacket_rcv() is queueing skbs with garbage in skb->cb[], triggering a too big copy [1] Presumably, users of af_packet using mmap() already gets correct metadata from the mapped buffer, we can simply make sure to clear 12 bytes that might be copied to user space later. BUG: KASAN: stack-out-of-bounds in memcpy include/linux/fortify-string.h:225 [inline] BUG: KASAN: stack-out-of-bounds in packet_recvmsg+0x56c/0x1150 net/packet/af_packet.c:3489 Write of size 165 at addr ffffc9000385fb78 by task syz-executor233/3631 CPU: 0 PID: 3631 Comm: syz-executor233 Not tainted 5.17.0-rc7-syzkaller-02396-g0b3660695e80 #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0xcd/0x134 lib/dump_stack.c:106 print_address_description.constprop.0.cold+0xf/0x336 mm/kasan/report.c:255 __kasan_report mm/kasan/report.c:442 [inline] kasan_report.cold+0x83/0xdf mm/kasan/report.c:459 check_region_inline mm/kasan/generic.c:183 [inline] kasan_check_range+0x13d/0x180 mm/kasan/generic.c:189 memcpy+0x39/0x60 mm/kasan/shadow.c:66 memcpy include/linux/fortify-string.h:225 [inline] packet_recvmsg+0x56c/0x1150 net/packet/af_packet.c:3489 sock_recvmsg_nosec net/socket.c:948 [inline] sock_recvmsg net/socket.c:966 [inline] sock_recvmsg net/socket.c:962 [inline] ____sys_recvmsg+0x2c4/0x600 net/socket.c:2632 ___sys_recvmsg+0x127/0x200 net/socket.c:2674 __sys_recvmsg+0xe2/0x1a0 net/socket.c:2704 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7fdfd5954c29 Code: 28 00 00 00 75 05 48 83 c4 28 c3 e8 41 15 00 00 90 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 c0 ff ff ff f7 d8 64 89 01 48 RSP: 002b:00007ffcf8e71e48 EFLAGS: 00000246 ORIG_RAX: 000000000000002f RAX: ffffffffffffffda RBX: 0000000000000003 RCX: 00007fdfd5954c29 RDX: 0000000000000000 RSI: 0000000020000500 RDI: 0000000000000005 RBP: 0000000000000000 R08: 000000000000000d R09: 000000000000000d R10: 0000000000000000 R11: 0000000000000246 R12: 00007ffcf8e71e60 R13: 00000000000f4240 R14: 000000000000c1ff R15: 00007ffcf8e71e54 addr ffffc9000385fb78 is located in stack of task syz-executor233/3631 at offset 32 in frame: ____sys_recvmsg+0x0/0x600 include/linux/uio.h:246 this frame has 1 object: [32, 160) 'addr' Memory state around the buggy address: ffffc9000385fa80: 00 04 f3 f3 f3 f3 f3 00 00 00 00 00 00 00 00 00 ffffc9000385fb00: 00 00 00 00 00 00 00 00 00 00 00 f1 f1 f1 f1 00 >ffffc9000385fb80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 ^ ffffc9000385fc00: f3 f3 f3 00 00 00 00 00 00 00 00 00 00 00 00 f1 ffffc9000385fc80: f1 f1 f1 00 f2 f2 f2 00 f2 f2 f2 00 00 00 00 00 ================================================================== Bug: 224546354 Fixes: 0fb375fb9b93 ("[AF_PACKET]: Allow for > 8 byte hardware addresses.") Signed-off-by: Eric Dumazet Reported-by: syzbot Link: https://lore.kernel.org/r/20220312232958.3535620-1-eric.dumazet@gmail.com Signed-off-by: Jakub Kicinski Signed-off-by: Sasha Levin Signed-off-by: Lee Jones Change-Id: I37e4a05a8d81b2645bc65db002e644b40d1a984d --- net/packet/af_packet.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index a31334b92be7..d0c95d7dd292 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2278,8 +2278,11 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, copy_skb = skb_get(skb); skb_head = skb->data; } - if (copy_skb) + if (copy_skb) { + memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0, + sizeof(PACKET_SKB_CB(copy_skb)->sa.ll)); skb_set_owner_r(copy_skb, sk); + } } snaplen = po->rx_ring.frame_size - macoff; if ((int)snaplen < 0) { @@ -3434,6 +3437,8 @@ static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, sock_recv_ts_and_drops(msg, sk, skb); if (msg->msg_name) { + const size_t max_len = min(sizeof(skb->cb), + sizeof(struct sockaddr_storage)); int copy_len; /* If the address length field is there to be filled @@ -3456,6 +3461,10 @@ static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, msg->msg_namelen = sizeof(struct sockaddr_ll); } } + if (WARN_ON_ONCE(copy_len > max_len)) { + copy_len = max_len; + msg->msg_namelen = copy_len; + } memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len); } From bc19dd9a510c1700d91dbbb3ed8329eb55282ae8 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 7 Sep 2021 19:56:28 -0700 Subject: [PATCH 046/156] UPSTREAM: mm: introduce Data Access MONitor (DAMON) Patch series "Introduce Data Access MONitor (DAMON)", v34. Introduction ============ DAMON is a data access monitoring framework for the Linux kernel. The core mechanisms of DAMON called 'region based sampling' and 'adaptive regions adjustment' (refer to 'mechanisms.rst' in the 11th patch of this patchset for the detail) make it - accurate (The monitored information is useful for DRAM level memory management. It might not appropriate for Cache-level accuracy, though.), - light-weight (The monitoring overhead is low enough to be applied online while making no impact on the performance of the target workloads.), and - scalable (the upper-bound of the instrumentation overhead is controllable regardless of the size of target workloads.). Using this framework, therefore, several memory management mechanisms such as reclamation and THP can be optimized to aware real data access patterns. Experimental access pattern aware memory management optimization works that incurring high instrumentation overhead will be able to have another try. Though DAMON is for kernel subsystems, it can be easily exposed to the user space by writing a DAMON-wrapper kernel subsystem. Then, user space users who have some special workloads will be able to write personalized tools or applications for deeper understanding and specialized optimizations of their systems. DAMON is also merged in two public Amazon Linux kernel trees that based on v5.4.y[1] and v5.10.y[2]. [1] https://github.com/amazonlinux/linux/tree/amazon-5.4.y/master/mm/damon [2] https://github.com/amazonlinux/linux/tree/amazon-5.10.y/master/mm/damon The userspace tool[1] is available, released under GPLv2, and actively being maintained. I am also planning to implement another basic user interface in perf[2]. Also, the basic test suite for DAMON is available under GPLv2[3]. [1] https://github.com/awslabs/damo [2] https://lore.kernel.org/linux-mm/20210107120729.22328-1-sjpark@amazon.com/ [3] https://github.com/awslabs/damon-tests Long-term Plan -------------- DAMON is a part of a project called Data Access-aware Operating System (DAOS). As the name implies, I want to improve the performance and efficiency of systems using fine-grained data access patterns. The optimizations are for both kernel and user spaces. I will therefore modify or create kernel subsystems, export some of those to user space and implement user space library / tools. Below shows the layers and components for the project. --------------------------------------------------------------------------- Primitives: PTE Accessed bit, PG_idle, rmap, (Intel CMT), ... Framework: DAMON Features: DAMOS, virtual addr, physical addr, ... Applications: DAMON-debugfs, (DARC), ... ^^^^^^^^^^^^^^^^^^^^^^^ KERNEL SPACE ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Raw Interface: debugfs, (sysfs), (damonfs), tracepoints, (sys_damon), ... vvvvvvvvvvvvvvvvvvvvvvv USER SPACE vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv Library: (libdamon), ... Tools: DAMO, (perf), ... --------------------------------------------------------------------------- The components in parentheses or marked as '...' are not implemented yet but in the future plan. IOW, those are the TODO tasks of DAOS project. For more detail, please refer to the plans: https://lore.kernel.org/linux-mm/20201202082731.24828-1-sjpark@amazon.com/ Evaluations =========== We evaluated DAMON's overhead, monitoring quality and usefulness using 24 realistic workloads on my QEMU/KVM based virtual machine running a kernel that v24 DAMON patchset is applied. DAMON is lightweight. It increases system memory usage by 0.39% and slows target workloads down by 1.16%. DAMON is accurate and useful for memory management optimizations. An experimental DAMON-based operation scheme for THP, namely 'ethp', removes 76.15% of THP memory overheads while preserving 51.25% of THP speedup. Another experimental DAMON-based 'proactive reclamation' implementation, 'prcl', reduces 93.38% of residential sets and 23.63% of system memory footprint while incurring only 1.22% runtime overhead in the best case (parsec3/freqmine). NOTE that the experimental THP optimization and proactive reclamation are not for production but only for proof of concepts. Please refer to the official document[1] or "Documentation/admin-guide/mm: Add a document for DAMON" patch in this patchset for detailed evaluation setup and results. [1] https://damonitor.github.io/doc/html/latest-damon/admin-guide/mm/damon/eval.html Real-world User Story ===================== In summary, DAMON has used on production systems and proved its usefulness. DAMON as a profiler ------------------- We analyzed characteristics of a large scale production systems of our customers using DAMON. The systems utilize 70GB DRAM and 36 CPUs. From this, we were able to find interesting things below. There were obviously different access pattern under idle workload and active workload. Under the idle workload, it accessed large memory regions with low frequency, while the active workload accessed small memory regions with high freuqnecy. DAMON found a 7GB memory region that showing obviously high access frequency under the active workload. We believe this is the performance-effective working set and need to be protected. There was a 4KB memory region that showing highest access frequency under not only active but also idle workloads. We think this must be a hottest code section like thing that should never be paged out. For this analysis, DAMON used only 0.3-1% of single CPU time. Because we used recording-based analysis, it consumed about 3-12 MB of disk space per 20 minutes. This is only small amount of disk space, but we can further reduce the disk usage by using non-recording-based DAMON features. I'd like to argue that only DAMON can do such detailed analysis (finding 4KB highest region in 70GB memory) with the light overhead. DAMON as a system optimization tool ----------------------------------- We also found below potential performance problems on the systems and made DAMON-based solutions. The system doesn't want to make the workload suffer from the page reclamation and thus it utilizes enough DRAM but no swap device. However, we found the system is actively reclaiming file-backed pages, because the system has intensive file IO. The file IO turned out to be not performance critical for the workload, but the customer wanted to ensure performance critical file-backed pages like code section to not mistakenly be evicted. Using direct IO should or `mlock()` would be a straightforward solution, but modifying the user space code is not easy for the customer. Alternatively, we could use DAMON-based operation scheme[1]. By using it, we can ask DAMON to track access frequency of each region and make 'process_madvise(MADV_WILLNEED)[2]' call for regions having specific size and access frequency for a time interval. We also found the system is having high number of TLB misses. We tried 'always' THP enabled policy and it greatly reduced TLB misses, but the page reclamation also been more frequent due to the THP internal fragmentation caused memory bloat. We could try another DAMON-based operation scheme that applies 'MADV_HUGEPAGE' to memory regions having >=2MB size and high access frequency, while applying 'MADV_NOHUGEPAGE' to regions having <2MB size and low access frequency. We do not own the systems so we only reported the analysis results and possible optimization solutions to the customers. The customers satisfied about the analysis results and promised to try the optimization guides. [1] https://lore.kernel.org/linux-mm/20201006123931.5847-1-sjpark@amazon.com/ [2] https://lore.kernel.org/linux-api/20200622192900.22757-4-minchan@kernel.org/ Comparison with Idle Page Tracking ================================== Idle Page Tracking allows users to set and read idleness of pages using a bitmap file which represents each page with each bit of the file. One recommended usage of it is working set size detection. Users can do that by 1. find PFN of each page for workloads in interest, 2. set all the pages as idle by doing writes to the bitmap file, 3. wait until the workload accesses its working set, and 4. read the idleness of the pages again and count pages became not idle. NOTE: While Idle Page Tracking is for user space users, DAMON is primarily designed for kernel subsystems though it can easily exposed to the user space. Hence, this section only assumes such user space use of DAMON. For what use cases Idle Page Tracking would be better? ------------------------------------------------------ 1. Flexible usecases other than hotness monitoring. Because Idle Page Tracking allows users to control the primitive (Page idleness) by themselves, Idle Page Tracking users can do anything they want. Meanwhile, DAMON is primarily designed to monitor the hotness of each memory region. For this, DAMON asks users to provide sampling interval and aggregation interval. For the reason, there could be some use case that using Idle Page Tracking is simpler. 2. Physical memory monitoring. Idle Page Tracking receives PFN range as input, so natively supports physical memory monitoring. DAMON is designed to be extensible for multiple address spaces and use cases by implementing and using primitives for the given use case. Therefore, by theory, DAMON has no limitation in the type of target address space as long as primitives for the given address space exists. However, the default primitives introduced by this patchset supports only virtual address spaces. Therefore, for physical memory monitoring, you should implement your own primitives and use it, or simply use Idle Page Tracking. Nonetheless, RFC patchsets[1] for the physical memory address space primitives is already available. It also supports user memory same to Idle Page Tracking. [1] https://lore.kernel.org/linux-mm/20200831104730.28970-1-sjpark@amazon.com/ For what use cases DAMON is better? ----------------------------------- 1. Hotness Monitoring. Idle Page Tracking let users know only if a page frame is accessed or not. For hotness check, the user should write more code and use more memory. DAMON do that by itself. 2. Low Monitoring Overhead DAMON receives user's monitoring request with one step and then provide the results. So, roughly speaking, DAMON require only O(1) user/kernel context switches. In case of Idle Page Tracking, however, because the interface receives contiguous page frames, the number of user/kernel context switches increases as the monitoring target becomes complex and huge. As a result, the context switch overhead could be not negligible. Moreover, DAMON is born to handle with the monitoring overhead. Because the core mechanism is pure logical, Idle Page Tracking users might be able to implement the mechanism on their own, but it would be time consuming and the user/kernel context switching will still more frequent than that of DAMON. Also, the kernel subsystems cannot use the logic in this case. 3. Page granularity working set size detection. Until v22 of this patchset, this was categorized as the thing Idle Page Tracking could do better, because DAMON basically maintains additional metadata for each of the monitoring target regions. So, in the page granularity working set size detection use case, DAMON would incur (number of monitoring target pages * size of metadata) memory overhead. Size of the single metadata item is about 54 bytes, so assuming 4KB pages, about 1.3% of monitoring target pages will be additionally used. All essential metadata for Idle Page Tracking are embedded in 'struct page' and page table entries. Therefore, in this use case, only one counter variable for working set size accounting is required if Idle Page Tracking is used. There are more details to consider, but roughly speaking, this is true in most cases. However, the situation changed from v23. Now DAMON supports arbitrary types of monitoring targets, which don't use the metadata. Using that, DAMON can do the working set size detection with no additional space overhead but less user-kernel context switch. A first draft for the implementation of monitoring primitives for this usage is available in a DAMON development tree[1]. An RFC patchset for it based on this patchset will also be available soon. Since v24, the arbitrary type support is dropped from this patchset because this patchset doesn't introduce real use of the type. You can still get it from the DAMON development tree[2], though. [1] https://github.com/sjp38/linux/tree/damon/pgidle_hack [2] https://github.com/sjp38/linux/tree/damon/master 4. More future usecases While Idle Page Tracking has tight coupling with base primitives (PG_Idle and page table Accessed bits), DAMON is designed to be extensible for many use cases and address spaces. If you need some special address type or want to use special h/w access check primitives, you can write your own primitives for that and configure DAMON to use those. Therefore, if your use case could be changed a lot in future, using DAMON could be better. Can I use both Idle Page Tracking and DAMON? -------------------------------------------- Yes, though using them concurrently for overlapping memory regions could result in interference to each other. Nevertheless, such use case would be rare or makes no sense at all. Even in the case, the noise would bot be really significant. So, you can choose whatever you want depending on the characteristics of your use cases. More Information ================ We prepared a showcase web site[1] that you can get more information. There are - the official documentations[2], - the heatmap format dynamic access pattern of various realistic workloads for heap area[3], mmap()-ed area[4], and stack[5] area, - the dynamic working set size distribution[6] and chronological working set size changes[7], and - the latest performance test results[8]. [1] https://damonitor.github.io/_index [2] https://damonitor.github.io/doc/html/latest-damon [3] https://damonitor.github.io/test/result/visual/latest/rec.heatmap.0.png.html [4] https://damonitor.github.io/test/result/visual/latest/rec.heatmap.1.png.html [5] https://damonitor.github.io/test/result/visual/latest/rec.heatmap.2.png.html [6] https://damonitor.github.io/test/result/visual/latest/rec.wss_sz.png.html [7] https://damonitor.github.io/test/result/visual/latest/rec.wss_time.png.html [8] https://damonitor.github.io/test/result/perf/latest/html/index.html Baseline and Complete Git Trees =============================== The patches are based on the latest -mm tree, specifically v5.14-rc1-mmots-2021-07-15-18-47 of https://github.com/hnaz/linux-mm. You can also clone the complete git tree: $ git clone git://github.com/sjp38/linux -b damon/patches/v34 The web is also available: https://github.com/sjp38/linux/releases/tag/damon/patches/v34 Development Trees ----------------- There are a couple of trees for entire DAMON patchset series and features for future release. - For latest release: https://github.com/sjp38/linux/tree/damon/master - For next release: https://github.com/sjp38/linux/tree/damon/next Long-term Support Trees ----------------------- For people who want to test DAMON but using LTS kernels, there are another couple of trees based on two latest LTS kernels respectively and containing the 'damon/master' backports. - For v5.4.y: https://github.com/sjp38/linux/tree/damon/for-v5.4.y - For v5.10.y: https://github.com/sjp38/linux/tree/damon/for-v5.10.y Amazon Linux Kernel Trees ------------------------- DAMON is also merged in two public Amazon Linux kernel trees that based on v5.4.y[1] and v5.10.y[2]. [1] https://github.com/amazonlinux/linux/tree/amazon-5.4.y/master/mm/damon [2] https://github.com/amazonlinux/linux/tree/amazon-5.10.y/master/mm/damon Git Tree for Diff of Patches ============================ For easy review of diff between different versions of each patch, I prepared a git tree containing all versions of the DAMON patchset series: https://github.com/sjp38/damon-patches You can clone it and use 'diff' for easy review of changes between different versions of the patchset. For example: $ git clone https://github.com/sjp38/damon-patches && cd damon-patches $ diff -u damon/v33 damon/v34 Sequence Of Patches =================== First three patches implement the core logics of DAMON. The 1st patch introduces basic sampling based hotness monitoring for arbitrary types of targets. Following two patches implement the core mechanisms for control of overhead and accuracy, namely regions based sampling (patch 2) and adaptive regions adjustment (patch 3). Now the essential parts of DAMON is complete, but it cannot work unless someone provides monitoring primitives for a specific use case. The following two patches make it just work for virtual address spaces monitoring. The 4th patch makes 'PG_idle' can be used by DAMON and the 5th patch implements the virtual memory address space specific monitoring primitives using page table Accessed bits and the 'PG_idle' page flag. Now DAMON just works for virtual address space monitoring via the kernel space api. To let the user space users can use DAMON, following four patches add interfaces for them. The 6th patch adds a tracepoint for monitoring results. The 7th patch implements a DAMON application kernel module, namely damon-dbgfs, that simply wraps DAMON and exposes DAMON interface to the user space via the debugfs interface. The 8th patch further exports pid of monitoring thread (kdamond) to user space for easier cpu usage accounting, and the 9th patch makes the debugfs interface to support multiple contexts. Three patches for maintainability follows. The 10th patch adds documentations for both the user space and the kernel space. The 11th patch provides unit tests (based on the kunit) while the 12th patch adds user space tests (based on the kselftest). Finally, the last patch (13th) updates the MAINTAINERS file. This patch (of 13): DAMON is a data access monitoring framework for the Linux kernel. The core mechanisms of DAMON make it - accurate (the monitoring output is useful enough for DRAM level performance-centric memory management; It might be inappropriate for CPU cache levels, though), - light-weight (the monitoring overhead is normally low enough to be applied online), and - scalable (the upper-bound of the overhead is in constant range regardless of the size of target workloads). Using this framework, hence, we can easily write efficient kernel space data access monitoring applications. For example, the kernel's memory management mechanisms can make advanced decisions using this. Experimental data access aware optimization works that incurring high access monitoring overhead could again be implemented on top of this. Due to its simple and flexible interface, providing user space interface would be also easy. Then, user space users who have some special workloads can write personalized applications for better understanding and optimizations of their workloads and systems. === Nevertheless, this commit is defining and implementing only basic access check part without the overhead-accuracy handling core logic. The basic access check is as below. The output of DAMON says what memory regions are how frequently accessed for a given duration. The resolution of the access frequency is controlled by setting ``sampling interval`` and ``aggregation interval``. In detail, DAMON checks access to each page per ``sampling interval`` and aggregates the results. In other words, counts the number of the accesses to each region. After each ``aggregation interval`` passes, DAMON calls callback functions that previously registered by users so that users can read the aggregated results and then clears the results. This can be described in below simple pseudo-code:: init() while monitoring_on: for page in monitoring_target: if accessed(page): nr_accesses[page] += 1 if time() % aggregation_interval == 0: for callback in user_registered_callbacks: callback(monitoring_target, nr_accesses) for page in monitoring_target: nr_accesses[page] = 0 if time() % update_interval == 0: update() sleep(sampling interval) The target regions constructed at the beginning of the monitoring and updated after each ``regions_update_interval``, because the target regions could be dynamically changed (e.g., mmap() or memory hotplug). The monitoring overhead of this mechanism will arbitrarily increase as the size of the target workload grows. The basic monitoring primitives for actual access check and dynamic target regions construction aren't in the core part of DAMON. Instead, it allows users to implement their own primitives that are optimized for their use case and configure DAMON to use those. In other words, users cannot use current version of DAMON without some additional works. Following commits will implement the core mechanisms for the overhead-accuracy control and default primitives implementations. Link: https://lkml.kernel.org/r/20210716081449.22187-1-sj38.park@gmail.com Link: https://lkml.kernel.org/r/20210716081449.22187-2-sj38.park@gmail.com Signed-off-by: SeongJae Park Reviewed-by: Leonard Foerster Reviewed-by: Fernand Sieber Acked-by: Shakeel Butt Cc: Jonathan Cameron Cc: Alexander Shishkin Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Jonathan Corbet Cc: David Hildenbrand Cc: David Woodhouse Cc: Marco Elver Cc: Fan Du Cc: Greg Kroah-Hartman Cc: Greg Thelen Cc: Joe Perches Cc: Mel Gorman Cc: Maximilian Heyne Cc: Minchan Kim Cc: Ingo Molnar Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rik van Riel Cc: David Rientjes Cc: Steven Rostedt (VMware) Cc: Shuah Khan Cc: Vlastimil Babka Cc: Vladimir Davydov Cc: Brendan Higgins Cc: Markus Boehme Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 2224d8485492e499ca2e5d25407f8502cc06f149) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I578b47551306e4c416e7b97c57335f810783615c --- include/linux/damon.h | 167 ++++++++++++++++++++++ mm/Kconfig | 2 + mm/Makefile | 1 + mm/damon/Kconfig | 15 ++ mm/damon/Makefile | 3 + mm/damon/core.c | 320 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 508 insertions(+) create mode 100644 include/linux/damon.h create mode 100644 mm/damon/Kconfig create mode 100644 mm/damon/Makefile create mode 100644 mm/damon/core.c diff --git a/include/linux/damon.h b/include/linux/damon.h new file mode 100644 index 000000000000..2f652602b1ea --- /dev/null +++ b/include/linux/damon.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * DAMON api + * + * Author: SeongJae Park + */ + +#ifndef _DAMON_H_ +#define _DAMON_H_ + +#include +#include +#include + +struct damon_ctx; + +/** + * struct damon_primitive Monitoring primitives for given use cases. + * + * @init: Initialize primitive-internal data structures. + * @update: Update primitive-internal data structures. + * @prepare_access_checks: Prepare next access check of target regions. + * @check_accesses: Check the accesses to target regions. + * @reset_aggregated: Reset aggregated accesses monitoring results. + * @target_valid: Determine if the target is valid. + * @cleanup: Clean up the context. + * + * DAMON can be extended for various address spaces and usages. For this, + * users should register the low level primitives for their target address + * space and usecase via the &damon_ctx.primitive. Then, the monitoring thread + * (&damon_ctx.kdamond) calls @init and @prepare_access_checks before starting + * the monitoring, @update after each &damon_ctx.primitive_update_interval, and + * @check_accesses, @target_valid and @prepare_access_checks after each + * &damon_ctx.sample_interval. Finally, @reset_aggregated is called after each + * &damon_ctx.aggr_interval. + * + * @init should initialize primitive-internal data structures. For example, + * this could be used to construct proper monitoring target regions and link + * those to @damon_ctx.target. + * @update should update the primitive-internal data structures. For example, + * this could be used to update monitoring target regions for current status. + * @prepare_access_checks should manipulate the monitoring regions to be + * prepared for the next access check. + * @check_accesses should check the accesses to each region that made after the + * last preparation and update the number of observed accesses of each region. + * @reset_aggregated should reset the access monitoring results that aggregated + * by @check_accesses. + * @target_valid should check whether the target is still valid for the + * monitoring. + * @cleanup is called from @kdamond just before its termination. + */ +struct damon_primitive { + void (*init)(struct damon_ctx *context); + void (*update)(struct damon_ctx *context); + void (*prepare_access_checks)(struct damon_ctx *context); + void (*check_accesses)(struct damon_ctx *context); + void (*reset_aggregated)(struct damon_ctx *context); + bool (*target_valid)(void *target); + void (*cleanup)(struct damon_ctx *context); +}; + +/* + * struct damon_callback Monitoring events notification callbacks. + * + * @before_start: Called before starting the monitoring. + * @after_sampling: Called after each sampling. + * @after_aggregation: Called after each aggregation. + * @before_terminate: Called before terminating the monitoring. + * @private: User private data. + * + * The monitoring thread (&damon_ctx.kdamond) calls @before_start and + * @before_terminate just before starting and finishing the monitoring, + * respectively. Therefore, those are good places for installing and cleaning + * @private. + * + * The monitoring thread calls @after_sampling and @after_aggregation for each + * of the sampling intervals and aggregation intervals, respectively. + * Therefore, users can safely access the monitoring results without additional + * protection. For the reason, users are recommended to use these callback for + * the accesses to the results. + * + * If any callback returns non-zero, monitoring stops. + */ +struct damon_callback { + void *private; + + int (*before_start)(struct damon_ctx *context); + int (*after_sampling)(struct damon_ctx *context); + int (*after_aggregation)(struct damon_ctx *context); + int (*before_terminate)(struct damon_ctx *context); +}; + +/** + * struct damon_ctx - Represents a context for each monitoring. This is the + * main interface that allows users to set the attributes and get the results + * of the monitoring. + * + * @sample_interval: The time between access samplings. + * @aggr_interval: The time between monitor results aggregations. + * @primitive_update_interval: The time between monitoring primitive updates. + * + * For each @sample_interval, DAMON checks whether each region is accessed or + * not. It aggregates and keeps the access information (number of accesses to + * each region) for @aggr_interval time. DAMON also checks whether the target + * memory regions need update (e.g., by ``mmap()`` calls from the application, + * in case of virtual memory monitoring) and applies the changes for each + * @primitive_update_interval. All time intervals are in micro-seconds. + * Please refer to &struct damon_primitive and &struct damon_callback for more + * detail. + * + * @kdamond: Kernel thread who does the monitoring. + * @kdamond_stop: Notifies whether kdamond should stop. + * @kdamond_lock: Mutex for the synchronizations with @kdamond. + * + * For each monitoring context, one kernel thread for the monitoring is + * created. The pointer to the thread is stored in @kdamond. + * + * Once started, the monitoring thread runs until explicitly required to be + * terminated or every monitoring target is invalid. The validity of the + * targets is checked via the &damon_primitive.target_valid of @primitive. The + * termination can also be explicitly requested by writing non-zero to + * @kdamond_stop. The thread sets @kdamond to NULL when it terminates. + * Therefore, users can know whether the monitoring is ongoing or terminated by + * reading @kdamond. Reads and writes to @kdamond and @kdamond_stop from + * outside of the monitoring thread must be protected by @kdamond_lock. + * + * Note that the monitoring thread protects only @kdamond and @kdamond_stop via + * @kdamond_lock. Accesses to other fields must be protected by themselves. + * + * @primitive: Set of monitoring primitives for given use cases. + * @callback: Set of callbacks for monitoring events notifications. + * + * @target: Pointer to the user-defined monitoring target. + */ +struct damon_ctx { + unsigned long sample_interval; + unsigned long aggr_interval; + unsigned long primitive_update_interval; + +/* private: internal use only */ + struct timespec64 last_aggregation; + struct timespec64 last_primitive_update; + +/* public: */ + struct task_struct *kdamond; + bool kdamond_stop; + struct mutex kdamond_lock; + + struct damon_primitive primitive; + struct damon_callback callback; + + void *target; +}; + +#ifdef CONFIG_DAMON + +struct damon_ctx *damon_new_ctx(void); +void damon_destroy_ctx(struct damon_ctx *ctx); +int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, + unsigned long aggr_int, unsigned long primitive_upd_int); + +int damon_start(struct damon_ctx **ctxs, int nr_ctxs); +int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); + +#endif /* CONFIG_DAMON */ + +#endif /* _DAMON_H */ diff --git a/mm/Kconfig b/mm/Kconfig index 884ba3860ec8..5f5a66efcaba 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -888,4 +888,6 @@ config ARCH_HAS_HUGEPD config MAPPING_DIRTY_HELPERS bool +source "mm/damon/Kconfig" + endmenu diff --git a/mm/Makefile b/mm/Makefile index 8de8651da069..b90583a89fb5 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -116,6 +116,7 @@ obj-$(CONFIG_USERFAULTFD) += userfaultfd.o obj-$(CONFIG_IDLE_PAGE_TRACKING) += page_idle.o obj-$(CONFIG_FRAME_VECTOR) += frame_vector.o obj-$(CONFIG_DEBUG_PAGE_REF) += debug_page_ref.o +obj-$(CONFIG_DAMON) += damon/ obj-$(CONFIG_HARDENED_USERCOPY) += usercopy.o obj-$(CONFIG_PERCPU_STATS) += percpu-stats.o obj-$(CONFIG_ZONE_DEVICE) += memremap.o diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig new file mode 100644 index 000000000000..d00e99ac1a15 --- /dev/null +++ b/mm/damon/Kconfig @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only + +menu "Data Access Monitoring" + +config DAMON + bool "DAMON: Data Access Monitoring Framework" + help + This builds a framework that allows kernel subsystems to monitor + access frequency of each memory region. The information can be useful + for performance-centric DRAM level memory management. + + See https://damonitor.github.io/doc/html/latest-damon/index.html for + more information. + +endmenu diff --git a/mm/damon/Makefile b/mm/damon/Makefile new file mode 100644 index 000000000000..4fd2edb4becf --- /dev/null +++ b/mm/damon/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_DAMON) := core.o diff --git a/mm/damon/core.c b/mm/damon/core.c new file mode 100644 index 000000000000..651590bf49b1 --- /dev/null +++ b/mm/damon/core.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Data Access Monitor + * + * Author: SeongJae Park + */ + +#define pr_fmt(fmt) "damon: " fmt + +#include +#include +#include +#include + +static DEFINE_MUTEX(damon_lock); +static int nr_running_ctxs; + +struct damon_ctx *damon_new_ctx(void) +{ + struct damon_ctx *ctx; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + + ctx->sample_interval = 5 * 1000; + ctx->aggr_interval = 100 * 1000; + ctx->primitive_update_interval = 60 * 1000 * 1000; + + ktime_get_coarse_ts64(&ctx->last_aggregation); + ctx->last_primitive_update = ctx->last_aggregation; + + mutex_init(&ctx->kdamond_lock); + + ctx->target = NULL; + + return ctx; +} + +void damon_destroy_ctx(struct damon_ctx *ctx) +{ + if (ctx->primitive.cleanup) + ctx->primitive.cleanup(ctx); + kfree(ctx); +} + +/** + * damon_set_attrs() - Set attributes for the monitoring. + * @ctx: monitoring context + * @sample_int: time interval between samplings + * @aggr_int: time interval between aggregations + * @primitive_upd_int: time interval between monitoring primitive updates + * + * This function should not be called while the kdamond is running. + * Every time interval is in micro-seconds. + * + * Return: 0 on success, negative error code otherwise. + */ +int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, + unsigned long aggr_int, unsigned long primitive_upd_int) +{ + ctx->sample_interval = sample_int; + ctx->aggr_interval = aggr_int; + ctx->primitive_update_interval = primitive_upd_int; + + return 0; +} + +static bool damon_kdamond_running(struct damon_ctx *ctx) +{ + bool running; + + mutex_lock(&ctx->kdamond_lock); + running = ctx->kdamond != NULL; + mutex_unlock(&ctx->kdamond_lock); + + return running; +} + +static int kdamond_fn(void *data); + +/* + * __damon_start() - Starts monitoring with given context. + * @ctx: monitoring context + * + * This function should be called while damon_lock is hold. + * + * Return: 0 on success, negative error code otherwise. + */ +static int __damon_start(struct damon_ctx *ctx) +{ + int err = -EBUSY; + + mutex_lock(&ctx->kdamond_lock); + if (!ctx->kdamond) { + err = 0; + ctx->kdamond_stop = false; + ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d", + nr_running_ctxs); + if (IS_ERR(ctx->kdamond)) { + err = PTR_ERR(ctx->kdamond); + ctx->kdamond = 0; + } + } + mutex_unlock(&ctx->kdamond_lock); + + return err; +} + +/** + * damon_start() - Starts the monitorings for a given group of contexts. + * @ctxs: an array of the pointers for contexts to start monitoring + * @nr_ctxs: size of @ctxs + * + * This function starts a group of monitoring threads for a group of monitoring + * contexts. One thread per each context is created and run in parallel. The + * caller should handle synchronization between the threads by itself. If a + * group of threads that created by other 'damon_start()' call is currently + * running, this function does nothing but returns -EBUSY. + * + * Return: 0 on success, negative error code otherwise. + */ +int damon_start(struct damon_ctx **ctxs, int nr_ctxs) +{ + int i; + int err = 0; + + mutex_lock(&damon_lock); + if (nr_running_ctxs) { + mutex_unlock(&damon_lock); + return -EBUSY; + } + + for (i = 0; i < nr_ctxs; i++) { + err = __damon_start(ctxs[i]); + if (err) + break; + nr_running_ctxs++; + } + mutex_unlock(&damon_lock); + + return err; +} + +/* + * __damon_stop() - Stops monitoring of given context. + * @ctx: monitoring context + * + * Return: 0 on success, negative error code otherwise. + */ +static int __damon_stop(struct damon_ctx *ctx) +{ + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + ctx->kdamond_stop = true; + mutex_unlock(&ctx->kdamond_lock); + while (damon_kdamond_running(ctx)) + usleep_range(ctx->sample_interval, + ctx->sample_interval * 2); + return 0; + } + mutex_unlock(&ctx->kdamond_lock); + + return -EPERM; +} + +/** + * damon_stop() - Stops the monitorings for a given group of contexts. + * @ctxs: an array of the pointers for contexts to stop monitoring + * @nr_ctxs: size of @ctxs + * + * Return: 0 on success, negative error code otherwise. + */ +int damon_stop(struct damon_ctx **ctxs, int nr_ctxs) +{ + int i, err = 0; + + for (i = 0; i < nr_ctxs; i++) { + /* nr_running_ctxs is decremented in kdamond_fn */ + err = __damon_stop(ctxs[i]); + if (err) + return err; + } + + return err; +} + +/* + * damon_check_reset_time_interval() - Check if a time interval is elapsed. + * @baseline: the time to check whether the interval has elapsed since + * @interval: the time interval (microseconds) + * + * See whether the given time interval has passed since the given baseline + * time. If so, it also updates the baseline to current time for next check. + * + * Return: true if the time interval has passed, or false otherwise. + */ +static bool damon_check_reset_time_interval(struct timespec64 *baseline, + unsigned long interval) +{ + struct timespec64 now; + + ktime_get_coarse_ts64(&now); + if ((timespec64_to_ns(&now) - timespec64_to_ns(baseline)) < + interval * 1000) + return false; + *baseline = now; + return true; +} + +/* + * Check whether it is time to flush the aggregated information + */ +static bool kdamond_aggregate_interval_passed(struct damon_ctx *ctx) +{ + return damon_check_reset_time_interval(&ctx->last_aggregation, + ctx->aggr_interval); +} + +/* + * Check whether it is time to check and apply the target monitoring regions + * + * Returns true if it is. + */ +static bool kdamond_need_update_primitive(struct damon_ctx *ctx) +{ + return damon_check_reset_time_interval(&ctx->last_primitive_update, + ctx->primitive_update_interval); +} + +/* + * Check whether current monitoring should be stopped + * + * The monitoring is stopped when either the user requested to stop, or all + * monitoring targets are invalid. + * + * Returns true if need to stop current monitoring. + */ +static bool kdamond_need_stop(struct damon_ctx *ctx) +{ + bool stop; + + mutex_lock(&ctx->kdamond_lock); + stop = ctx->kdamond_stop; + mutex_unlock(&ctx->kdamond_lock); + if (stop) + return true; + + if (!ctx->primitive.target_valid) + return false; + + return !ctx->primitive.target_valid(ctx->target); +} + +static void set_kdamond_stop(struct damon_ctx *ctx) +{ + mutex_lock(&ctx->kdamond_lock); + ctx->kdamond_stop = true; + mutex_unlock(&ctx->kdamond_lock); +} + +/* + * The monitoring daemon that runs as a kernel thread + */ +static int kdamond_fn(void *data) +{ + struct damon_ctx *ctx = (struct damon_ctx *)data; + + mutex_lock(&ctx->kdamond_lock); + pr_info("kdamond (%d) starts\n", ctx->kdamond->pid); + mutex_unlock(&ctx->kdamond_lock); + + if (ctx->primitive.init) + ctx->primitive.init(ctx); + if (ctx->callback.before_start && ctx->callback.before_start(ctx)) + set_kdamond_stop(ctx); + + while (!kdamond_need_stop(ctx)) { + if (ctx->primitive.prepare_access_checks) + ctx->primitive.prepare_access_checks(ctx); + if (ctx->callback.after_sampling && + ctx->callback.after_sampling(ctx)) + set_kdamond_stop(ctx); + + usleep_range(ctx->sample_interval, ctx->sample_interval + 1); + + if (ctx->primitive.check_accesses) + ctx->primitive.check_accesses(ctx); + + if (kdamond_aggregate_interval_passed(ctx)) { + if (ctx->callback.after_aggregation && + ctx->callback.after_aggregation(ctx)) + set_kdamond_stop(ctx); + if (ctx->primitive.reset_aggregated) + ctx->primitive.reset_aggregated(ctx); + } + + if (kdamond_need_update_primitive(ctx)) { + if (ctx->primitive.update) + ctx->primitive.update(ctx); + } + } + + if (ctx->callback.before_terminate && + ctx->callback.before_terminate(ctx)) + set_kdamond_stop(ctx); + if (ctx->primitive.cleanup) + ctx->primitive.cleanup(ctx); + + pr_debug("kdamond (%d) finishes\n", ctx->kdamond->pid); + mutex_lock(&ctx->kdamond_lock); + ctx->kdamond = NULL; + mutex_unlock(&ctx->kdamond_lock); + + mutex_lock(&damon_lock); + nr_running_ctxs--; + mutex_unlock(&damon_lock); + + do_exit(0); +} From 488e19fc9134f2d6e457105ecfa02a3b143977b7 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 7 Sep 2021 19:56:32 -0700 Subject: [PATCH 047/156] UPSTREAM: mm/damon/core: implement region-based sampling To avoid the unbounded increase of the overhead, DAMON groups adjacent pages that are assumed to have the same access frequencies into a region. As long as the assumption (pages in a region have the same access frequencies) is kept, only one page in the region is required to be checked. Thus, for each ``sampling interval``, 1. the 'prepare_access_checks' primitive picks one page in each region, 2. waits for one ``sampling interval``, 3. checks whether the page is accessed meanwhile, and 4. increases the access count of the region if so. Therefore, the monitoring overhead is controllable by adjusting the number of regions. DAMON allows both the underlying primitives and user callbacks to adjust regions for the trade-off. In other words, this commit makes DAMON to use not only time-based sampling but also space-based sampling. This scheme, however, cannot preserve the quality of the output if the assumption is not guaranteed. Next commit will address this problem. Link: https://lkml.kernel.org/r/20210716081449.22187-3-sj38.park@gmail.com Signed-off-by: SeongJae Park Reviewed-by: Leonard Foerster Reviewed-by: Fernand Sieber Acked-by: Shakeel Butt Cc: Alexander Shishkin Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Fan Du Cc: Greg Kroah-Hartman Cc: Greg Thelen Cc: Ingo Molnar Cc: Joe Perches Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Marco Elver Cc: Markus Boehme Cc: Maximilian Heyne Cc: Mel Gorman Cc: Minchan Kim Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rik van Riel Cc: Shuah Khan Cc: Steven Rostedt (VMware) Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit f23b8eee1871a6db5c37f90831147de5426c40b7) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Iaecd1ea4d5123e7ffc21ca63de0fe3555bab079d --- include/linux/damon.h | 77 ++++++++++++++++++++++- mm/damon/core.c | 143 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 213 insertions(+), 7 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 2f652602b1ea..67db309ad61b 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -12,6 +12,48 @@ #include #include +/** + * struct damon_addr_range - Represents an address region of [@start, @end). + * @start: Start address of the region (inclusive). + * @end: End address of the region (exclusive). + */ +struct damon_addr_range { + unsigned long start; + unsigned long end; +}; + +/** + * struct damon_region - Represents a monitoring target region. + * @ar: The address range of the region. + * @sampling_addr: Address of the sample for the next access check. + * @nr_accesses: Access frequency of this region. + * @list: List head for siblings. + */ +struct damon_region { + struct damon_addr_range ar; + unsigned long sampling_addr; + unsigned int nr_accesses; + struct list_head list; +}; + +/** + * struct damon_target - Represents a monitoring target. + * @id: Unique identifier for this target. + * @regions_list: Head of the monitoring target regions of this target. + * @list: List head for siblings. + * + * Each monitoring context could have multiple targets. For example, a context + * for virtual memory address spaces could have multiple target processes. The + * @id of each target should be unique among the targets of the context. For + * example, in the virtual address monitoring context, it could be a pidfd or + * an address of an mm_struct. + */ +struct damon_target { + unsigned long id; + struct list_head regions_list; + struct list_head list; +}; + struct damon_ctx; /** @@ -36,7 +78,7 @@ struct damon_ctx; * * @init should initialize primitive-internal data structures. For example, * this could be used to construct proper monitoring target regions and link - * those to @damon_ctx.target. + * those to @damon_ctx.adaptive_targets. * @update should update the primitive-internal data structures. For example, * this could be used to update monitoring target regions for current status. * @prepare_access_checks should manipulate the monitoring regions to be @@ -130,7 +172,7 @@ struct damon_callback { * @primitive: Set of monitoring primitives for given use cases. * @callback: Set of callbacks for monitoring events notifications. * - * @target: Pointer to the user-defined monitoring target. + * @region_targets: Head of monitoring targets (&damon_target) list. */ struct damon_ctx { unsigned long sample_interval; @@ -149,11 +191,40 @@ struct damon_ctx { struct damon_primitive primitive; struct damon_callback callback; - void *target; + struct list_head region_targets; }; +#define damon_next_region(r) \ + (container_of(r->list.next, struct damon_region, list)) + +#define damon_prev_region(r) \ + (container_of(r->list.prev, struct damon_region, list)) + +#define damon_for_each_region(r, t) \ + list_for_each_entry(r, &t->regions_list, list) + +#define damon_for_each_region_safe(r, next, t) \ + list_for_each_entry_safe(r, next, &t->regions_list, list) + +#define damon_for_each_target(t, ctx) \ + list_for_each_entry(t, &(ctx)->region_targets, list) + +#define damon_for_each_target_safe(t, next, ctx) \ + list_for_each_entry_safe(t, next, &(ctx)->region_targets, list) + #ifdef CONFIG_DAMON +struct damon_region *damon_new_region(unsigned long start, unsigned long end); +inline void damon_insert_region(struct damon_region *r, + struct damon_region *prev, struct damon_region *next); +void damon_add_region(struct damon_region *r, struct damon_target *t); +void damon_destroy_region(struct damon_region *r); + +struct damon_target *damon_new_target(unsigned long id); +void damon_add_target(struct damon_ctx *ctx, struct damon_target *t); +void damon_free_target(struct damon_target *t); +void damon_destroy_target(struct damon_target *t); + struct damon_ctx *damon_new_ctx(void); void damon_destroy_ctx(struct damon_ctx *ctx); int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, diff --git a/mm/damon/core.c b/mm/damon/core.c index 651590bf49b1..947486a150ce 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -15,6 +15,101 @@ static DEFINE_MUTEX(damon_lock); static int nr_running_ctxs; +/* + * Construct a damon_region struct + * + * Returns the pointer to the new struct if success, or NULL otherwise + */ +struct damon_region *damon_new_region(unsigned long start, unsigned long end) +{ + struct damon_region *region; + + region = kmalloc(sizeof(*region), GFP_KERNEL); + if (!region) + return NULL; + + region->ar.start = start; + region->ar.end = end; + region->nr_accesses = 0; + INIT_LIST_HEAD(®ion->list); + + return region; +} + +/* + * Add a region between two other regions + */ +inline void damon_insert_region(struct damon_region *r, + struct damon_region *prev, struct damon_region *next) +{ + __list_add(&r->list, &prev->list, &next->list); +} + +void damon_add_region(struct damon_region *r, struct damon_target *t) +{ + list_add_tail(&r->list, &t->regions_list); +} + +static void damon_del_region(struct damon_region *r) +{ + list_del(&r->list); +} + +static void damon_free_region(struct damon_region *r) +{ + kfree(r); +} + +void damon_destroy_region(struct damon_region *r) +{ + damon_del_region(r); + damon_free_region(r); +} + +/* + * Construct a damon_target struct + * + * Returns the pointer to the new struct if success, or NULL otherwise + */ +struct damon_target *damon_new_target(unsigned long id) +{ + struct damon_target *t; + + t = kmalloc(sizeof(*t), GFP_KERNEL); + if (!t) + return NULL; + + t->id = id; + INIT_LIST_HEAD(&t->regions_list); + + return t; +} + +void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) +{ + list_add_tail(&t->list, &ctx->region_targets); +} + +static void damon_del_target(struct damon_target *t) +{ + list_del(&t->list); +} + +void damon_free_target(struct damon_target *t) +{ + struct damon_region *r, *next; + + damon_for_each_region_safe(r, next, t) + damon_free_region(r); + kfree(t); +} + +void damon_destroy_target(struct damon_target *t) +{ + damon_del_target(t); + damon_free_target(t); +} + struct damon_ctx *damon_new_ctx(void) { struct damon_ctx *ctx; @@ -32,15 +127,27 @@ struct damon_ctx *damon_new_ctx(void) mutex_init(&ctx->kdamond_lock); - ctx->target = NULL; + INIT_LIST_HEAD(&ctx->region_targets); return ctx; } +static void damon_destroy_targets(struct damon_ctx *ctx) +{ + struct damon_target *t, *next_t; + + if (ctx->primitive.cleanup) { + ctx->primitive.cleanup(ctx); + return; + } + + damon_for_each_target_safe(t, next_t, ctx) + damon_destroy_target(t); +} + void damon_destroy_ctx(struct damon_ctx *ctx) { - if (ctx->primitive.cleanup) - ctx->primitive.cleanup(ctx); + damon_destroy_targets(ctx); kfree(ctx); } @@ -217,6 +324,21 @@ static bool kdamond_aggregate_interval_passed(struct damon_ctx *ctx) ctx->aggr_interval); } +/* + * Reset the aggregated monitoring results ('nr_accesses' of each region). + */ +static void kdamond_reset_aggregated(struct damon_ctx *c) +{ + struct damon_target *t; + + damon_for_each_target(t, c) { + struct damon_region *r; + + damon_for_each_region(r, t) + r->nr_accesses = 0; + } +} + /* * Check whether it is time to check and apply the target monitoring regions * @@ -238,6 +360,7 @@ static bool kdamond_need_update_primitive(struct damon_ctx *ctx) */ static bool kdamond_need_stop(struct damon_ctx *ctx) { + struct damon_target *t; bool stop; mutex_lock(&ctx->kdamond_lock); @@ -249,7 +372,12 @@ static bool kdamond_need_stop(struct damon_ctx *ctx) if (!ctx->primitive.target_valid) return false; - return !ctx->primitive.target_valid(ctx->target); + damon_for_each_target(t, ctx) { + if (ctx->primitive.target_valid(t)) + return false; + } + + return true; } static void set_kdamond_stop(struct damon_ctx *ctx) @@ -265,6 +393,8 @@ static void set_kdamond_stop(struct damon_ctx *ctx) static int kdamond_fn(void *data) { struct damon_ctx *ctx = (struct damon_ctx *)data; + struct damon_target *t; + struct damon_region *r, *next; mutex_lock(&ctx->kdamond_lock); pr_info("kdamond (%d) starts\n", ctx->kdamond->pid); @@ -291,6 +421,7 @@ static int kdamond_fn(void *data) if (ctx->callback.after_aggregation && ctx->callback.after_aggregation(ctx)) set_kdamond_stop(ctx); + kdamond_reset_aggregated(ctx); if (ctx->primitive.reset_aggregated) ctx->primitive.reset_aggregated(ctx); } @@ -300,6 +431,10 @@ static int kdamond_fn(void *data) ctx->primitive.update(ctx); } } + damon_for_each_target(t, ctx) { + damon_for_each_region_safe(r, next, t) + damon_destroy_region(r); + } if (ctx->callback.before_terminate && ctx->callback.before_terminate(ctx)) From 0f1bc2a61d3ff9a3ab7e240b67bf6e36a9fbab87 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 7 Sep 2021 19:56:36 -0700 Subject: [PATCH 048/156] UPSTREAM: mm/damon: adaptively adjust regions Even somehow the initial monitoring target regions are well constructed to fulfill the assumption (pages in same region have similar access frequencies), the data access pattern can be dynamically changed. This will result in low monitoring quality. To keep the assumption as much as possible, DAMON adaptively merges and splits each region based on their access frequency. For each ``aggregation interval``, it compares the access frequencies of adjacent regions and merges those if the frequency difference is small. Then, after it reports and clears the aggregated access frequency of each region, it splits each region into two or three regions if the total number of regions will not exceed the user-specified maximum number of regions after the split. In this way, DAMON provides its best-effort quality and minimal overhead while keeping the upper-bound overhead that users set. Link: https://lkml.kernel.org/r/20210716081449.22187-4-sj38.park@gmail.com Signed-off-by: SeongJae Park Reviewed-by: Leonard Foerster Reviewed-by: Fernand Sieber Acked-by: Shakeel Butt Cc: Alexander Shishkin Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Fan Du Cc: Greg Kroah-Hartman Cc: Greg Thelen Cc: Ingo Molnar Cc: Joe Perches Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Marco Elver Cc: Markus Boehme Cc: Maximilian Heyne Cc: Mel Gorman Cc: Minchan Kim Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rik van Riel Cc: Shuah Khan Cc: Steven Rostedt (VMware) Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit b9a6ac4e4ede4172d165c133398b93e3233b0ba7) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I330416cf16f4e72fd8f26e01d3bd425738216c70 --- include/linux/damon.h | 30 ++++-- mm/damon/core.c | 224 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 237 insertions(+), 17 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 67db309ad61b..ce2a84b26cd7 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -12,6 +12,9 @@ #include #include +/* Minimal region size. Every damon_region is aligned by this. */ +#define DAMON_MIN_REGION PAGE_SIZE + /** * struct damon_addr_range - Represents an address region of [@start, @end). * @start: Start address of the region (inclusive). @@ -39,6 +42,7 @@ struct damon_region { /** * struct damon_target - Represents a monitoring target. * @id: Unique identifier for this target. + * @nr_regions: Number of monitoring target regions of this target. * @regions_list: Head of the monitoring target regions of this target. * @list: List head for siblings. * @@ -50,6 +54,7 @@ struct damon_region { */ struct damon_target { unsigned long id; + unsigned int nr_regions; struct list_head regions_list; struct list_head list; }; @@ -85,6 +90,8 @@ struct damon_ctx; * prepared for the next access check. * @check_accesses should check the accesses to each region that made after the * last preparation and update the number of observed accesses of each region. + * It should also return max number of observed accesses that made as a result + * of its update. The value will be used for regions adjustment threshold. * @reset_aggregated should reset the access monitoring results that aggregated * by @check_accesses. * @target_valid should check whether the target is still valid for the @@ -95,7 +102,7 @@ struct damon_primitive { void (*init)(struct damon_ctx *context); void (*update)(struct damon_ctx *context); void (*prepare_access_checks)(struct damon_ctx *context); - void (*check_accesses)(struct damon_ctx *context); + unsigned int (*check_accesses)(struct damon_ctx *context); void (*reset_aggregated)(struct damon_ctx *context); bool (*target_valid)(void *target); void (*cleanup)(struct damon_ctx *context); @@ -172,7 +179,9 @@ struct damon_callback { * @primitive: Set of monitoring primitives for given use cases. * @callback: Set of callbacks for monitoring events notifications. * - * @region_targets: Head of monitoring targets (&damon_target) list. + * @min_nr_regions: The minimum number of adaptive monitoring regions. + * @max_nr_regions: The maximum number of adaptive monitoring regions. + * @adaptive_targets: Head of monitoring targets (&damon_target) list. */ struct damon_ctx { unsigned long sample_interval; @@ -191,7 +200,9 @@ struct damon_ctx { struct damon_primitive primitive; struct damon_callback callback; - struct list_head region_targets; + unsigned long min_nr_regions; + unsigned long max_nr_regions; + struct list_head adaptive_targets; }; #define damon_next_region(r) \ @@ -207,28 +218,31 @@ struct damon_ctx { list_for_each_entry_safe(r, next, &t->regions_list, list) #define damon_for_each_target(t, ctx) \ - list_for_each_entry(t, &(ctx)->region_targets, list) + list_for_each_entry(t, &(ctx)->adaptive_targets, list) #define damon_for_each_target_safe(t, next, ctx) \ - list_for_each_entry_safe(t, next, &(ctx)->region_targets, list) + list_for_each_entry_safe(t, next, &(ctx)->adaptive_targets, list) #ifdef CONFIG_DAMON struct damon_region *damon_new_region(unsigned long start, unsigned long end); inline void damon_insert_region(struct damon_region *r, - struct damon_region *prev, struct damon_region *next); + struct damon_region *prev, struct damon_region *next, + struct damon_target *t); void damon_add_region(struct damon_region *r, struct damon_target *t); -void damon_destroy_region(struct damon_region *r); +void damon_destroy_region(struct damon_region *r, struct damon_target *t); struct damon_target *damon_new_target(unsigned long id); void damon_add_target(struct damon_ctx *ctx, struct damon_target *t); void damon_free_target(struct damon_target *t); void damon_destroy_target(struct damon_target *t); +unsigned int damon_nr_regions(struct damon_target *t); struct damon_ctx *damon_new_ctx(void); void damon_destroy_ctx(struct damon_ctx *ctx); int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, - unsigned long aggr_int, unsigned long primitive_upd_int); + unsigned long aggr_int, unsigned long primitive_upd_int, + unsigned long min_nr_reg, unsigned long max_nr_reg); int damon_start(struct damon_ctx **ctxs, int nr_ctxs); int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); diff --git a/mm/damon/core.c b/mm/damon/core.c index 947486a150ce..28a2c78914fa 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -10,8 +10,12 @@ #include #include #include +#include #include +/* Get a random number in [l, r) */ +#define damon_rand(l, r) (l + prandom_u32_max(r - l)) + static DEFINE_MUTEX(damon_lock); static int nr_running_ctxs; @@ -40,19 +44,23 @@ struct damon_region *damon_new_region(unsigned long start, unsigned long end) * Add a region between two other regions */ inline void damon_insert_region(struct damon_region *r, - struct damon_region *prev, struct damon_region *next) + struct damon_region *prev, struct damon_region *next, + struct damon_target *t) { __list_add(&r->list, &prev->list, &next->list); + t->nr_regions++; } void damon_add_region(struct damon_region *r, struct damon_target *t) { list_add_tail(&r->list, &t->regions_list); + t->nr_regions++; } -static void damon_del_region(struct damon_region *r) +static void damon_del_region(struct damon_region *r, struct damon_target *t) { list_del(&r->list); + t->nr_regions--; } static void damon_free_region(struct damon_region *r) @@ -60,9 +68,9 @@ static void damon_free_region(struct damon_region *r) kfree(r); } -void damon_destroy_region(struct damon_region *r) +void damon_destroy_region(struct damon_region *r, struct damon_target *t) { - damon_del_region(r); + damon_del_region(r, t); damon_free_region(r); } @@ -80,6 +88,7 @@ struct damon_target *damon_new_target(unsigned long id) return NULL; t->id = id; + t->nr_regions = 0; INIT_LIST_HEAD(&t->regions_list); return t; @@ -87,7 +96,7 @@ struct damon_target *damon_new_target(unsigned long id) void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) { - list_add_tail(&t->list, &ctx->region_targets); + list_add_tail(&t->list, &ctx->adaptive_targets); } static void damon_del_target(struct damon_target *t) @@ -110,6 +119,11 @@ void damon_destroy_target(struct damon_target *t) damon_free_target(t); } +unsigned int damon_nr_regions(struct damon_target *t) +{ + return t->nr_regions; +} + struct damon_ctx *damon_new_ctx(void) { struct damon_ctx *ctx; @@ -127,7 +141,10 @@ struct damon_ctx *damon_new_ctx(void) mutex_init(&ctx->kdamond_lock); - INIT_LIST_HEAD(&ctx->region_targets); + ctx->min_nr_regions = 10; + ctx->max_nr_regions = 1000; + + INIT_LIST_HEAD(&ctx->adaptive_targets); return ctx; } @@ -157,6 +174,8 @@ void damon_destroy_ctx(struct damon_ctx *ctx) * @sample_int: time interval between samplings * @aggr_int: time interval between aggregations * @primitive_upd_int: time interval between monitoring primitive updates + * @min_nr_reg: minimal number of regions + * @max_nr_reg: maximum number of regions * * This function should not be called while the kdamond is running. * Every time interval is in micro-seconds. @@ -164,15 +183,49 @@ void damon_destroy_ctx(struct damon_ctx *ctx) * Return: 0 on success, negative error code otherwise. */ int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, - unsigned long aggr_int, unsigned long primitive_upd_int) + unsigned long aggr_int, unsigned long primitive_upd_int, + unsigned long min_nr_reg, unsigned long max_nr_reg) { + if (min_nr_reg < 3) { + pr_err("min_nr_regions (%lu) must be at least 3\n", + min_nr_reg); + return -EINVAL; + } + if (min_nr_reg > max_nr_reg) { + pr_err("invalid nr_regions. min (%lu) > max (%lu)\n", + min_nr_reg, max_nr_reg); + return -EINVAL; + } + ctx->sample_interval = sample_int; ctx->aggr_interval = aggr_int; ctx->primitive_update_interval = primitive_upd_int; + ctx->min_nr_regions = min_nr_reg; + ctx->max_nr_regions = max_nr_reg; return 0; } +/* Returns the size upper limit for each monitoring region */ +static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) +{ + struct damon_target *t; + struct damon_region *r; + unsigned long sz = 0; + + damon_for_each_target(t, ctx) { + damon_for_each_region(r, t) + sz += r->ar.end - r->ar.start; + } + + if (ctx->min_nr_regions) + sz /= ctx->min_nr_regions; + if (sz < DAMON_MIN_REGION) + sz = DAMON_MIN_REGION; + + return sz; +} + static bool damon_kdamond_running(struct damon_ctx *ctx) { bool running; @@ -339,6 +392,150 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) } } +#define sz_damon_region(r) (r->ar.end - r->ar.start) + +/* + * Merge two adjacent regions into one region + */ +static void damon_merge_two_regions(struct damon_target *t, + struct damon_region *l, struct damon_region *r) +{ + unsigned long sz_l = sz_damon_region(l), sz_r = sz_damon_region(r); + + l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / + (sz_l + sz_r); + l->ar.end = r->ar.end; + damon_destroy_region(r, t); +} + +#define diff_of(a, b) (a > b ? a - b : b - a) + +/* + * Merge adjacent regions having similar access frequencies + * + * t target affected by this merge operation + * thres '->nr_accesses' diff threshold for the merge + * sz_limit size upper limit of each region + */ +static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, + unsigned long sz_limit) +{ + struct damon_region *r, *prev = NULL, *next; + + damon_for_each_region_safe(r, next, t) { + if (prev && prev->ar.end == r->ar.start && + diff_of(prev->nr_accesses, r->nr_accesses) <= thres && + sz_damon_region(prev) + sz_damon_region(r) <= sz_limit) + damon_merge_two_regions(t, prev, r); + else + prev = r; + } +} + +/* + * Merge adjacent regions having similar access frequencies + * + * threshold '->nr_accesses' diff threshold for the merge + * sz_limit size upper limit of each region + * + * This function merges monitoring target regions which are adjacent and their + * access frequencies are similar. This is for minimizing the monitoring + * overhead under the dynamically changeable access pattern. If a merge was + * unnecessarily made, later 'kdamond_split_regions()' will revert it. + */ +static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, + unsigned long sz_limit) +{ + struct damon_target *t; + + damon_for_each_target(t, c) + damon_merge_regions_of(t, threshold, sz_limit); +} + +/* + * Split a region in two + * + * r the region to be split + * sz_r size of the first sub-region that will be made + */ +static void damon_split_region_at(struct damon_ctx *ctx, + struct damon_target *t, struct damon_region *r, + unsigned long sz_r) +{ + struct damon_region *new; + + new = damon_new_region(r->ar.start + sz_r, r->ar.end); + if (!new) + return; + + r->ar.end = new->ar.start; + + damon_insert_region(new, r, damon_next_region(r), t); +} + +/* Split every region in the given target into 'nr_subs' regions */ +static void damon_split_regions_of(struct damon_ctx *ctx, + struct damon_target *t, int nr_subs) +{ + struct damon_region *r, *next; + unsigned long sz_region, sz_sub = 0; + int i; + + damon_for_each_region_safe(r, next, t) { + sz_region = r->ar.end - r->ar.start; + + for (i = 0; i < nr_subs - 1 && + sz_region > 2 * DAMON_MIN_REGION; i++) { + /* + * Randomly select size of left sub-region to be at + * least 10 percent and at most 90% of original region + */ + sz_sub = ALIGN_DOWN(damon_rand(1, 10) * + sz_region / 10, DAMON_MIN_REGION); + /* Do not allow blank region */ + if (sz_sub == 0 || sz_sub >= sz_region) + continue; + + damon_split_region_at(ctx, t, r, sz_sub); + sz_region = sz_sub; + } + } +} + +/* + * Split every target region into randomly-sized small regions + * + * This function splits every target region into random-sized small regions if + * current total number of the regions is equal or smaller than half of the + * user-specified maximum number of regions. This is for maximizing the + * monitoring accuracy under the dynamically changeable access patterns. If a + * split was unnecessarily made, later 'kdamond_merge_regions()' will revert + * it. + */ +static void kdamond_split_regions(struct damon_ctx *ctx) +{ + struct damon_target *t; + unsigned int nr_regions = 0; + static unsigned int last_nr_regions; + int nr_subregions = 2; + + damon_for_each_target(t, ctx) + nr_regions += damon_nr_regions(t); + + if (nr_regions > ctx->max_nr_regions / 2) + return; + + /* Maybe the middle of the region has different access frequency */ + if (last_nr_regions == nr_regions && + nr_regions < ctx->max_nr_regions / 3) + nr_subregions = 3; + + damon_for_each_target(t, ctx) + damon_split_regions_of(ctx, t, nr_subregions); + + last_nr_regions = nr_regions; +} + /* * Check whether it is time to check and apply the target monitoring regions * @@ -395,6 +592,8 @@ static int kdamond_fn(void *data) struct damon_ctx *ctx = (struct damon_ctx *)data; struct damon_target *t; struct damon_region *r, *next; + unsigned int max_nr_accesses = 0; + unsigned long sz_limit = 0; mutex_lock(&ctx->kdamond_lock); pr_info("kdamond (%d) starts\n", ctx->kdamond->pid); @@ -405,6 +604,8 @@ static int kdamond_fn(void *data) if (ctx->callback.before_start && ctx->callback.before_start(ctx)) set_kdamond_stop(ctx); + sz_limit = damon_region_sz_limit(ctx); + while (!kdamond_need_stop(ctx)) { if (ctx->primitive.prepare_access_checks) ctx->primitive.prepare_access_checks(ctx); @@ -415,13 +616,17 @@ static int kdamond_fn(void *data) usleep_range(ctx->sample_interval, ctx->sample_interval + 1); if (ctx->primitive.check_accesses) - ctx->primitive.check_accesses(ctx); + max_nr_accesses = ctx->primitive.check_accesses(ctx); if (kdamond_aggregate_interval_passed(ctx)) { + kdamond_merge_regions(ctx, + max_nr_accesses / 10, + sz_limit); if (ctx->callback.after_aggregation && ctx->callback.after_aggregation(ctx)) set_kdamond_stop(ctx); kdamond_reset_aggregated(ctx); + kdamond_split_regions(ctx); if (ctx->primitive.reset_aggregated) ctx->primitive.reset_aggregated(ctx); } @@ -429,11 +634,12 @@ static int kdamond_fn(void *data) if (kdamond_need_update_primitive(ctx)) { if (ctx->primitive.update) ctx->primitive.update(ctx); + sz_limit = damon_region_sz_limit(ctx); } } damon_for_each_target(t, ctx) { damon_for_each_region_safe(r, next, t) - damon_destroy_region(r); + damon_destroy_region(r, t); } if (ctx->callback.before_terminate && From 75e13bad972b7c6ac8d2b47658e07960286c4b67 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 7 Sep 2021 19:56:40 -0700 Subject: [PATCH 049/156] UPSTREAM: mm/idle_page_tracking: make PG_idle reusable PG_idle and PG_young allow the two PTE Accessed bit users, Idle Page Tracking and the reclaim logic concurrently work while not interfering with each other. That is, when they need to clear the Accessed bit, they set PG_young to represent the previous state of the bit, respectively. And when they need to read the bit, if the bit is cleared, they further read the PG_young to know whether the other has cleared the bit meanwhile or not. For yet another user of the PTE Accessed bit, we could add another page flag, or extend the mechanism to use the flags. For the DAMON usecase, however, we don't need to do that just yet. IDLE_PAGE_TRACKING and DAMON are mutually exclusive, so there's only ever going to be one user of the current set of flags. In this commit, we split out the CONFIG options to allow for the use of PG_young and PG_idle outside of idle page tracking. In the next commit, DAMON's reference implementation of the virtual memory address space monitoring primitives will use it. [sjpark@amazon.de: set PAGE_EXTENSION for non-64BIT] Link: https://lkml.kernel.org/r/20210806095153.6444-1-sj38.park@gmail.com [akpm@linux-foundation.org: tweak Kconfig text] [sjpark@amazon.de: hide PAGE_IDLE_FLAG from users] Link: https://lkml.kernel.org/r/20210813081238.34705-1-sj38.park@gmail.com Link: https://lkml.kernel.org/r/20210716081449.22187-5-sj38.park@gmail.com Signed-off-by: SeongJae Park Reviewed-by: Shakeel Butt Reviewed-by: Fernand Sieber Cc: Alexander Shishkin Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Fan Du Cc: Greg Kroah-Hartman Cc: Greg Thelen Cc: Ingo Molnar Cc: Joe Perches Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Maximilian Heyne Cc: Mel Gorman Cc: Minchan Kim Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rik van Riel Cc: Shuah Khan Cc: Steven Rostedt (VMware) Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 1c676e0d9b1a59b98885b24a0e16a81fe4cc8301) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I4b2c3087b069b0306ab8743402fabcd195d63e54 --- include/linux/page-flags.h | 4 ++-- include/linux/page_ext.h | 2 +- include/linux/page_idle.h | 6 +++--- include/trace/events/mmflags.h | 2 +- mm/Kconfig | 10 +++++++++- mm/page_ext.c | 12 +++++++++++- mm/page_idle.c | 10 ---------- 7 files changed, 27 insertions(+), 19 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index a2738944340c..dc636f162194 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -132,7 +132,7 @@ enum pageflags { #ifdef CONFIG_MEMORY_FAILURE PG_hwpoison, /* hardware poisoned page. Don't touch */ #endif -#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT) +#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) PG_young, PG_idle, #endif @@ -440,7 +440,7 @@ PAGEFLAG_FALSE(HWPoison) #define __PG_HWPOISON 0 #endif -#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT) +#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) TESTPAGEFLAG(Young, young, PF_ANY) SETPAGEFLAG(Young, young, PF_ANY) TESTCLEARFLAG(Young, young, PF_ANY) diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index cd45c1927d90..8e58a1b73771 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -25,7 +25,7 @@ enum page_ext_flags { /* page migration failed */ PAGE_EXT_PINNER_MIGRATION_FAILED, #endif -#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT) +#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) PAGE_EXT_YOUNG, PAGE_EXT_IDLE, #endif diff --git a/include/linux/page_idle.h b/include/linux/page_idle.h index 1e894d34bdce..d8a6aecf99cb 100644 --- a/include/linux/page_idle.h +++ b/include/linux/page_idle.h @@ -6,7 +6,7 @@ #include #include -#ifdef CONFIG_IDLE_PAGE_TRACKING +#ifdef CONFIG_PAGE_IDLE_FLAG #ifdef CONFIG_64BIT static inline bool page_is_young(struct page *page) @@ -106,7 +106,7 @@ static inline void clear_page_idle(struct page *page) } #endif /* CONFIG_64BIT */ -#else /* !CONFIG_IDLE_PAGE_TRACKING */ +#else /* !CONFIG_PAGE_IDLE_FLAG */ static inline bool page_is_young(struct page *page) { @@ -135,6 +135,6 @@ static inline void clear_page_idle(struct page *page) { } -#endif /* CONFIG_IDLE_PAGE_TRACKING */ +#endif /* CONFIG_PAGE_IDLE_FLAG */ #endif /* _LINUX_MM_PAGE_IDLE_H */ diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index 390270e00a1d..d428f0137c49 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -73,7 +73,7 @@ #define IF_HAVE_PG_HWPOISON(flag,string) #endif -#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT) +#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) #define IF_HAVE_PG_IDLE(flag,string) ,{1UL << flag, string} #else #define IF_HAVE_PG_IDLE(flag,string) diff --git a/mm/Kconfig b/mm/Kconfig index 5f5a66efcaba..6a020a0d7bc5 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -756,10 +756,18 @@ config DEFERRED_STRUCT_PAGE_INIT lifetime of the system until these kthreads finish the initialisation. +config PAGE_IDLE_FLAG + bool + select PAGE_EXTENSION if !64BIT + help + This adds PG_idle and PG_young flags to 'struct page'. PTE Accessed + bit writers can set the state of the bit in the flags so that PTE + Accessed bit readers may avoid disturbance. + config IDLE_PAGE_TRACKING bool "Enable idle page tracking" depends on SYSFS && MMU - select PAGE_EXTENSION if !64BIT + select PAGE_IDLE_FLAG help This feature allows to estimate the amount of user pages that have not been touched during a given period of time. This information can diff --git a/mm/page_ext.c b/mm/page_ext.c index 7e44726b3549..e5e31ff1adba 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -58,11 +58,21 @@ * can utilize this callback to initialize the state of it correctly. */ +#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) +static bool need_page_idle(void) +{ + return true; +} +struct page_ext_operations page_idle_ops = { + .need = need_page_idle, +}; +#endif + static struct page_ext_operations *page_ext_ops[] = { #ifdef CONFIG_PAGE_OWNER &page_owner_ops, #endif -#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT) +#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) &page_idle_ops, #endif #ifdef CONFIG_PAGE_PINNER diff --git a/mm/page_idle.c b/mm/page_idle.c index 057c61df12db..144fb4ed961d 100644 --- a/mm/page_idle.c +++ b/mm/page_idle.c @@ -211,16 +211,6 @@ static const struct attribute_group page_idle_attr_group = { .name = "page_idle", }; -#ifndef CONFIG_64BIT -static bool need_page_idle(void) -{ - return true; -} -struct page_ext_operations page_idle_ops = { - .need = need_page_idle, -}; -#endif - static int __init page_idle_init(void) { int err; From 2afdd88030499b5ce445be2652890bce97fb6b8e Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 7 Sep 2021 19:56:44 -0700 Subject: [PATCH 050/156] UPSTREAM: mm/damon: implement primitives for the virtual memory address spaces This commit introduces a reference implementation of the address space specific low level primitives for the virtual address space, so that users of DAMON can easily monitor the data accesses on virtual address spaces of specific processes by simply configuring the implementation to be used by DAMON. The low level primitives for the fundamental access monitoring are defined in two parts: 1. Identification of the monitoring target address range for the address space. 2. Access check of specific address range in the target space. The reference implementation for the virtual address space does the works as below. PTE Accessed-bit Based Access Check ----------------------------------- The implementation uses PTE Accessed-bit for basic access checks. That is, it clears the bit for the next sampling target page and checks whether it is set again after one sampling period. This could disturb the reclaim logic. DAMON uses ``PG_idle`` and ``PG_young`` page flags to solve the conflict, as Idle page tracking does. VMA-based Target Address Range Construction ------------------------------------------- Only small parts in the super-huge virtual address space of the processes are mapped to physical memory and accessed. Thus, tracking the unmapped address regions is just wasteful. However, because DAMON can deal with some level of noise using the adaptive regions adjustment mechanism, tracking every mapping is not strictly required but could even incur a high overhead in some cases. That said, too huge unmapped areas inside the monitoring target should be removed to not take the time for the adaptive mechanism. For the reason, this implementation converts the complex mappings to three distinct regions that cover every mapped area of the address space. Also, the two gaps between the three regions are the two biggest unmapped areas in the given address space. The two biggest unmapped areas would be the gap between the heap and the uppermost mmap()-ed region, and the gap between the lowermost mmap()-ed region and the stack in most of the cases. Because these gaps are exceptionally huge in usual address spaces, excluding these will be sufficient to make a reasonable trade-off. Below shows this in detail:: (small mmap()-ed regions and munmap()-ed regions) [akpm@linux-foundation.org: mm/damon/vaddr.c needs highmem.h for kunmap_atomic()] [sjpark@amazon.de: remove unnecessary PAGE_EXTENSION setup] Link: https://lkml.kernel.org/r/20210806095153.6444-2-sj38.park@gmail.com [sjpark@amazon.de: safely walk page table] Link: https://lkml.kernel.org/r/20210831161800.29419-1-sj38.park@gmail.com Link: https://lkml.kernel.org/r/20210716081449.22187-6-sj38.park@gmail.com Signed-off-by: SeongJae Park Reviewed-by: Leonard Foerster Reviewed-by: Fernand Sieber Acked-by: Shakeel Butt Cc: Alexander Shishkin Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Fan Du Cc: Greg Kroah-Hartman Cc: Greg Thelen Cc: Ingo Molnar Cc: Joe Perches Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Marco Elver Cc: Markus Boehme Cc: Maximilian Heyne Cc: Mel Gorman Cc: Minchan Kim Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rik van Riel Cc: Shuah Khan Cc: Steven Rostedt (VMware) Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 3f49584b262cf8f42b25f4c1ad9f5bfd3bdc1bca) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ib7d134932b1dd20cd926968991654d71717c197d --- include/linux/damon.h | 13 + mm/damon/Kconfig | 8 + mm/damon/Makefile | 1 + mm/damon/vaddr.c | 665 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 687 insertions(+) create mode 100644 mm/damon/vaddr.c diff --git a/include/linux/damon.h b/include/linux/damon.h index ce2a84b26cd7..edb350e52b93 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -249,4 +249,17 @@ int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); #endif /* CONFIG_DAMON */ +#ifdef CONFIG_DAMON_VADDR + +/* Monitoring primitives for virtual memory address spaces */ +void damon_va_init(struct damon_ctx *ctx); +void damon_va_update(struct damon_ctx *ctx); +void damon_va_prepare_access_checks(struct damon_ctx *ctx); +unsigned int damon_va_check_accesses(struct damon_ctx *ctx); +bool damon_va_target_valid(void *t); +void damon_va_cleanup(struct damon_ctx *ctx); +void damon_va_set_primitives(struct damon_ctx *ctx); + +#endif /* CONFIG_DAMON_VADDR */ + #endif /* _DAMON_H */ diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig index d00e99ac1a15..5cbb5db54158 100644 --- a/mm/damon/Kconfig +++ b/mm/damon/Kconfig @@ -12,4 +12,12 @@ config DAMON See https://damonitor.github.io/doc/html/latest-damon/index.html for more information. +config DAMON_VADDR + bool "Data access monitoring primitives for virtual address spaces" + depends on DAMON && MMU + select PAGE_IDLE_FLAG + help + This builds the default data access monitoring primitives for DAMON + that works for virtual address spaces. + endmenu diff --git a/mm/damon/Makefile b/mm/damon/Makefile index 4fd2edb4becf..6ebbd08aed67 100644 --- a/mm/damon/Makefile +++ b/mm/damon/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DAMON) := core.o +obj-$(CONFIG_DAMON_VADDR) += vaddr.o diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c new file mode 100644 index 000000000000..897aa8cf96c8 --- /dev/null +++ b/mm/damon/vaddr.c @@ -0,0 +1,665 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DAMON Primitives for Virtual Address Spaces + * + * Author: SeongJae Park + */ + +#define pr_fmt(fmt) "damon-va: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Get a random number in [l, r) */ +#define damon_rand(l, r) (l + prandom_u32_max(r - l)) + +/* + * 't->id' should be the pointer to the relevant 'struct pid' having reference + * count. Caller must put the returned task, unless it is NULL. + */ +#define damon_get_task_struct(t) \ + (get_pid_task((struct pid *)t->id, PIDTYPE_PID)) + +/* + * Get the mm_struct of the given target + * + * Caller _must_ put the mm_struct after use, unless it is NULL. + * + * Returns the mm_struct of the target on success, NULL on failure + */ +static struct mm_struct *damon_get_mm(struct damon_target *t) +{ + struct task_struct *task; + struct mm_struct *mm; + + task = damon_get_task_struct(t); + if (!task) + return NULL; + + mm = get_task_mm(task); + put_task_struct(task); + return mm; +} + +/* + * Functions for the initial monitoring target regions construction + */ + +/* + * Size-evenly split a region into 'nr_pieces' small regions + * + * Returns 0 on success, or negative error code otherwise. + */ +static int damon_va_evenly_split_region(struct damon_target *t, + struct damon_region *r, unsigned int nr_pieces) +{ + unsigned long sz_orig, sz_piece, orig_end; + struct damon_region *n = NULL, *next; + unsigned long start; + + if (!r || !nr_pieces) + return -EINVAL; + + orig_end = r->ar.end; + sz_orig = r->ar.end - r->ar.start; + sz_piece = ALIGN_DOWN(sz_orig / nr_pieces, DAMON_MIN_REGION); + + if (!sz_piece) + return -EINVAL; + + r->ar.end = r->ar.start + sz_piece; + next = damon_next_region(r); + for (start = r->ar.end; start + sz_piece <= orig_end; + start += sz_piece) { + n = damon_new_region(start, start + sz_piece); + if (!n) + return -ENOMEM; + damon_insert_region(n, r, next, t); + r = n; + } + /* complement last region for possible rounding error */ + if (n) + n->ar.end = orig_end; + + return 0; +} + +static unsigned long sz_range(struct damon_addr_range *r) +{ + return r->end - r->start; +} + +static void swap_ranges(struct damon_addr_range *r1, + struct damon_addr_range *r2) +{ + struct damon_addr_range tmp; + + tmp = *r1; + *r1 = *r2; + *r2 = tmp; +} + +/* + * Find three regions separated by two biggest unmapped regions + * + * vma the head vma of the target address space + * regions an array of three address ranges that results will be saved + * + * This function receives an address space and finds three regions in it which + * separated by the two biggest unmapped regions in the space. Please refer to + * below comments of '__damon_va_init_regions()' function to know why this is + * necessary. + * + * Returns 0 if success, or negative error code otherwise. + */ +static int __damon_va_three_regions(struct vm_area_struct *vma, + struct damon_addr_range regions[3]) +{ + struct damon_addr_range gap = {0}, first_gap = {0}, second_gap = {0}; + struct vm_area_struct *last_vma = NULL; + unsigned long start = 0; + struct rb_root rbroot; + + /* Find two biggest gaps so that first_gap > second_gap > others */ + for (; vma; vma = vma->vm_next) { + if (!last_vma) { + start = vma->vm_start; + goto next; + } + + if (vma->rb_subtree_gap <= sz_range(&second_gap)) { + rbroot.rb_node = &vma->vm_rb; + vma = rb_entry(rb_last(&rbroot), + struct vm_area_struct, vm_rb); + goto next; + } + + gap.start = last_vma->vm_end; + gap.end = vma->vm_start; + if (sz_range(&gap) > sz_range(&second_gap)) { + swap_ranges(&gap, &second_gap); + if (sz_range(&second_gap) > sz_range(&first_gap)) + swap_ranges(&second_gap, &first_gap); + } +next: + last_vma = vma; + } + + if (!sz_range(&second_gap) || !sz_range(&first_gap)) + return -EINVAL; + + /* Sort the two biggest gaps by address */ + if (first_gap.start > second_gap.start) + swap_ranges(&first_gap, &second_gap); + + /* Store the result */ + regions[0].start = ALIGN(start, DAMON_MIN_REGION); + regions[0].end = ALIGN(first_gap.start, DAMON_MIN_REGION); + regions[1].start = ALIGN(first_gap.end, DAMON_MIN_REGION); + regions[1].end = ALIGN(second_gap.start, DAMON_MIN_REGION); + regions[2].start = ALIGN(second_gap.end, DAMON_MIN_REGION); + regions[2].end = ALIGN(last_vma->vm_end, DAMON_MIN_REGION); + + return 0; +} + +/* + * Get the three regions in the given target (task) + * + * Returns 0 on success, negative error code otherwise. + */ +static int damon_va_three_regions(struct damon_target *t, + struct damon_addr_range regions[3]) +{ + struct mm_struct *mm; + int rc; + + mm = damon_get_mm(t); + if (!mm) + return -EINVAL; + + mmap_read_lock(mm); + rc = __damon_va_three_regions(mm->mmap, regions); + mmap_read_unlock(mm); + + mmput(mm); + return rc; +} + +/* + * Initialize the monitoring target regions for the given target (task) + * + * t the given target + * + * Because only a number of small portions of the entire address space + * is actually mapped to the memory and accessed, monitoring the unmapped + * regions is wasteful. That said, because we can deal with small noises, + * tracking every mapping is not strictly required but could even incur a high + * overhead if the mapping frequently changes or the number of mappings is + * high. The adaptive regions adjustment mechanism will further help to deal + * with the noise by simply identifying the unmapped areas as a region that + * has no access. Moreover, applying the real mappings that would have many + * unmapped areas inside will make the adaptive mechanism quite complex. That + * said, too huge unmapped areas inside the monitoring target should be removed + * to not take the time for the adaptive mechanism. + * + * For the reason, we convert the complex mappings to three distinct regions + * that cover every mapped area of the address space. Also the two gaps + * between the three regions are the two biggest unmapped areas in the given + * address space. In detail, this function first identifies the start and the + * end of the mappings and the two biggest unmapped areas of the address space. + * Then, it constructs the three regions as below: + * + * [mappings[0]->start, big_two_unmapped_areas[0]->start) + * [big_two_unmapped_areas[0]->end, big_two_unmapped_areas[1]->start) + * [big_two_unmapped_areas[1]->end, mappings[nr_mappings - 1]->end) + * + * As usual memory map of processes is as below, the gap between the heap and + * the uppermost mmap()-ed region, and the gap between the lowermost mmap()-ed + * region and the stack will be two biggest unmapped regions. Because these + * gaps are exceptionally huge areas in usual address space, excluding these + * two biggest unmapped regions will be sufficient to make a trade-off. + * + * + * + * + * (other mmap()-ed regions and small unmapped regions) + * + * + * + */ +static void __damon_va_init_regions(struct damon_ctx *ctx, + struct damon_target *t) +{ + struct damon_region *r; + struct damon_addr_range regions[3]; + unsigned long sz = 0, nr_pieces; + int i; + + if (damon_va_three_regions(t, regions)) { + pr_err("Failed to get three regions of target %lu\n", t->id); + return; + } + + for (i = 0; i < 3; i++) + sz += regions[i].end - regions[i].start; + if (ctx->min_nr_regions) + sz /= ctx->min_nr_regions; + if (sz < DAMON_MIN_REGION) + sz = DAMON_MIN_REGION; + + /* Set the initial three regions of the target */ + for (i = 0; i < 3; i++) { + r = damon_new_region(regions[i].start, regions[i].end); + if (!r) { + pr_err("%d'th init region creation failed\n", i); + return; + } + damon_add_region(r, t); + + nr_pieces = (regions[i].end - regions[i].start) / sz; + damon_va_evenly_split_region(t, r, nr_pieces); + } +} + +/* Initialize '->regions_list' of every target (task) */ +void damon_va_init(struct damon_ctx *ctx) +{ + struct damon_target *t; + + damon_for_each_target(t, ctx) { + /* the user may set the target regions as they want */ + if (!damon_nr_regions(t)) + __damon_va_init_regions(ctx, t); + } +} + +/* + * Functions for the dynamic monitoring target regions update + */ + +/* + * Check whether a region is intersecting an address range + * + * Returns true if it is. + */ +static bool damon_intersect(struct damon_region *r, struct damon_addr_range *re) +{ + return !(r->ar.end <= re->start || re->end <= r->ar.start); +} + +/* + * Update damon regions for the three big regions of the given target + * + * t the given target + * bregions the three big regions of the target + */ +static void damon_va_apply_three_regions(struct damon_target *t, + struct damon_addr_range bregions[3]) +{ + struct damon_region *r, *next; + unsigned int i = 0; + + /* Remove regions which are not in the three big regions now */ + damon_for_each_region_safe(r, next, t) { + for (i = 0; i < 3; i++) { + if (damon_intersect(r, &bregions[i])) + break; + } + if (i == 3) + damon_destroy_region(r, t); + } + + /* Adjust intersecting regions to fit with the three big regions */ + for (i = 0; i < 3; i++) { + struct damon_region *first = NULL, *last; + struct damon_region *newr; + struct damon_addr_range *br; + + br = &bregions[i]; + /* Get the first and last regions which intersects with br */ + damon_for_each_region(r, t) { + if (damon_intersect(r, br)) { + if (!first) + first = r; + last = r; + } + if (r->ar.start >= br->end) + break; + } + if (!first) { + /* no damon_region intersects with this big region */ + newr = damon_new_region( + ALIGN_DOWN(br->start, + DAMON_MIN_REGION), + ALIGN(br->end, DAMON_MIN_REGION)); + if (!newr) + continue; + damon_insert_region(newr, damon_prev_region(r), r, t); + } else { + first->ar.start = ALIGN_DOWN(br->start, + DAMON_MIN_REGION); + last->ar.end = ALIGN(br->end, DAMON_MIN_REGION); + } + } +} + +/* + * Update regions for current memory mappings + */ +void damon_va_update(struct damon_ctx *ctx) +{ + struct damon_addr_range three_regions[3]; + struct damon_target *t; + + damon_for_each_target(t, ctx) { + if (damon_va_three_regions(t, three_regions)) + continue; + damon_va_apply_three_regions(t, three_regions); + } +} + +/* + * Get an online page for a pfn if it's in the LRU list. Otherwise, returns + * NULL. + * + * The body of this function is stolen from the 'page_idle_get_page()'. We + * steal rather than reuse it because the code is quite simple. + */ +static struct page *damon_get_page(unsigned long pfn) +{ + struct page *page = pfn_to_online_page(pfn); + + if (!page || !PageLRU(page) || !get_page_unless_zero(page)) + return NULL; + + if (unlikely(!PageLRU(page))) { + put_page(page); + page = NULL; + } + return page; +} + +static void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, + unsigned long addr) +{ + bool referenced = false; + struct page *page = damon_get_page(pte_pfn(*pte)); + + if (!page) + return; + + if (pte_young(*pte)) { + referenced = true; + *pte = pte_mkold(*pte); + } + +#ifdef CONFIG_MMU_NOTIFIER + if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE)) + referenced = true; +#endif /* CONFIG_MMU_NOTIFIER */ + + if (referenced) + set_page_young(page); + + set_page_idle(page); + put_page(page); +} + +static void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, + unsigned long addr) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + bool referenced = false; + struct page *page = damon_get_page(pmd_pfn(*pmd)); + + if (!page) + return; + + if (pmd_young(*pmd)) { + referenced = true; + *pmd = pmd_mkold(*pmd); + } + +#ifdef CONFIG_MMU_NOTIFIER + if (mmu_notifier_clear_young(mm, addr, + addr + ((1UL) << HPAGE_PMD_SHIFT))) + referenced = true; +#endif /* CONFIG_MMU_NOTIFIER */ + + if (referenced) + set_page_young(page); + + set_page_idle(page); + put_page(page); +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +} + +static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pte_t *pte; + spinlock_t *ptl; + + if (pmd_huge(*pmd)) { + ptl = pmd_lock(walk->mm, pmd); + if (pmd_huge(*pmd)) { + damon_pmdp_mkold(pmd, walk->mm, addr); + spin_unlock(ptl); + return 0; + } + spin_unlock(ptl); + } + + if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) + return 0; + pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); + if (!pte_present(*pte)) + goto out; + damon_ptep_mkold(pte, walk->mm, addr); +out: + pte_unmap_unlock(pte, ptl); + return 0; +} + +static struct mm_walk_ops damon_mkold_ops = { + .pmd_entry = damon_mkold_pmd_entry, +}; + +static void damon_va_mkold(struct mm_struct *mm, unsigned long addr) +{ + mmap_read_lock(mm); + walk_page_range(mm, addr, addr + 1, &damon_mkold_ops, NULL); + mmap_read_unlock(mm); +} + +/* + * Functions for the access checking of the regions + */ + +static void damon_va_prepare_access_check(struct damon_ctx *ctx, + struct mm_struct *mm, struct damon_region *r) +{ + r->sampling_addr = damon_rand(r->ar.start, r->ar.end); + + damon_va_mkold(mm, r->sampling_addr); +} + +void damon_va_prepare_access_checks(struct damon_ctx *ctx) +{ + struct damon_target *t; + struct mm_struct *mm; + struct damon_region *r; + + damon_for_each_target(t, ctx) { + mm = damon_get_mm(t); + if (!mm) + continue; + damon_for_each_region(r, t) + damon_va_prepare_access_check(ctx, mm, r); + mmput(mm); + } +} + +struct damon_young_walk_private { + unsigned long *page_sz; + bool young; +}; + +static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pte_t *pte; + spinlock_t *ptl; + struct page *page; + struct damon_young_walk_private *priv = walk->private; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (pmd_huge(*pmd)) { + ptl = pmd_lock(walk->mm, pmd); + if (!pmd_huge(*pmd)) { + spin_unlock(ptl); + goto regular_page; + } + page = damon_get_page(pmd_pfn(*pmd)); + if (!page) + goto huge_out; + if (pmd_young(*pmd) || !page_is_idle(page) || + mmu_notifier_test_young(walk->mm, + addr)) { + *priv->page_sz = ((1UL) << HPAGE_PMD_SHIFT); + priv->young = true; + } + put_page(page); +huge_out: + spin_unlock(ptl); + return 0; + } + +regular_page: +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + + if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) + return -EINVAL; + pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); + if (!pte_present(*pte)) + goto out; + page = damon_get_page(pte_pfn(*pte)); + if (!page) + goto out; + if (pte_young(*pte) || !page_is_idle(page) || + mmu_notifier_test_young(walk->mm, addr)) { + *priv->page_sz = PAGE_SIZE; + priv->young = true; + } + put_page(page); +out: + pte_unmap_unlock(pte, ptl); + return 0; +} + +static struct mm_walk_ops damon_young_ops = { + .pmd_entry = damon_young_pmd_entry, +}; + +static bool damon_va_young(struct mm_struct *mm, unsigned long addr, + unsigned long *page_sz) +{ + struct damon_young_walk_private arg = { + .page_sz = page_sz, + .young = false, + }; + + mmap_read_lock(mm); + walk_page_range(mm, addr, addr + 1, &damon_young_ops, &arg); + mmap_read_unlock(mm); + return arg.young; +} + +/* + * Check whether the region was accessed after the last preparation + * + * mm 'mm_struct' for the given virtual address space + * r the region to be checked + */ +static void damon_va_check_access(struct damon_ctx *ctx, + struct mm_struct *mm, struct damon_region *r) +{ + static struct mm_struct *last_mm; + static unsigned long last_addr; + static unsigned long last_page_sz = PAGE_SIZE; + static bool last_accessed; + + /* If the region is in the last checked page, reuse the result */ + if (mm == last_mm && (ALIGN_DOWN(last_addr, last_page_sz) == + ALIGN_DOWN(r->sampling_addr, last_page_sz))) { + if (last_accessed) + r->nr_accesses++; + return; + } + + last_accessed = damon_va_young(mm, r->sampling_addr, &last_page_sz); + if (last_accessed) + r->nr_accesses++; + + last_mm = mm; + last_addr = r->sampling_addr; +} + +unsigned int damon_va_check_accesses(struct damon_ctx *ctx) +{ + struct damon_target *t; + struct mm_struct *mm; + struct damon_region *r; + unsigned int max_nr_accesses = 0; + + damon_for_each_target(t, ctx) { + mm = damon_get_mm(t); + if (!mm) + continue; + damon_for_each_region(r, t) { + damon_va_check_access(ctx, mm, r); + max_nr_accesses = max(r->nr_accesses, max_nr_accesses); + } + mmput(mm); + } + + return max_nr_accesses; +} + +/* + * Functions for the target validity check and cleanup + */ + +bool damon_va_target_valid(void *target) +{ + struct damon_target *t = target; + struct task_struct *task; + + task = damon_get_task_struct(t); + if (task) { + put_task_struct(task); + return true; + } + + return false; +} + +void damon_va_set_primitives(struct damon_ctx *ctx) +{ + ctx->primitive.init = damon_va_init; + ctx->primitive.update = damon_va_update; + ctx->primitive.prepare_access_checks = damon_va_prepare_access_checks; + ctx->primitive.check_accesses = damon_va_check_accesses; + ctx->primitive.reset_aggregated = NULL; + ctx->primitive.target_valid = damon_va_target_valid; + ctx->primitive.cleanup = NULL; +} From 3ea808dca1b002a141feab8389215ca73c23bc50 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 7 Sep 2021 19:56:48 -0700 Subject: [PATCH 051/156] UPSTREAM: mm/damon: add a tracepoint This commit adds a tracepoint for DAMON. It traces the monitoring results of each region for each aggregation interval. Using this, DAMON can easily integrated with tracepoints supporting tools such as perf. Link: https://lkml.kernel.org/r/20210716081449.22187-7-sj38.park@gmail.com Signed-off-by: SeongJae Park Reviewed-by: Leonard Foerster Reviewed-by: Steven Rostedt (VMware) Reviewed-by: Fernand Sieber Acked-by: Shakeel Butt Cc: Alexander Shishkin Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Fan Du Cc: Greg Kroah-Hartman Cc: Greg Thelen Cc: Ingo Molnar Cc: Joe Perches Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Marco Elver Cc: Markus Boehme Cc: Maximilian Heyne Cc: Mel Gorman Cc: Minchan Kim Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rik van Riel Cc: Shuah Khan Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 2fcb93629ad8911c846cdc44521c746e53cc4e6d) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I502a57938f7aa60c55b05525b45e50ddb1aeda65 --- include/trace/events/damon.h | 43 ++++++++++++++++++++++++++++++++++++ mm/damon/core.c | 7 +++++- 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 include/trace/events/damon.h diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h new file mode 100644 index 000000000000..2f422f4f1fb9 --- /dev/null +++ b/include/trace/events/damon.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM damon + +#if !defined(_TRACE_DAMON_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_DAMON_H + +#include +#include +#include + +TRACE_EVENT(damon_aggregated, + + TP_PROTO(struct damon_target *t, struct damon_region *r, + unsigned int nr_regions), + + TP_ARGS(t, r, nr_regions), + + TP_STRUCT__entry( + __field(unsigned long, target_id) + __field(unsigned int, nr_regions) + __field(unsigned long, start) + __field(unsigned long, end) + __field(unsigned int, nr_accesses) + ), + + TP_fast_assign( + __entry->target_id = t->id; + __entry->nr_regions = nr_regions; + __entry->start = r->ar.start; + __entry->end = r->ar.end; + __entry->nr_accesses = r->nr_accesses; + ), + + TP_printk("target_id=%lu nr_regions=%u %lu-%lu: %u", + __entry->target_id, __entry->nr_regions, + __entry->start, __entry->end, __entry->nr_accesses) +); + +#endif /* _TRACE_DAMON_H */ + +/* This part must be outside protection */ +#include diff --git a/mm/damon/core.c b/mm/damon/core.c index 28a2c78914fa..ee24d64e8019 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -13,6 +13,9 @@ #include #include +#define CREATE_TRACE_POINTS +#include + /* Get a random number in [l, r) */ #define damon_rand(l, r) (l + prandom_u32_max(r - l)) @@ -387,8 +390,10 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) damon_for_each_target(t, c) { struct damon_region *r; - damon_for_each_region(r, t) + damon_for_each_region(r, t) { + trace_damon_aggregated(t, r, damon_nr_regions(t)); r->nr_accesses = 0; + } } } From c10dc7808e150edcdc7b47ff087829ab63d2b78a Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 7 Sep 2021 19:56:53 -0700 Subject: [PATCH 052/156] UPSTREAM: mm/damon: implement a debugfs-based user space interface DAMON is designed to be used by kernel space code such as the memory management subsystems, and therefore it provides only kernel space API. That said, letting the user space control DAMON could provide some benefits to them. For example, it will allow user space to analyze their specific workloads and make their own special optimizations. For such cases, this commit implements a simple DAMON application kernel module, namely 'damon-dbgfs', which merely wraps the DAMON api and exports those to the user space via the debugfs. 'damon-dbgfs' exports three files, ``attrs``, ``target_ids``, and ``monitor_on`` under its debugfs directory, ``/damon/``. Attributes ---------- Users can read and write the ``sampling interval``, ``aggregation interval``, ``regions update interval``, and min/max number of monitoring target regions by reading from and writing to the ``attrs`` file. For example, below commands set those values to 5 ms, 100 ms, 1,000 ms, 10, 1000 and check it again:: # cd /damon # echo 5000 100000 1000000 10 1000 > attrs # cat attrs 5000 100000 1000000 10 1000 Target IDs ---------- Some types of address spaces supports multiple monitoring target. For example, the virtual memory address spaces monitoring can have multiple processes as the monitoring targets. Users can set the targets by writing relevant id values of the targets to, and get the ids of the current targets by reading from the ``target_ids`` file. In case of the virtual address spaces monitoring, the values should be pids of the monitoring target processes. For example, below commands set processes having pids 42 and 4242 as the monitoring targets and check it again:: # cd /damon # echo 42 4242 > target_ids # cat target_ids 42 4242 Note that setting the target ids doesn't start the monitoring. Turning On/Off -------------- Setting the files as described above doesn't incur effect unless you explicitly start the monitoring. You can start, stop, and check the current status of the monitoring by writing to and reading from the ``monitor_on`` file. Writing ``on`` to the file starts the monitoring of the targets with the attributes. Writing ``off`` to the file stops those. DAMON also stops if every targets are invalidated (in case of the virtual memory monitoring, target processes are invalidated when terminated). Below example commands turn on, off, and check the status of DAMON:: # cd /damon # echo on > monitor_on # echo off > monitor_on # cat monitor_on off Please note that you cannot write to the above-mentioned debugfs files while the monitoring is turned on. If you write to the files while DAMON is running, an error code such as ``-EBUSY`` will be returned. [akpm@linux-foundation.org: remove unneeded "alloc failed" printks] [akpm@linux-foundation.org: replace macro with static inline] Link: https://lkml.kernel.org/r/20210716081449.22187-8-sj38.park@gmail.com Signed-off-by: SeongJae Park Reviewed-by: Leonard Foerster Reviewed-by: Fernand Sieber Cc: Alexander Shishkin Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Fan Du Cc: Greg Kroah-Hartman Cc: Greg Thelen Cc: Ingo Molnar Cc: Joe Perches Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Marco Elver Cc: Markus Boehme Cc: Maximilian Heyne Cc: Mel Gorman Cc: Minchan Kim Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rik van Riel Cc: Shakeel Butt Cc: Shuah Khan Cc: Steven Rostedt (VMware) Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 4bc05954d0076655cfaf6f0135585bdc20cd6b11) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I99664d4372dc5c510d7e16ffe384345e52579b0e --- include/linux/damon.h | 3 + mm/damon/Kconfig | 9 + mm/damon/Makefile | 1 + mm/damon/core.c | 47 +++++ mm/damon/dbgfs.c | 397 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 457 insertions(+) create mode 100644 mm/damon/dbgfs.c diff --git a/include/linux/damon.h b/include/linux/damon.h index edb350e52b93..d68b67b8d458 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -240,9 +240,12 @@ unsigned int damon_nr_regions(struct damon_target *t); struct damon_ctx *damon_new_ctx(void); void damon_destroy_ctx(struct damon_ctx *ctx); +int damon_set_targets(struct damon_ctx *ctx, + unsigned long *ids, ssize_t nr_ids); int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, unsigned long aggr_int, unsigned long primitive_upd_int, unsigned long min_nr_reg, unsigned long max_nr_reg); +int damon_nr_running_ctxs(void); int damon_start(struct damon_ctx **ctxs, int nr_ctxs); int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig index 5cbb5db54158..c8e3dba6fb4c 100644 --- a/mm/damon/Kconfig +++ b/mm/damon/Kconfig @@ -20,4 +20,13 @@ config DAMON_VADDR This builds the default data access monitoring primitives for DAMON that works for virtual address spaces. +config DAMON_DBGFS + bool "DAMON debugfs interface" + depends on DAMON_VADDR && DEBUG_FS + help + This builds the debugfs interface for DAMON. The user space admins + can use the interface for arbitrary data access monitoring. + + If unsure, say N. + endmenu diff --git a/mm/damon/Makefile b/mm/damon/Makefile index 6ebbd08aed67..fed4be3bace3 100644 --- a/mm/damon/Makefile +++ b/mm/damon/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_DAMON) := core.o obj-$(CONFIG_DAMON_VADDR) += vaddr.o +obj-$(CONFIG_DAMON_DBGFS) += dbgfs.o diff --git a/mm/damon/core.c b/mm/damon/core.c index ee24d64e8019..59033488402e 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -171,6 +171,39 @@ void damon_destroy_ctx(struct damon_ctx *ctx) kfree(ctx); } +/** + * damon_set_targets() - Set monitoring targets. + * @ctx: monitoring context + * @ids: array of target ids + * @nr_ids: number of entries in @ids + * + * This function should not be called while the kdamond is running. + * + * Return: 0 on success, negative error code otherwise. + */ +int damon_set_targets(struct damon_ctx *ctx, + unsigned long *ids, ssize_t nr_ids) +{ + ssize_t i; + struct damon_target *t, *next; + + damon_destroy_targets(ctx); + + for (i = 0; i < nr_ids; i++) { + t = damon_new_target(ids[i]); + if (!t) { + pr_err("Failed to alloc damon_target\n"); + /* The caller should do cleanup of the ids itself */ + damon_for_each_target_safe(t, next, ctx) + damon_destroy_target(t); + return -ENOMEM; + } + damon_add_target(ctx, t); + } + + return 0; +} + /** * damon_set_attrs() - Set attributes for the monitoring. * @ctx: monitoring context @@ -209,6 +242,20 @@ int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, return 0; } +/** + * damon_nr_running_ctxs() - Return number of currently running contexts. + */ +int damon_nr_running_ctxs(void) +{ + int nr_ctxs; + + mutex_lock(&damon_lock); + nr_ctxs = nr_running_ctxs; + mutex_unlock(&damon_lock); + + return nr_ctxs; +} + /* Returns the size upper limit for each monitoring region */ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) { diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c new file mode 100644 index 000000000000..d2e0a547eb3f --- /dev/null +++ b/mm/damon/dbgfs.c @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DAMON Debugfs Interface + * + * Author: SeongJae Park + */ + +#define pr_fmt(fmt) "damon-dbgfs: " fmt + +#include +#include +#include +#include +#include +#include +#include + +static struct damon_ctx **dbgfs_ctxs; +static int dbgfs_nr_ctxs; +static struct dentry **dbgfs_dirs; + +/* + * Returns non-empty string on success, negative error code otherwise. + */ +static char *user_input_str(const char __user *buf, size_t count, loff_t *ppos) +{ + char *kbuf; + ssize_t ret; + + /* We do not accept continuous write */ + if (*ppos) + return ERR_PTR(-EINVAL); + + kbuf = kmalloc(count + 1, GFP_KERNEL); + if (!kbuf) + return ERR_PTR(-ENOMEM); + + ret = simple_write_to_buffer(kbuf, count + 1, ppos, buf, count); + if (ret != count) { + kfree(kbuf); + return ERR_PTR(-EIO); + } + kbuf[ret] = '\0'; + + return kbuf; +} + +static ssize_t dbgfs_attrs_read(struct file *file, + char __user *buf, size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char kbuf[128]; + int ret; + + mutex_lock(&ctx->kdamond_lock); + ret = scnprintf(kbuf, ARRAY_SIZE(kbuf), "%lu %lu %lu %lu %lu\n", + ctx->sample_interval, ctx->aggr_interval, + ctx->primitive_update_interval, ctx->min_nr_regions, + ctx->max_nr_regions); + mutex_unlock(&ctx->kdamond_lock); + + return simple_read_from_buffer(buf, count, ppos, kbuf, ret); +} + +static ssize_t dbgfs_attrs_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + unsigned long s, a, r, minr, maxr; + char *kbuf; + ssize_t ret = count; + int err; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + if (sscanf(kbuf, "%lu %lu %lu %lu %lu", + &s, &a, &r, &minr, &maxr) != 5) { + ret = -EINVAL; + goto out; + } + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + ret = -EBUSY; + goto unlock_out; + } + + err = damon_set_attrs(ctx, s, a, r, minr, maxr); + if (err) + ret = err; +unlock_out: + mutex_unlock(&ctx->kdamond_lock); +out: + kfree(kbuf); + return ret; +} + +static inline bool targetid_is_pid(const struct damon_ctx *ctx) +{ + return ctx->primitive.target_valid == damon_va_target_valid; +} + +static ssize_t sprint_target_ids(struct damon_ctx *ctx, char *buf, ssize_t len) +{ + struct damon_target *t; + unsigned long id; + int written = 0; + int rc; + + damon_for_each_target(t, ctx) { + id = t->id; + if (targetid_is_pid(ctx)) + /* Show pid numbers to debugfs users */ + id = (unsigned long)pid_vnr((struct pid *)id); + + rc = scnprintf(&buf[written], len - written, "%lu ", id); + if (!rc) + return -ENOMEM; + written += rc; + } + if (written) + written -= 1; + written += scnprintf(&buf[written], len - written, "\n"); + return written; +} + +static ssize_t dbgfs_target_ids_read(struct file *file, + char __user *buf, size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + ssize_t len; + char ids_buf[320]; + + mutex_lock(&ctx->kdamond_lock); + len = sprint_target_ids(ctx, ids_buf, 320); + mutex_unlock(&ctx->kdamond_lock); + if (len < 0) + return len; + + return simple_read_from_buffer(buf, count, ppos, ids_buf, len); +} + +/* + * Converts a string into an array of unsigned long integers + * + * Returns an array of unsigned long integers if the conversion success, or + * NULL otherwise. + */ +static unsigned long *str_to_target_ids(const char *str, ssize_t len, + ssize_t *nr_ids) +{ + unsigned long *ids; + const int max_nr_ids = 32; + unsigned long id; + int pos = 0, parsed, ret; + + *nr_ids = 0; + ids = kmalloc_array(max_nr_ids, sizeof(id), GFP_KERNEL); + if (!ids) + return NULL; + while (*nr_ids < max_nr_ids && pos < len) { + ret = sscanf(&str[pos], "%lu%n", &id, &parsed); + pos += parsed; + if (ret != 1) + break; + ids[*nr_ids] = id; + *nr_ids += 1; + } + + return ids; +} + +static void dbgfs_put_pids(unsigned long *ids, int nr_ids) +{ + int i; + + for (i = 0; i < nr_ids; i++) + put_pid((struct pid *)ids[i]); +} + +static ssize_t dbgfs_target_ids_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char *kbuf, *nrs; + unsigned long *targets; + ssize_t nr_targets; + ssize_t ret = count; + int i; + int err; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + nrs = kbuf; + + targets = str_to_target_ids(nrs, ret, &nr_targets); + if (!targets) { + ret = -ENOMEM; + goto out; + } + + if (targetid_is_pid(ctx)) { + for (i = 0; i < nr_targets; i++) { + targets[i] = (unsigned long)find_get_pid( + (int)targets[i]); + if (!targets[i]) { + dbgfs_put_pids(targets, i); + ret = -EINVAL; + goto free_targets_out; + } + } + } + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + if (targetid_is_pid(ctx)) + dbgfs_put_pids(targets, nr_targets); + ret = -EBUSY; + goto unlock_out; + } + + err = damon_set_targets(ctx, targets, nr_targets); + if (err) { + if (targetid_is_pid(ctx)) + dbgfs_put_pids(targets, nr_targets); + ret = err; + } + +unlock_out: + mutex_unlock(&ctx->kdamond_lock); +free_targets_out: + kfree(targets); +out: + kfree(kbuf); + return ret; +} + +static int damon_dbgfs_open(struct inode *inode, struct file *file) +{ + file->private_data = inode->i_private; + + return nonseekable_open(inode, file); +} + +static const struct file_operations attrs_fops = { + .open = damon_dbgfs_open, + .read = dbgfs_attrs_read, + .write = dbgfs_attrs_write, +}; + +static const struct file_operations target_ids_fops = { + .open = damon_dbgfs_open, + .read = dbgfs_target_ids_read, + .write = dbgfs_target_ids_write, +}; + +static void dbgfs_fill_ctx_dir(struct dentry *dir, struct damon_ctx *ctx) +{ + const char * const file_names[] = {"attrs", "target_ids"}; + const struct file_operations *fops[] = {&attrs_fops, &target_ids_fops}; + int i; + + for (i = 0; i < ARRAY_SIZE(file_names); i++) + debugfs_create_file(file_names[i], 0600, dir, ctx, fops[i]); +} + +static int dbgfs_before_terminate(struct damon_ctx *ctx) +{ + struct damon_target *t, *next; + + if (!targetid_is_pid(ctx)) + return 0; + + damon_for_each_target_safe(t, next, ctx) { + put_pid((struct pid *)t->id); + damon_destroy_target(t); + } + return 0; +} + +static struct damon_ctx *dbgfs_new_ctx(void) +{ + struct damon_ctx *ctx; + + ctx = damon_new_ctx(); + if (!ctx) + return NULL; + + damon_va_set_primitives(ctx); + ctx->callback.before_terminate = dbgfs_before_terminate; + return ctx; +} + +static ssize_t dbgfs_monitor_on_read(struct file *file, + char __user *buf, size_t count, loff_t *ppos) +{ + char monitor_on_buf[5]; + bool monitor_on = damon_nr_running_ctxs() != 0; + int len; + + len = scnprintf(monitor_on_buf, 5, monitor_on ? "on\n" : "off\n"); + + return simple_read_from_buffer(buf, count, ppos, monitor_on_buf, len); +} + +static ssize_t dbgfs_monitor_on_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + ssize_t ret = count; + char *kbuf; + int err; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + /* Remove white space */ + if (sscanf(kbuf, "%s", kbuf) != 1) { + kfree(kbuf); + return -EINVAL; + } + + if (!strncmp(kbuf, "on", count)) + err = damon_start(dbgfs_ctxs, dbgfs_nr_ctxs); + else if (!strncmp(kbuf, "off", count)) + err = damon_stop(dbgfs_ctxs, dbgfs_nr_ctxs); + else + err = -EINVAL; + + if (err) + ret = err; + kfree(kbuf); + return ret; +} + +static const struct file_operations monitor_on_fops = { + .read = dbgfs_monitor_on_read, + .write = dbgfs_monitor_on_write, +}; + +static int __init __damon_dbgfs_init(void) +{ + struct dentry *dbgfs_root; + const char * const file_names[] = {"monitor_on"}; + const struct file_operations *fops[] = {&monitor_on_fops}; + int i; + + dbgfs_root = debugfs_create_dir("damon", NULL); + + for (i = 0; i < ARRAY_SIZE(file_names); i++) + debugfs_create_file(file_names[i], 0600, dbgfs_root, NULL, + fops[i]); + dbgfs_fill_ctx_dir(dbgfs_root, dbgfs_ctxs[0]); + + dbgfs_dirs = kmalloc_array(1, sizeof(dbgfs_root), GFP_KERNEL); + if (!dbgfs_dirs) { + debugfs_remove(dbgfs_root); + return -ENOMEM; + } + dbgfs_dirs[0] = dbgfs_root; + + return 0; +} + +/* + * Functions for the initialization + */ + +static int __init damon_dbgfs_init(void) +{ + int rc; + + dbgfs_ctxs = kmalloc(sizeof(*dbgfs_ctxs), GFP_KERNEL); + if (!dbgfs_ctxs) + return -ENOMEM; + dbgfs_ctxs[0] = dbgfs_new_ctx(); + if (!dbgfs_ctxs[0]) { + kfree(dbgfs_ctxs); + return -ENOMEM; + } + dbgfs_nr_ctxs = 1; + + rc = __damon_dbgfs_init(); + if (rc) { + kfree(dbgfs_ctxs[0]); + kfree(dbgfs_ctxs); + pr_err("%s: dbgfs init failed\n", __func__); + } + + return rc; +} + +module_init(damon_dbgfs_init); From 562b676ce9883157042686b028fce83f5b2cd99e Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 7 Sep 2021 19:56:57 -0700 Subject: [PATCH 053/156] UPSTREAM: mm/damon/dbgfs: export kdamond pid to the user space For CPU usage accounting, knowing pid of the monitoring thread could be helpful. For example, users could use cpuaccount cgroups with the pid. This commit therefore exports the pid of currently running monitoring thread to the user space via 'kdamond_pid' file in the debugfs directory. Link: https://lkml.kernel.org/r/20210716081449.22187-9-sj38.park@gmail.com Signed-off-by: SeongJae Park Reviewed-by: Fernand Sieber Cc: Alexander Shishkin Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Fan Du Cc: Greg Kroah-Hartman Cc: Greg Thelen Cc: Ingo Molnar Cc: Joe Perches Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Maximilian Heyne Cc: Mel Gorman Cc: Minchan Kim Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rik van Riel Cc: Shakeel Butt Cc: Shuah Khan Cc: Steven Rostedt (VMware) Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 429538e85410c3ae12719ec42b89ab873ed6d47b) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I32d41776a1850ed189b2759194061004ca8a83cf --- mm/damon/dbgfs.c | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index d2e0a547eb3f..e850be4077f5 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -239,6 +239,32 @@ out: return ret; } +static ssize_t dbgfs_kdamond_pid_read(struct file *file, + char __user *buf, size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char *kbuf; + ssize_t len; + + kbuf = kmalloc(count, GFP_KERNEL); + if (!kbuf) + return -ENOMEM; + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) + len = scnprintf(kbuf, count, "%d\n", ctx->kdamond->pid); + else + len = scnprintf(kbuf, count, "none\n"); + mutex_unlock(&ctx->kdamond_lock); + if (!len) + goto out; + len = simple_read_from_buffer(buf, count, ppos, kbuf, len); + +out: + kfree(kbuf); + return len; +} + static int damon_dbgfs_open(struct inode *inode, struct file *file) { file->private_data = inode->i_private; @@ -258,10 +284,17 @@ static const struct file_operations target_ids_fops = { .write = dbgfs_target_ids_write, }; +static const struct file_operations kdamond_pid_fops = { + .open = damon_dbgfs_open, + .read = dbgfs_kdamond_pid_read, +}; + static void dbgfs_fill_ctx_dir(struct dentry *dir, struct damon_ctx *ctx) { - const char * const file_names[] = {"attrs", "target_ids"}; - const struct file_operations *fops[] = {&attrs_fops, &target_ids_fops}; + const char * const file_names[] = {"attrs", "target_ids", + "kdamond_pid"}; + const struct file_operations *fops[] = {&attrs_fops, &target_ids_fops, + &kdamond_pid_fops}; int i; for (i = 0; i < ARRAY_SIZE(file_names); i++) From fe62a24792e5fe56294cb2edcbd8c203fa09f24d Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 7 Sep 2021 19:57:01 -0700 Subject: [PATCH 054/156] UPSTREAM: mm/damon/dbgfs: support multiple contexts In some use cases, users would want to run multiple monitoring context. For example, if a user wants a high precision monitoring and dedicating multiple CPUs for the job is ok, because DAMON creates one monitoring thread per one context, the user can split the monitoring target regions into multiple small regions and create one context for each region. Or, someone might want to simultaneously monitor different address spaces, e.g., both virtual address space and physical address space. The DAMON's API allows such usage, but 'damon-dbgfs' does not. Therefore, only kernel space DAMON users can do multiple contexts monitoring. This commit allows the user space DAMON users to use multiple contexts monitoring by introducing two new 'damon-dbgfs' debugfs files, 'mk_context' and 'rm_context'. Users can create a new monitoring context by writing the desired name of the new context to 'mk_context'. Then, a new directory with the name and having the files for setting of the context ('attrs', 'target_ids' and 'record') will be created under the debugfs directory. Writing the name of the context to remove to 'rm_context' will remove the related context and directory. Link: https://lkml.kernel.org/r/20210716081449.22187-10-sj38.park@gmail.com Signed-off-by: SeongJae Park Reviewed-by: Fernand Sieber Cc: Alexander Shishkin Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Fan Du Cc: Greg Kroah-Hartman Cc: Greg Thelen Cc: Ingo Molnar Cc: Joe Perches Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Maximilian Heyne Cc: Mel Gorman Cc: Minchan Kim Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rik van Riel Cc: Shakeel Butt Cc: Shuah Khan Cc: Steven Rostedt (VMware) Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 75c1c2b53c78bf3b3188ebb7b3508dadbf98bba1) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Idba5509b6ef147cd6f8cd820d241718b284a34a3 --- mm/damon/dbgfs.c | 195 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 193 insertions(+), 2 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index e850be4077f5..31ad550ecba2 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -18,6 +18,7 @@ static struct damon_ctx **dbgfs_ctxs; static int dbgfs_nr_ctxs; static struct dentry **dbgfs_dirs; +static DEFINE_MUTEX(damon_dbgfs_lock); /* * Returns non-empty string on success, negative error code otherwise. @@ -328,6 +329,186 @@ static struct damon_ctx *dbgfs_new_ctx(void) return ctx; } +static void dbgfs_destroy_ctx(struct damon_ctx *ctx) +{ + damon_destroy_ctx(ctx); +} + +/* + * Make a context of @name and create a debugfs directory for it. + * + * This function should be called while holding damon_dbgfs_lock. + * + * Returns 0 on success, negative error code otherwise. + */ +static int dbgfs_mk_context(char *name) +{ + struct dentry *root, **new_dirs, *new_dir; + struct damon_ctx **new_ctxs, *new_ctx; + + if (damon_nr_running_ctxs()) + return -EBUSY; + + new_ctxs = krealloc(dbgfs_ctxs, sizeof(*dbgfs_ctxs) * + (dbgfs_nr_ctxs + 1), GFP_KERNEL); + if (!new_ctxs) + return -ENOMEM; + dbgfs_ctxs = new_ctxs; + + new_dirs = krealloc(dbgfs_dirs, sizeof(*dbgfs_dirs) * + (dbgfs_nr_ctxs + 1), GFP_KERNEL); + if (!new_dirs) + return -ENOMEM; + dbgfs_dirs = new_dirs; + + root = dbgfs_dirs[0]; + if (!root) + return -ENOENT; + + new_dir = debugfs_create_dir(name, root); + dbgfs_dirs[dbgfs_nr_ctxs] = new_dir; + + new_ctx = dbgfs_new_ctx(); + if (!new_ctx) { + debugfs_remove(new_dir); + dbgfs_dirs[dbgfs_nr_ctxs] = NULL; + return -ENOMEM; + } + + dbgfs_ctxs[dbgfs_nr_ctxs] = new_ctx; + dbgfs_fill_ctx_dir(dbgfs_dirs[dbgfs_nr_ctxs], + dbgfs_ctxs[dbgfs_nr_ctxs]); + dbgfs_nr_ctxs++; + + return 0; +} + +static ssize_t dbgfs_mk_context_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + char *kbuf; + char *ctx_name; + ssize_t ret = count; + int err; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + ctx_name = kmalloc(count + 1, GFP_KERNEL); + if (!ctx_name) { + kfree(kbuf); + return -ENOMEM; + } + + /* Trim white space */ + if (sscanf(kbuf, "%s", ctx_name) != 1) { + ret = -EINVAL; + goto out; + } + + mutex_lock(&damon_dbgfs_lock); + err = dbgfs_mk_context(ctx_name); + if (err) + ret = err; + mutex_unlock(&damon_dbgfs_lock); + +out: + kfree(kbuf); + kfree(ctx_name); + return ret; +} + +/* + * Remove a context of @name and its debugfs directory. + * + * This function should be called while holding damon_dbgfs_lock. + * + * Return 0 on success, negative error code otherwise. + */ +static int dbgfs_rm_context(char *name) +{ + struct dentry *root, *dir, **new_dirs; + struct damon_ctx **new_ctxs; + int i, j; + + if (damon_nr_running_ctxs()) + return -EBUSY; + + root = dbgfs_dirs[0]; + if (!root) + return -ENOENT; + + dir = debugfs_lookup(name, root); + if (!dir) + return -ENOENT; + + new_dirs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_dirs), + GFP_KERNEL); + if (!new_dirs) + return -ENOMEM; + + new_ctxs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_ctxs), + GFP_KERNEL); + if (!new_ctxs) { + kfree(new_dirs); + return -ENOMEM; + } + + for (i = 0, j = 0; i < dbgfs_nr_ctxs; i++) { + if (dbgfs_dirs[i] == dir) { + debugfs_remove(dbgfs_dirs[i]); + dbgfs_destroy_ctx(dbgfs_ctxs[i]); + continue; + } + new_dirs[j] = dbgfs_dirs[i]; + new_ctxs[j++] = dbgfs_ctxs[i]; + } + + kfree(dbgfs_dirs); + kfree(dbgfs_ctxs); + + dbgfs_dirs = new_dirs; + dbgfs_ctxs = new_ctxs; + dbgfs_nr_ctxs--; + + return 0; +} + +static ssize_t dbgfs_rm_context_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + char *kbuf; + ssize_t ret = count; + int err; + char *ctx_name; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + ctx_name = kmalloc(count + 1, GFP_KERNEL); + if (!ctx_name) { + kfree(kbuf); + return -ENOMEM; + } + + /* Trim white space */ + if (sscanf(kbuf, "%s", ctx_name) != 1) { + ret = -EINVAL; + goto out; + } + + mutex_lock(&damon_dbgfs_lock); + err = dbgfs_rm_context(ctx_name); + if (err) + ret = err; + mutex_unlock(&damon_dbgfs_lock); + +out: + kfree(kbuf); + kfree(ctx_name); + return ret; +} + static ssize_t dbgfs_monitor_on_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { @@ -370,6 +551,14 @@ static ssize_t dbgfs_monitor_on_write(struct file *file, return ret; } +static const struct file_operations mk_contexts_fops = { + .write = dbgfs_mk_context_write, +}; + +static const struct file_operations rm_contexts_fops = { + .write = dbgfs_rm_context_write, +}; + static const struct file_operations monitor_on_fops = { .read = dbgfs_monitor_on_read, .write = dbgfs_monitor_on_write, @@ -378,8 +567,10 @@ static const struct file_operations monitor_on_fops = { static int __init __damon_dbgfs_init(void) { struct dentry *dbgfs_root; - const char * const file_names[] = {"monitor_on"}; - const struct file_operations *fops[] = {&monitor_on_fops}; + const char * const file_names[] = {"mk_contexts", "rm_contexts", + "monitor_on"}; + const struct file_operations *fops[] = {&mk_contexts_fops, + &rm_contexts_fops, &monitor_on_fops}; int i; dbgfs_root = debugfs_create_dir("damon", NULL); From b5131d9c10825d581427986dcce6798da7743d20 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 7 Sep 2021 19:57:13 -0700 Subject: [PATCH 055/156] UPSTREAM: mm/damon: add user space selftests This commit adds a simple user space tests for DAMON. The tests are using kselftest framework. Link: https://lkml.kernel.org/r/20210716081449.22187-13-sj38.park@gmail.com Signed-off-by: SeongJae Park Reviewed-by: Markus Boehme Cc: Alexander Shishkin Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Fan Du Cc: Fernand Sieber Cc: Greg Kroah-Hartman Cc: Greg Thelen Cc: Ingo Molnar Cc: Joe Perches Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Maximilian Heyne Cc: Mel Gorman Cc: Minchan Kim Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rik van Riel Cc: Shakeel Butt Cc: Shuah Khan Cc: Steven Rostedt (VMware) Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit b348eb7abd0987b849420113ced27ad7a1bc6cf3) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I4406106b9d7a0b3774de638fc014357762c30f8e --- tools/testing/selftests/damon/Makefile | 7 ++ .../selftests/damon/_chk_dependency.sh | 28 +++++++ .../testing/selftests/damon/debugfs_attrs.sh | 75 +++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 tools/testing/selftests/damon/Makefile create mode 100644 tools/testing/selftests/damon/_chk_dependency.sh create mode 100644 tools/testing/selftests/damon/debugfs_attrs.sh diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile new file mode 100644 index 000000000000..8a3f2cd9fec0 --- /dev/null +++ b/tools/testing/selftests/damon/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for damon selftests + +TEST_FILES = _chk_dependency.sh +TEST_PROGS = debugfs_attrs.sh + +include ../lib.mk diff --git a/tools/testing/selftests/damon/_chk_dependency.sh b/tools/testing/selftests/damon/_chk_dependency.sh new file mode 100644 index 000000000000..0189db81550b --- /dev/null +++ b/tools/testing/selftests/damon/_chk_dependency.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# Kselftest framework requirement - SKIP code is 4. +ksft_skip=4 + +DBGFS=/sys/kernel/debug/damon + +if [ $EUID -ne 0 ]; +then + echo "Run as root" + exit $ksft_skip +fi + +if [ ! -d "$DBGFS" ] +then + echo "$DBGFS not found" + exit $ksft_skip +fi + +for f in attrs target_ids monitor_on +do + if [ ! -f "$DBGFS/$f" ] + then + echo "$f not found" + exit 1 + fi +done diff --git a/tools/testing/selftests/damon/debugfs_attrs.sh b/tools/testing/selftests/damon/debugfs_attrs.sh new file mode 100644 index 000000000000..bfabb19dc0d3 --- /dev/null +++ b/tools/testing/selftests/damon/debugfs_attrs.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +test_write_result() { + file=$1 + content=$2 + orig_content=$3 + expect_reason=$4 + expected=$5 + + echo "$content" > "$file" + if [ $? -ne "$expected" ] + then + echo "writing $content to $file doesn't return $expected" + echo "expected because: $expect_reason" + echo "$orig_content" > "$file" + exit 1 + fi +} + +test_write_succ() { + test_write_result "$1" "$2" "$3" "$4" 0 +} + +test_write_fail() { + test_write_result "$1" "$2" "$3" "$4" 1 +} + +test_content() { + file=$1 + orig_content=$2 + expected=$3 + expect_reason=$4 + + content=$(cat "$file") + if [ "$content" != "$expected" ] + then + echo "reading $file expected $expected but $content" + echo "expected because: $expect_reason" + echo "$orig_content" > "$file" + exit 1 + fi +} + +source ./_chk_dependency.sh + +# Test attrs file +# =============== + +file="$DBGFS/attrs" +orig_content=$(cat "$file") + +test_write_succ "$file" "1 2 3 4 5" "$orig_content" "valid input" +test_write_fail "$file" "1 2 3 4" "$orig_content" "no enough fields" +test_write_fail "$file" "1 2 3 5 4" "$orig_content" \ + "min_nr_regions > max_nr_regions" +test_content "$file" "$orig_content" "1 2 3 4 5" "successfully written" +echo "$orig_content" > "$file" + +# Test target_ids file +# ==================== + +file="$DBGFS/target_ids" +orig_content=$(cat "$file") + +test_write_succ "$file" "1 2 3 4" "$orig_content" "valid input" +test_write_succ "$file" "1 2 abc 4" "$orig_content" "still valid input" +test_content "$file" "$orig_content" "1 2" "non-integer was there" +test_write_succ "$file" "abc 2 3" "$orig_content" "the file allows wrong input" +test_content "$file" "$orig_content" "" "wrong input written" +test_write_succ "$file" "" "$orig_content" "empty input" +test_content "$file" "$orig_content" "" "empty input written" +echo "$orig_content" > "$file" + +echo "PASS" From c1a4fca349d5e93055463c4050451e069680d3f8 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 7 Sep 2021 19:57:09 -0700 Subject: [PATCH 056/156] UPSTREAM: mm/damon: add kunit tests This commit adds kunit based unit tests for the core and the virtual address spaces monitoring primitives of DAMON. Link: https://lkml.kernel.org/r/20210716081449.22187-12-sj38.park@gmail.com Signed-off-by: SeongJae Park Reviewed-by: Brendan Higgins Cc: Alexander Shishkin Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Fan Du Cc: Fernand Sieber Cc: Greg Kroah-Hartman Cc: Greg Thelen Cc: Ingo Molnar Cc: Joe Perches Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Maximilian Heyne Cc: Mel Gorman Cc: Minchan Kim Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rik van Riel Cc: Shakeel Butt Cc: Shuah Khan Cc: Steven Rostedt (VMware) Cc: Vladimir Davydov Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 17ccae8bb5c928946f6f3af14626ec458f74e6ad) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I3f660b6064a2db350b6b2a422889d189c7a72f62 --- mm/damon/Kconfig | 36 +++++ mm/damon/core-test.h | 253 ++++++++++++++++++++++++++++++++ mm/damon/core.c | 7 + mm/damon/dbgfs-test.h | 126 ++++++++++++++++ mm/damon/dbgfs.c | 2 + mm/damon/vaddr-test.h | 329 ++++++++++++++++++++++++++++++++++++++++++ mm/damon/vaddr.c | 7 + 7 files changed, 760 insertions(+) create mode 100644 mm/damon/core-test.h create mode 100644 mm/damon/dbgfs-test.h create mode 100644 mm/damon/vaddr-test.h diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig index c8e3dba6fb4c..37024798a97c 100644 --- a/mm/damon/Kconfig +++ b/mm/damon/Kconfig @@ -12,6 +12,18 @@ config DAMON See https://damonitor.github.io/doc/html/latest-damon/index.html for more information. +config DAMON_KUNIT_TEST + bool "Test for damon" if !KUNIT_ALL_TESTS + depends on DAMON && KUNIT=y + default KUNIT_ALL_TESTS + help + This builds the DAMON Kunit test suite. + + For more information on KUnit and unit tests in general, please refer + to the KUnit documentation. + + If unsure, say N. + config DAMON_VADDR bool "Data access monitoring primitives for virtual address spaces" depends on DAMON && MMU @@ -20,6 +32,18 @@ config DAMON_VADDR This builds the default data access monitoring primitives for DAMON that works for virtual address spaces. +config DAMON_VADDR_KUNIT_TEST + bool "Test for DAMON primitives" if !KUNIT_ALL_TESTS + depends on DAMON_VADDR && KUNIT=y + default KUNIT_ALL_TESTS + help + This builds the DAMON virtual addresses primitives Kunit test suite. + + For more information on KUnit and unit tests in general, please refer + to the KUnit documentation. + + If unsure, say N. + config DAMON_DBGFS bool "DAMON debugfs interface" depends on DAMON_VADDR && DEBUG_FS @@ -29,4 +53,16 @@ config DAMON_DBGFS If unsure, say N. +config DAMON_DBGFS_KUNIT_TEST + bool "Test for damon debugfs interface" if !KUNIT_ALL_TESTS + depends on DAMON_DBGFS && KUNIT=y + default KUNIT_ALL_TESTS + help + This builds the DAMON debugfs interface Kunit test suite. + + For more information on KUnit and unit tests in general, please refer + to the KUnit documentation. + + If unsure, say N. + endmenu diff --git a/mm/damon/core-test.h b/mm/damon/core-test.h new file mode 100644 index 000000000000..c938a9c34e6c --- /dev/null +++ b/mm/damon/core-test.h @@ -0,0 +1,253 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Data Access Monitor Unit Tests + * + * Copyright 2019 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Author: SeongJae Park + */ + +#ifdef CONFIG_DAMON_KUNIT_TEST + +#ifndef _DAMON_CORE_TEST_H +#define _DAMON_CORE_TEST_H + +#include + +static void damon_test_regions(struct kunit *test) +{ + struct damon_region *r; + struct damon_target *t; + + r = damon_new_region(1, 2); + KUNIT_EXPECT_EQ(test, 1ul, r->ar.start); + KUNIT_EXPECT_EQ(test, 2ul, r->ar.end); + KUNIT_EXPECT_EQ(test, 0u, r->nr_accesses); + + t = damon_new_target(42); + KUNIT_EXPECT_EQ(test, 0u, damon_nr_regions(t)); + + damon_add_region(r, t); + KUNIT_EXPECT_EQ(test, 1u, damon_nr_regions(t)); + + damon_del_region(r, t); + KUNIT_EXPECT_EQ(test, 0u, damon_nr_regions(t)); + + damon_free_target(t); +} + +static unsigned int nr_damon_targets(struct damon_ctx *ctx) +{ + struct damon_target *t; + unsigned int nr_targets = 0; + + damon_for_each_target(t, ctx) + nr_targets++; + + return nr_targets; +} + +static void damon_test_target(struct kunit *test) +{ + struct damon_ctx *c = damon_new_ctx(); + struct damon_target *t; + + t = damon_new_target(42); + KUNIT_EXPECT_EQ(test, 42ul, t->id); + KUNIT_EXPECT_EQ(test, 0u, nr_damon_targets(c)); + + damon_add_target(c, t); + KUNIT_EXPECT_EQ(test, 1u, nr_damon_targets(c)); + + damon_destroy_target(t); + KUNIT_EXPECT_EQ(test, 0u, nr_damon_targets(c)); + + damon_destroy_ctx(c); +} + +/* + * Test kdamond_reset_aggregated() + * + * DAMON checks access to each region and aggregates this information as the + * access frequency of each region. In detail, it increases '->nr_accesses' of + * regions that an access has confirmed. 'kdamond_reset_aggregated()' flushes + * the aggregated information ('->nr_accesses' of each regions) to the result + * buffer. As a result of the flushing, the '->nr_accesses' of regions are + * initialized to zero. + */ +static void damon_test_aggregate(struct kunit *test) +{ + struct damon_ctx *ctx = damon_new_ctx(); + unsigned long target_ids[] = {1, 2, 3}; + unsigned long saddr[][3] = {{10, 20, 30}, {5, 42, 49}, {13, 33, 55} }; + unsigned long eaddr[][3] = {{15, 27, 40}, {31, 45, 55}, {23, 44, 66} }; + unsigned long accesses[][3] = {{42, 95, 84}, {10, 20, 30}, {0, 1, 2} }; + struct damon_target *t; + struct damon_region *r; + int it, ir; + + damon_set_targets(ctx, target_ids, 3); + + it = 0; + damon_for_each_target(t, ctx) { + for (ir = 0; ir < 3; ir++) { + r = damon_new_region(saddr[it][ir], eaddr[it][ir]); + r->nr_accesses = accesses[it][ir]; + damon_add_region(r, t); + } + it++; + } + kdamond_reset_aggregated(ctx); + it = 0; + damon_for_each_target(t, ctx) { + ir = 0; + /* '->nr_accesses' should be zeroed */ + damon_for_each_region(r, t) { + KUNIT_EXPECT_EQ(test, 0u, r->nr_accesses); + ir++; + } + /* regions should be preserved */ + KUNIT_EXPECT_EQ(test, 3, ir); + it++; + } + /* targets also should be preserved */ + KUNIT_EXPECT_EQ(test, 3, it); + + damon_destroy_ctx(ctx); +} + +static void damon_test_split_at(struct kunit *test) +{ + struct damon_ctx *c = damon_new_ctx(); + struct damon_target *t; + struct damon_region *r; + + t = damon_new_target(42); + r = damon_new_region(0, 100); + damon_add_region(r, t); + damon_split_region_at(c, t, r, 25); + KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); + KUNIT_EXPECT_EQ(test, r->ar.end, 25ul); + + r = damon_next_region(r); + KUNIT_EXPECT_EQ(test, r->ar.start, 25ul); + KUNIT_EXPECT_EQ(test, r->ar.end, 100ul); + + damon_free_target(t); + damon_destroy_ctx(c); +} + +static void damon_test_merge_two(struct kunit *test) +{ + struct damon_target *t; + struct damon_region *r, *r2, *r3; + int i; + + t = damon_new_target(42); + r = damon_new_region(0, 100); + r->nr_accesses = 10; + damon_add_region(r, t); + r2 = damon_new_region(100, 300); + r2->nr_accesses = 20; + damon_add_region(r2, t); + + damon_merge_two_regions(t, r, r2); + KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); + KUNIT_EXPECT_EQ(test, r->ar.end, 300ul); + KUNIT_EXPECT_EQ(test, r->nr_accesses, 16u); + + i = 0; + damon_for_each_region(r3, t) { + KUNIT_EXPECT_PTR_EQ(test, r, r3); + i++; + } + KUNIT_EXPECT_EQ(test, i, 1); + + damon_free_target(t); +} + +static struct damon_region *__nth_region_of(struct damon_target *t, int idx) +{ + struct damon_region *r; + unsigned int i = 0; + + damon_for_each_region(r, t) { + if (i++ == idx) + return r; + } + + return NULL; +} + +static void damon_test_merge_regions_of(struct kunit *test) +{ + struct damon_target *t; + struct damon_region *r; + unsigned long sa[] = {0, 100, 114, 122, 130, 156, 170, 184}; + unsigned long ea[] = {100, 112, 122, 130, 156, 170, 184, 230}; + unsigned int nrs[] = {0, 0, 10, 10, 20, 30, 1, 2}; + + unsigned long saddrs[] = {0, 114, 130, 156, 170}; + unsigned long eaddrs[] = {112, 130, 156, 170, 230}; + int i; + + t = damon_new_target(42); + for (i = 0; i < ARRAY_SIZE(sa); i++) { + r = damon_new_region(sa[i], ea[i]); + r->nr_accesses = nrs[i]; + damon_add_region(r, t); + } + + damon_merge_regions_of(t, 9, 9999); + /* 0-112, 114-130, 130-156, 156-170 */ + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 5u); + for (i = 0; i < 5; i++) { + r = __nth_region_of(t, i); + KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]); + KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]); + } + damon_free_target(t); +} + +static void damon_test_split_regions_of(struct kunit *test) +{ + struct damon_ctx *c = damon_new_ctx(); + struct damon_target *t; + struct damon_region *r; + + t = damon_new_target(42); + r = damon_new_region(0, 22); + damon_add_region(r, t); + damon_split_regions_of(c, t, 2); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2u); + damon_free_target(t); + + t = damon_new_target(42); + r = damon_new_region(0, 220); + damon_add_region(r, t); + damon_split_regions_of(c, t, 4); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 4u); + damon_free_target(t); + damon_destroy_ctx(c); +} + +static struct kunit_case damon_test_cases[] = { + KUNIT_CASE(damon_test_target), + KUNIT_CASE(damon_test_regions), + KUNIT_CASE(damon_test_aggregate), + KUNIT_CASE(damon_test_split_at), + KUNIT_CASE(damon_test_merge_two), + KUNIT_CASE(damon_test_merge_regions_of), + KUNIT_CASE(damon_test_split_regions_of), + {}, +}; + +static struct kunit_suite damon_test_suite = { + .name = "damon", + .test_cases = damon_test_cases, +}; +kunit_test_suite(damon_test_suite); + +#endif /* _DAMON_CORE_TEST_H */ + +#endif /* CONFIG_DAMON_KUNIT_TEST */ diff --git a/mm/damon/core.c b/mm/damon/core.c index 59033488402e..30e9211f494a 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -16,6 +16,11 @@ #define CREATE_TRACE_POINTS #include +#ifdef CONFIG_DAMON_KUNIT_TEST +#undef DAMON_MIN_REGION +#define DAMON_MIN_REGION 1 +#endif + /* Get a random number in [l, r) */ #define damon_rand(l, r) (l + prandom_u32_max(r - l)) @@ -711,3 +716,5 @@ static int kdamond_fn(void *data) do_exit(0); } + +#include "core-test.h" diff --git a/mm/damon/dbgfs-test.h b/mm/damon/dbgfs-test.h new file mode 100644 index 000000000000..930e83bceef0 --- /dev/null +++ b/mm/damon/dbgfs-test.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * DAMON Debugfs Interface Unit Tests + * + * Author: SeongJae Park + */ + +#ifdef CONFIG_DAMON_DBGFS_KUNIT_TEST + +#ifndef _DAMON_DBGFS_TEST_H +#define _DAMON_DBGFS_TEST_H + +#include + +static void damon_dbgfs_test_str_to_target_ids(struct kunit *test) +{ + char *question; + unsigned long *answers; + unsigned long expected[] = {12, 35, 46}; + ssize_t nr_integers = 0, i; + + question = "123"; + answers = str_to_target_ids(question, strnlen(question, 128), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)1, nr_integers); + KUNIT_EXPECT_EQ(test, 123ul, answers[0]); + kfree(answers); + + question = "123abc"; + answers = str_to_target_ids(question, strnlen(question, 128), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)1, nr_integers); + KUNIT_EXPECT_EQ(test, 123ul, answers[0]); + kfree(answers); + + question = "a123"; + answers = str_to_target_ids(question, strnlen(question, 128), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers); + kfree(answers); + + question = "12 35"; + answers = str_to_target_ids(question, strnlen(question, 128), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)2, nr_integers); + for (i = 0; i < nr_integers; i++) + KUNIT_EXPECT_EQ(test, expected[i], answers[i]); + kfree(answers); + + question = "12 35 46"; + answers = str_to_target_ids(question, strnlen(question, 128), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)3, nr_integers); + for (i = 0; i < nr_integers; i++) + KUNIT_EXPECT_EQ(test, expected[i], answers[i]); + kfree(answers); + + question = "12 35 abc 46"; + answers = str_to_target_ids(question, strnlen(question, 128), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)2, nr_integers); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, expected[i], answers[i]); + kfree(answers); + + question = ""; + answers = str_to_target_ids(question, strnlen(question, 128), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers); + kfree(answers); + + question = "\n"; + answers = str_to_target_ids(question, strnlen(question, 128), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers); + kfree(answers); +} + +static void damon_dbgfs_test_set_targets(struct kunit *test) +{ + struct damon_ctx *ctx = dbgfs_new_ctx(); + unsigned long ids[] = {1, 2, 3}; + char buf[64]; + + /* Make DAMON consider target id as plain number */ + ctx->primitive.target_valid = NULL; + ctx->primitive.cleanup = NULL; + + damon_set_targets(ctx, ids, 3); + sprint_target_ids(ctx, buf, 64); + KUNIT_EXPECT_STREQ(test, (char *)buf, "1 2 3\n"); + + damon_set_targets(ctx, NULL, 0); + sprint_target_ids(ctx, buf, 64); + KUNIT_EXPECT_STREQ(test, (char *)buf, "\n"); + + damon_set_targets(ctx, (unsigned long []){1, 2}, 2); + sprint_target_ids(ctx, buf, 64); + KUNIT_EXPECT_STREQ(test, (char *)buf, "1 2\n"); + + damon_set_targets(ctx, (unsigned long []){2}, 1); + sprint_target_ids(ctx, buf, 64); + KUNIT_EXPECT_STREQ(test, (char *)buf, "2\n"); + + damon_set_targets(ctx, NULL, 0); + sprint_target_ids(ctx, buf, 64); + KUNIT_EXPECT_STREQ(test, (char *)buf, "\n"); + + dbgfs_destroy_ctx(ctx); +} + +static struct kunit_case damon_test_cases[] = { + KUNIT_CASE(damon_dbgfs_test_str_to_target_ids), + KUNIT_CASE(damon_dbgfs_test_set_targets), + {}, +}; + +static struct kunit_suite damon_test_suite = { + .name = "damon-dbgfs", + .test_cases = damon_test_cases, +}; +kunit_test_suite(damon_test_suite); + +#endif /* _DAMON_TEST_H */ + +#endif /* CONFIG_DAMON_KUNIT_TEST */ diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 31ad550ecba2..faee070977d8 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -619,3 +619,5 @@ static int __init damon_dbgfs_init(void) } module_init(damon_dbgfs_init); + +#include "dbgfs-test.h" diff --git a/mm/damon/vaddr-test.h b/mm/damon/vaddr-test.h new file mode 100644 index 000000000000..1f5c13257dba --- /dev/null +++ b/mm/damon/vaddr-test.h @@ -0,0 +1,329 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Data Access Monitor Unit Tests + * + * Copyright 2019 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Author: SeongJae Park + */ + +#ifdef CONFIG_DAMON_VADDR_KUNIT_TEST + +#ifndef _DAMON_VADDR_TEST_H +#define _DAMON_VADDR_TEST_H + +#include + +static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas) +{ + int i, j; + unsigned long largest_gap, gap; + + if (!nr_vmas) + return; + + for (i = 0; i < nr_vmas - 1; i++) { + vmas[i].vm_next = &vmas[i + 1]; + + vmas[i].vm_rb.rb_left = NULL; + vmas[i].vm_rb.rb_right = &vmas[i + 1].vm_rb; + + largest_gap = 0; + for (j = i; j < nr_vmas; j++) { + if (j == 0) + continue; + gap = vmas[j].vm_start - vmas[j - 1].vm_end; + if (gap > largest_gap) + largest_gap = gap; + } + vmas[i].rb_subtree_gap = largest_gap; + } + vmas[i].vm_next = NULL; + vmas[i].vm_rb.rb_right = NULL; + vmas[i].rb_subtree_gap = 0; +} + +/* + * Test __damon_va_three_regions() function + * + * In case of virtual memory address spaces monitoring, DAMON converts the + * complex and dynamic memory mappings of each target task to three + * discontiguous regions which cover every mapped areas. However, the three + * regions should not include the two biggest unmapped areas in the original + * mapping, because the two biggest areas are normally the areas between 1) + * heap and the mmap()-ed regions, and 2) the mmap()-ed regions and stack. + * Because these two unmapped areas are very huge but obviously never accessed, + * covering the region is just a waste. + * + * '__damon_va_three_regions() receives an address space of a process. It + * first identifies the start of mappings, end of mappings, and the two biggest + * unmapped areas. After that, based on the information, it constructs the + * three regions and returns. For more detail, refer to the comment of + * 'damon_init_regions_of()' function definition in 'mm/damon.c' file. + * + * For example, suppose virtual address ranges of 10-20, 20-25, 200-210, + * 210-220, 300-305, and 307-330 (Other comments represent this mappings in + * more short form: 10-20-25, 200-210-220, 300-305, 307-330) of a process are + * mapped. To cover every mappings, the three regions should start with 10, + * and end with 305. The process also has three unmapped areas, 25-200, + * 220-300, and 305-307. Among those, 25-200 and 220-300 are the biggest two + * unmapped areas, and thus it should be converted to three regions of 10-25, + * 200-220, and 300-330. + */ +static void damon_test_three_regions_in_vmas(struct kunit *test) +{ + struct damon_addr_range regions[3] = {0,}; + /* 10-20-25, 200-210-220, 300-305, 307-330 */ + struct vm_area_struct vmas[] = { + (struct vm_area_struct) {.vm_start = 10, .vm_end = 20}, + (struct vm_area_struct) {.vm_start = 20, .vm_end = 25}, + (struct vm_area_struct) {.vm_start = 200, .vm_end = 210}, + (struct vm_area_struct) {.vm_start = 210, .vm_end = 220}, + (struct vm_area_struct) {.vm_start = 300, .vm_end = 305}, + (struct vm_area_struct) {.vm_start = 307, .vm_end = 330}, + }; + + __link_vmas(vmas, 6); + + __damon_va_three_regions(&vmas[0], regions); + + KUNIT_EXPECT_EQ(test, 10ul, regions[0].start); + KUNIT_EXPECT_EQ(test, 25ul, regions[0].end); + KUNIT_EXPECT_EQ(test, 200ul, regions[1].start); + KUNIT_EXPECT_EQ(test, 220ul, regions[1].end); + KUNIT_EXPECT_EQ(test, 300ul, regions[2].start); + KUNIT_EXPECT_EQ(test, 330ul, regions[2].end); +} + +static struct damon_region *__nth_region_of(struct damon_target *t, int idx) +{ + struct damon_region *r; + unsigned int i = 0; + + damon_for_each_region(r, t) { + if (i++ == idx) + return r; + } + + return NULL; +} + +/* + * Test 'damon_va_apply_three_regions()' + * + * test kunit object + * regions an array containing start/end addresses of current + * monitoring target regions + * nr_regions the number of the addresses in 'regions' + * three_regions The three regions that need to be applied now + * expected start/end addresses of monitoring target regions that + * 'three_regions' are applied + * nr_expected the number of addresses in 'expected' + * + * The memory mapping of the target processes changes dynamically. To follow + * the change, DAMON periodically reads the mappings, simplifies it to the + * three regions, and updates the monitoring target regions to fit in the three + * regions. The update of current target regions is the role of + * 'damon_va_apply_three_regions()'. + * + * This test passes the given target regions and the new three regions that + * need to be applied to the function and check whether it updates the regions + * as expected. + */ +static void damon_do_test_apply_three_regions(struct kunit *test, + unsigned long *regions, int nr_regions, + struct damon_addr_range *three_regions, + unsigned long *expected, int nr_expected) +{ + struct damon_ctx *ctx = damon_new_ctx(); + struct damon_target *t; + struct damon_region *r; + int i; + + t = damon_new_target(42); + for (i = 0; i < nr_regions / 2; i++) { + r = damon_new_region(regions[i * 2], regions[i * 2 + 1]); + damon_add_region(r, t); + } + damon_add_target(ctx, t); + + damon_va_apply_three_regions(t, three_regions); + + for (i = 0; i < nr_expected / 2; i++) { + r = __nth_region_of(t, i); + KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]); + KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]); + } + + damon_destroy_ctx(ctx); +} + +/* + * This function test most common case where the three big regions are only + * slightly changed. Target regions should adjust their boundary (10-20-30, + * 50-55, 70-80, 90-100) to fit with the new big regions or remove target + * regions (57-79) that now out of the three regions. + */ +static void damon_test_apply_three_regions1(struct kunit *test) +{ + /* 10-20-30, 50-55-57-59, 70-80-90-100 */ + unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59, + 70, 80, 80, 90, 90, 100}; + /* 5-27, 45-55, 73-104 */ + struct damon_addr_range new_three_regions[3] = { + (struct damon_addr_range){.start = 5, .end = 27}, + (struct damon_addr_range){.start = 45, .end = 55}, + (struct damon_addr_range){.start = 73, .end = 104} }; + /* 5-20-27, 45-55, 73-80-90-104 */ + unsigned long expected[] = {5, 20, 20, 27, 45, 55, + 73, 80, 80, 90, 90, 104}; + + damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions), + new_three_regions, expected, ARRAY_SIZE(expected)); +} + +/* + * Test slightly bigger change. Similar to above, but the second big region + * now require two target regions (50-55, 57-59) to be removed. + */ +static void damon_test_apply_three_regions2(struct kunit *test) +{ + /* 10-20-30, 50-55-57-59, 70-80-90-100 */ + unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59, + 70, 80, 80, 90, 90, 100}; + /* 5-27, 56-57, 65-104 */ + struct damon_addr_range new_three_regions[3] = { + (struct damon_addr_range){.start = 5, .end = 27}, + (struct damon_addr_range){.start = 56, .end = 57}, + (struct damon_addr_range){.start = 65, .end = 104} }; + /* 5-20-27, 56-57, 65-80-90-104 */ + unsigned long expected[] = {5, 20, 20, 27, 56, 57, + 65, 80, 80, 90, 90, 104}; + + damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions), + new_three_regions, expected, ARRAY_SIZE(expected)); +} + +/* + * Test a big change. The second big region has totally freed and mapped to + * different area (50-59 -> 61-63). The target regions which were in the old + * second big region (50-55-57-59) should be removed and new target region + * covering the second big region (61-63) should be created. + */ +static void damon_test_apply_three_regions3(struct kunit *test) +{ + /* 10-20-30, 50-55-57-59, 70-80-90-100 */ + unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59, + 70, 80, 80, 90, 90, 100}; + /* 5-27, 61-63, 65-104 */ + struct damon_addr_range new_three_regions[3] = { + (struct damon_addr_range){.start = 5, .end = 27}, + (struct damon_addr_range){.start = 61, .end = 63}, + (struct damon_addr_range){.start = 65, .end = 104} }; + /* 5-20-27, 61-63, 65-80-90-104 */ + unsigned long expected[] = {5, 20, 20, 27, 61, 63, + 65, 80, 80, 90, 90, 104}; + + damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions), + new_three_regions, expected, ARRAY_SIZE(expected)); +} + +/* + * Test another big change. Both of the second and third big regions (50-59 + * and 70-100) has totally freed and mapped to different area (30-32 and + * 65-68). The target regions which were in the old second and third big + * regions should now be removed and new target regions covering the new second + * and third big regions should be crated. + */ +static void damon_test_apply_three_regions4(struct kunit *test) +{ + /* 10-20-30, 50-55-57-59, 70-80-90-100 */ + unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59, + 70, 80, 80, 90, 90, 100}; + /* 5-7, 30-32, 65-68 */ + struct damon_addr_range new_three_regions[3] = { + (struct damon_addr_range){.start = 5, .end = 7}, + (struct damon_addr_range){.start = 30, .end = 32}, + (struct damon_addr_range){.start = 65, .end = 68} }; + /* expect 5-7, 30-32, 65-68 */ + unsigned long expected[] = {5, 7, 30, 32, 65, 68}; + + damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions), + new_three_regions, expected, ARRAY_SIZE(expected)); +} + +static void damon_test_split_evenly(struct kunit *test) +{ + struct damon_ctx *c = damon_new_ctx(); + struct damon_target *t; + struct damon_region *r; + unsigned long i; + + KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(NULL, NULL, 5), + -EINVAL); + + t = damon_new_target(42); + r = damon_new_region(0, 100); + KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 0), -EINVAL); + + damon_add_region(r, t); + KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 10), 0); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 10u); + + i = 0; + damon_for_each_region(r, t) { + KUNIT_EXPECT_EQ(test, r->ar.start, i++ * 10); + KUNIT_EXPECT_EQ(test, r->ar.end, i * 10); + } + damon_free_target(t); + + t = damon_new_target(42); + r = damon_new_region(5, 59); + damon_add_region(r, t); + KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 5), 0); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 5u); + + i = 0; + damon_for_each_region(r, t) { + if (i == 4) + break; + KUNIT_EXPECT_EQ(test, r->ar.start, 5 + 10 * i++); + KUNIT_EXPECT_EQ(test, r->ar.end, 5 + 10 * i); + } + KUNIT_EXPECT_EQ(test, r->ar.start, 5 + 10 * i); + KUNIT_EXPECT_EQ(test, r->ar.end, 59ul); + damon_free_target(t); + + t = damon_new_target(42); + r = damon_new_region(5, 6); + damon_add_region(r, t); + KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 2), -EINVAL); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1u); + + damon_for_each_region(r, t) { + KUNIT_EXPECT_EQ(test, r->ar.start, 5ul); + KUNIT_EXPECT_EQ(test, r->ar.end, 6ul); + } + damon_free_target(t); + damon_destroy_ctx(c); +} + +static struct kunit_case damon_test_cases[] = { + KUNIT_CASE(damon_test_three_regions_in_vmas), + KUNIT_CASE(damon_test_apply_three_regions1), + KUNIT_CASE(damon_test_apply_three_regions2), + KUNIT_CASE(damon_test_apply_three_regions3), + KUNIT_CASE(damon_test_apply_three_regions4), + KUNIT_CASE(damon_test_split_evenly), + {}, +}; + +static struct kunit_suite damon_test_suite = { + .name = "damon-primitives", + .test_cases = damon_test_cases, +}; +kunit_test_suite(damon_test_suite); + +#endif /* _DAMON_VADDR_TEST_H */ + +#endif /* CONFIG_DAMON_VADDR_KUNIT_TEST */ diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 897aa8cf96c8..58c1fb2aafa9 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -18,6 +18,11 @@ #include #include +#ifdef CONFIG_DAMON_VADDR_KUNIT_TEST +#undef DAMON_MIN_REGION +#define DAMON_MIN_REGION 1 +#endif + /* Get a random number in [l, r) */ #define damon_rand(l, r) (l + prandom_u32_max(r - l)) @@ -663,3 +668,5 @@ void damon_va_set_primitives(struct damon_ctx *ctx) ctx->primitive.target_valid = damon_va_target_valid; ctx->primitive.cleanup = NULL; } + +#include "vaddr-test.h" From b25e76d9c35c975f0916bc444b971b60ee3c1d50 Mon Sep 17 00:00:00 2001 From: Adam Borowski Date: Fri, 24 Sep 2021 15:43:26 -0700 Subject: [PATCH 057/156] UPSTREAM: mm/damon: don't use strnlen() with known-bogus source length gcc knows the true length too, and rightfully complains. Link: https://lkml.kernel.org/r/20210912204447.10427-1-kilobyte@angband.pl Signed-off-by: Adam Borowski Cc: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 892ab4bbd063cfe7f6bbb183e6be69d9907a61de) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ia3ad91fd9541a2e57156a999c5fe3e70efe740fb --- mm/damon/dbgfs-test.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mm/damon/dbgfs-test.h b/mm/damon/dbgfs-test.h index 930e83bceef0..4eddcfa73996 100644 --- a/mm/damon/dbgfs-test.h +++ b/mm/damon/dbgfs-test.h @@ -20,27 +20,27 @@ static void damon_dbgfs_test_str_to_target_ids(struct kunit *test) ssize_t nr_integers = 0, i; question = "123"; - answers = str_to_target_ids(question, strnlen(question, 128), + answers = str_to_target_ids(question, strlen(question), &nr_integers); KUNIT_EXPECT_EQ(test, (ssize_t)1, nr_integers); KUNIT_EXPECT_EQ(test, 123ul, answers[0]); kfree(answers); question = "123abc"; - answers = str_to_target_ids(question, strnlen(question, 128), + answers = str_to_target_ids(question, strlen(question), &nr_integers); KUNIT_EXPECT_EQ(test, (ssize_t)1, nr_integers); KUNIT_EXPECT_EQ(test, 123ul, answers[0]); kfree(answers); question = "a123"; - answers = str_to_target_ids(question, strnlen(question, 128), + answers = str_to_target_ids(question, strlen(question), &nr_integers); KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers); kfree(answers); question = "12 35"; - answers = str_to_target_ids(question, strnlen(question, 128), + answers = str_to_target_ids(question, strlen(question), &nr_integers); KUNIT_EXPECT_EQ(test, (ssize_t)2, nr_integers); for (i = 0; i < nr_integers; i++) @@ -48,7 +48,7 @@ static void damon_dbgfs_test_str_to_target_ids(struct kunit *test) kfree(answers); question = "12 35 46"; - answers = str_to_target_ids(question, strnlen(question, 128), + answers = str_to_target_ids(question, strlen(question), &nr_integers); KUNIT_EXPECT_EQ(test, (ssize_t)3, nr_integers); for (i = 0; i < nr_integers; i++) @@ -56,7 +56,7 @@ static void damon_dbgfs_test_str_to_target_ids(struct kunit *test) kfree(answers); question = "12 35 abc 46"; - answers = str_to_target_ids(question, strnlen(question, 128), + answers = str_to_target_ids(question, strlen(question), &nr_integers); KUNIT_EXPECT_EQ(test, (ssize_t)2, nr_integers); for (i = 0; i < 2; i++) @@ -64,13 +64,13 @@ static void damon_dbgfs_test_str_to_target_ids(struct kunit *test) kfree(answers); question = ""; - answers = str_to_target_ids(question, strnlen(question, 128), + answers = str_to_target_ids(question, strlen(question), &nr_integers); KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers); kfree(answers); question = "\n"; - answers = str_to_target_ids(question, strnlen(question, 128), + answers = str_to_target_ids(question, strlen(question), &nr_integers); KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers); kfree(answers); From e31da16e693b2b653b7363a7e6112f7fd105914d Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Thu, 28 Oct 2021 14:36:33 -0700 Subject: [PATCH 058/156] UPSTREAM: mm/damon/core-test: fix wrong expectations for 'damon_split_regions_of()' Kunit test cases for 'damon_split_regions_of()' expects the number of regions after calling the function will be same to their request ('nr_sub'). However, the requested number is just an upper-limit, because the function randomly decides the size of each sub-region. This fixes the wrong expectation. Link: https://lkml.kernel.org/r/20211028090628.14948-1-sj@kernel.org Fixes: 17ccae8bb5c9 ("mm/damon: add kunit tests") Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 2e014660b3e4b7bd0e75f845cdcf745c0f632889) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I3237d82880ca4308dd4161de9dd957925384a333 --- mm/damon/core-test.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/damon/core-test.h b/mm/damon/core-test.h index c938a9c34e6c..7008c3735e99 100644 --- a/mm/damon/core-test.h +++ b/mm/damon/core-test.h @@ -219,14 +219,14 @@ static void damon_test_split_regions_of(struct kunit *test) r = damon_new_region(0, 22); damon_add_region(r, t); damon_split_regions_of(c, t, 2); - KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2u); + KUNIT_EXPECT_LE(test, damon_nr_regions(t), 2u); damon_free_target(t); t = damon_new_target(42); r = damon_new_region(0, 220); damon_add_region(r, t); damon_split_regions_of(c, t, 4); - KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 4u); + KUNIT_EXPECT_LE(test, damon_nr_regions(t), 4u); damon_free_target(t); damon_destroy_ctx(c); } From 8c5ef4d641587378bf61dcef1a1dec20c583f483 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 5 Nov 2021 13:45:52 -0700 Subject: [PATCH 059/156] UPSTREAM: mm/damon: grammar s/works/work/ Correct a singular versus plural grammar mistake in the help text for the DAMON_VADDR config symbol. Link: https://lkml.kernel.org/r/20210914073451.3883834-1-geert@linux-m68k.org Fixes: 3f49584b262cf8f4 ("mm/damon: implement primitives for the virtual memory address spaces") Signed-off-by: Geert Uytterhoeven Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit f24b0626076783d56ef41c6459fedf70ab6dcbd0) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I194d0315cb93c2923d066092432e7dc997abe4e0 --- mm/damon/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig index 37024798a97c..ba8898c7eb8e 100644 --- a/mm/damon/Kconfig +++ b/mm/damon/Kconfig @@ -30,7 +30,7 @@ config DAMON_VADDR select PAGE_IDLE_FLAG help This builds the default data access monitoring primitives for DAMON - that works for virtual address spaces. + that work for virtual address spaces. config DAMON_VADDR_KUNIT_TEST bool "Test for DAMON primitives" if !KUNIT_ALL_TESTS From 6239bbbcea5f2ee4237c1619934d844d1e670b03 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:04 -0700 Subject: [PATCH 060/156] UPSTREAM: include/linux/damon.h: fix kernel-doc comments for 'damon_callback' A few Kernel-doc comments in 'damon.h' are broken. This fixes them. Link: https://lkml.kernel.org/r/20210917123958.3819-5-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit d2f272b35a84ace2ef04334a9822fd726a7f061b) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ic57dd7ca6528303cc07f2dca16487820ac100650 --- include/linux/damon.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index d68b67b8d458..755d70804705 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -62,7 +62,7 @@ struct damon_target { struct damon_ctx; /** - * struct damon_primitive Monitoring primitives for given use cases. + * struct damon_primitive - Monitoring primitives for given use cases. * * @init: Initialize primitive-internal data structures. * @update: Update primitive-internal data structures. @@ -108,8 +108,8 @@ struct damon_primitive { void (*cleanup)(struct damon_ctx *context); }; -/* - * struct damon_callback Monitoring events notification callbacks. +/** + * struct damon_callback - Monitoring events notification callbacks. * * @before_start: Called before starting the monitoring. * @after_sampling: Called after each sampling. From 526b5029ad4a1857771326050c24a7a0e7255a2e Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:06 -0700 Subject: [PATCH 061/156] UPSTREAM: mm/damon/core: print kdamond start log in debug mode only Logging of kdamond startup is using 'pr_info()' unnecessarily. This makes it to use 'pr_debug()' instead. Link: https://lkml.kernel.org/r/20210917123958.3819-6-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Cc: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 704571f997424ecd64b10b37ca6097e65690240a) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Idf59dd43a8cecfbbe2846bfa81a8c79744ce08c9 --- mm/damon/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 30e9211f494a..874558a790a0 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -653,7 +653,7 @@ static int kdamond_fn(void *data) unsigned long sz_limit = 0; mutex_lock(&ctx->kdamond_lock); - pr_info("kdamond (%d) starts\n", ctx->kdamond->pid); + pr_debug("kdamond (%d) starts\n", ctx->kdamond->pid); mutex_unlock(&ctx->kdamond_lock); if (ctx->primitive.init) From bcf5bbcaf438ef480e925668251b27c5bf346e99 Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Fri, 5 Nov 2021 13:46:09 -0700 Subject: [PATCH 062/156] UPSTREAM: mm/damon: remove unnecessary do_exit() from kdamond Just return from the kthread function. Link: https://lkml.kernel.org/r/20210927232421.17694-1-changbin.du@gmail.com Signed-off-by: Changbin Du Cc: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 5f7fe2b9b827662cf349ab45406d6cbf0cc6251f) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I3915c829f26c2d6e0e983156bf0229d067b03f4f --- mm/damon/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 874558a790a0..61a9e3b37bc9 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -714,7 +714,7 @@ static int kdamond_fn(void *data) nr_running_ctxs--; mutex_unlock(&damon_lock); - do_exit(0); + return 0; } #include "core-test.h" From e0fad2fbbe3adfce89834de118ca990513d9758e Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Fri, 5 Nov 2021 13:46:12 -0700 Subject: [PATCH 063/156] UPSTREAM: mm/damon: needn't hold kdamond_lock to print pid of kdamond Just get the pid by 'current->pid'. Meanwhile, to be symmetrical make the 'starts' and 'finishes' logs both use debug level. Link: https://lkml.kernel.org/r/20210927232432.17750-1-changbin.du@gmail.com Signed-off-by: Changbin Du Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 42e4cef5fe48333e0db6e98b019edf5f2c2f11fd) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I6ea6f697795a43663f7c20e87c07cddae891a231 --- mm/damon/core.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 61a9e3b37bc9..8171e7dddc30 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -652,9 +652,7 @@ static int kdamond_fn(void *data) unsigned int max_nr_accesses = 0; unsigned long sz_limit = 0; - mutex_lock(&ctx->kdamond_lock); - pr_debug("kdamond (%d) starts\n", ctx->kdamond->pid); - mutex_unlock(&ctx->kdamond_lock); + pr_debug("kdamond (%d) starts\n", current->pid); if (ctx->primitive.init) ctx->primitive.init(ctx); @@ -705,7 +703,7 @@ static int kdamond_fn(void *data) if (ctx->primitive.cleanup) ctx->primitive.cleanup(ctx); - pr_debug("kdamond (%d) finishes\n", ctx->kdamond->pid); + pr_debug("kdamond (%d) finishes\n", current->pid); mutex_lock(&ctx->kdamond_lock); ctx->kdamond = NULL; mutex_unlock(&ctx->kdamond_lock); From b1209ff347fc2073221e6af47cc4a8742cefb45b Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 5 Nov 2021 13:46:15 -0700 Subject: [PATCH 064/156] UPSTREAM: mm/damon/core: nullify pointer ctx->kdamond with a NULL Currently a plain integer is being used to nullify the pointer ctx->kdamond. Use NULL instead. Cleans up sparse warning: mm/damon/core.c:317:40: warning: Using plain integer as NULL pointer Link: https://lkml.kernel.org/r/20210925215908.181226-1-colin.king@canonical.com Signed-off-by: Colin Ian King Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 7ec1992b891e59dba0f04e0327980786e8f61b13) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Id5f9786633a785fd45bb6b25f0765671a21d3458 --- mm/damon/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 8171e7dddc30..d993db50280c 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -314,7 +314,7 @@ static int __damon_start(struct damon_ctx *ctx) nr_running_ctxs); if (IS_ERR(ctx->kdamond)) { err = PTR_ERR(ctx->kdamond); - ctx->kdamond = 0; + ctx->kdamond = NULL; } } mutex_unlock(&ctx->kdamond_lock); From 3d9ce6d28bf79b986ac9d8f2b72a20a725f756aa Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:18 -0700 Subject: [PATCH 065/156] UPSTREAM: mm/damon/core: account age of target regions Patch series "Implement Data Access Monitoring-based Memory Operation Schemes". Introduction ============ DAMON[1] can be used as a primitive for data access aware memory management optimizations. For that, users who want such optimizations should run DAMON, read the monitoring results, analyze it, plan a new memory management scheme, and apply the new scheme by themselves. Such efforts will be inevitable for some complicated optimizations. However, in many other cases, the users would simply want the system to apply a memory management action to a memory region of a specific size having a specific access frequency for a specific time. For example, "page out a memory region larger than 100 MiB keeping only rare accesses more than 2 minutes", or "Do not use THP for a memory region larger than 2 MiB rarely accessed for more than 1 seconds". To make the works easier and non-redundant, this patchset implements a new feature of DAMON, which is called Data Access Monitoring-based Operation Schemes (DAMOS). Using the feature, users can describe the normal schemes in a simple way and ask DAMON to execute those on its own. [1] https://damonitor.github.io Evaluations =========== DAMOS is accurate and useful for memory management optimizations. An experimental DAMON-based operation scheme for THP, 'ethp', removes 76.15% of THP memory overheads while preserving 51.25% of THP speedup. Another experimental DAMON-based 'proactive reclamation' implementation, 'prcl', reduces 93.38% of residential sets and 23.63% of system memory footprint while incurring only 1.22% runtime overhead in the best case (parsec3/freqmine). NOTE that the experimental THP optimization and proactive reclamation are not for production but only for proof of concepts. Please refer to the showcase web site's evaluation document[1] for detailed evaluation setup and results. [1] https://damonitor.github.io/doc/html/v34/vm/damon/eval.html Long-term Support Trees ----------------------- For people who want to test DAMON but using LTS kernels, there are another couple of trees based on two latest LTS kernels respectively and containing the 'damon/master' backports. - For v5.4.y: https://git.kernel.org/sj/h/damon/for-v5.4.y - For v5.10.y: https://git.kernel.org/sj/h/damon/for-v5.10.y Sequence Of Patches =================== The 1st patch accounts age of each region. The 2nd patch implements the core of the DAMON-based operation schemes feature. The 3rd patch makes the default monitoring primitives for virtual address spaces to support the schemes. From this point, the kernel space users can use DAMOS. The 4th patch exports the feature to the user space via the debugfs interface. The 5th patch implements schemes statistics feature for easier tuning of the schemes and runtime access pattern analysis, and the 6th patch adds selftests for these changes. Finally, the 7th patch documents this new feature. This patch (of 7): DAMON can be used for data access pattern aware memory management optimizations. For that, users should run DAMON, read the monitoring results, analyze it, plan a new memory management scheme, and apply the new scheme by themselves. It would not be too hard, but still require some level of effort. For complicated cases, this effort is inevitable. That said, in many cases, users would simply want to apply an actions to a memory region of a specific size having a specific access frequency for a specific time. For example, "page out a memory region larger than 100 MiB but having a low access frequency more than 10 minutes", or "Use THP for a memory region larger than 2 MiB having a high access frequency for more than 2 seconds". For such optimizations, users will need to first account the age of each region themselves. To reduce such efforts, this implements a simple age account of each region in DAMON. For each aggregation step, DAMON compares the access frequency with that from last aggregation and reset the age of the region if the change is significant. Else, the age is incremented. Also, in case of the merge of regions, the region size-weighted average of the ages is set as the age of merged new region. Link: https://lkml.kernel.org/r/20211001125604.29660-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211001125604.29660-2-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Cameron Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Jonathan Corbet Cc: David Hildenbrand Cc: David Woodhouse Cc: Marco Elver Cc: Leonard Foerster Cc: Greg Thelen Cc: Markus Boehme Cc: David Rienjes Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit fda504fade7f124858d7022341dc46ff35b45274) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ia5ddb3b5ce9c0d14e098a0af55dabf4b6a609aaa --- include/linux/damon.h | 10 ++++++++++ mm/damon/core.c | 13 +++++++++++++ 2 files changed, 23 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 755d70804705..3e8215debbd4 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -31,12 +31,22 @@ struct damon_addr_range { * @sampling_addr: Address of the sample for the next access check. * @nr_accesses: Access frequency of this region. * @list: List head for siblings. + * @age: Age of this region. + * + * @age is initially zero, increased for each aggregation interval, and reset + * to zero again if the access frequency is significantly changed. If two + * regions are merged into a new region, both @nr_accesses and @age of the new + * region are set as region size-weighted average of those of the two regions. */ struct damon_region { struct damon_addr_range ar; unsigned long sampling_addr; unsigned int nr_accesses; struct list_head list; + + unsigned int age; +/* private: Internal value for age calculation. */ + unsigned int last_nr_accesses; }; /** diff --git a/mm/damon/core.c b/mm/damon/core.c index d993db50280c..3efbe80779db 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -45,6 +45,9 @@ struct damon_region *damon_new_region(unsigned long start, unsigned long end) region->nr_accesses = 0; INIT_LIST_HEAD(®ion->list); + region->age = 0; + region->last_nr_accesses = 0; + return region; } @@ -444,6 +447,7 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) damon_for_each_region(r, t) { trace_damon_aggregated(t, r, damon_nr_regions(t)); + r->last_nr_accesses = r->nr_accesses; r->nr_accesses = 0; } } @@ -461,6 +465,7 @@ static void damon_merge_two_regions(struct damon_target *t, l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / (sz_l + sz_r); + l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); l->ar.end = r->ar.end; damon_destroy_region(r, t); } @@ -480,6 +485,11 @@ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, struct damon_region *r, *prev = NULL, *next; damon_for_each_region_safe(r, next, t) { + if (diff_of(r->nr_accesses, r->last_nr_accesses) > thres) + r->age = 0; + else + r->age++; + if (prev && prev->ar.end == r->ar.start && diff_of(prev->nr_accesses, r->nr_accesses) <= thres && sz_damon_region(prev) + sz_damon_region(r) <= sz_limit) @@ -527,6 +537,9 @@ static void damon_split_region_at(struct damon_ctx *ctx, r->ar.end = new->ar.start; + new->age = r->age; + new->last_nr_accesses = r->last_nr_accesses; + damon_insert_region(new, r, damon_next_region(r), t); } From 2a437378a53ba8b80453b24e53cdeaa7368bb9a2 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:22 -0700 Subject: [PATCH 066/156] UPSTREAM: mm/damon/core: implement DAMON-based Operation Schemes (DAMOS) In many cases, users might use DAMON for simple data access aware memory management optimizations such as applying an operation scheme to a memory region of a specific size having a specific access frequency for a specific time. For example, "page out a memory region larger than 100 MiB but having a low access frequency more than 10 minutes", or "Use THP for a memory region larger than 2 MiB having a high access frequency for more than 2 seconds". Most simple form of the solution would be doing offline data access pattern profiling using DAMON and modifying the application source code or system configuration based on the profiling results. Or, developing a daemon constructed with two modules (one for access monitoring and the other for applying memory management actions via mlock(), madvise(), sysctl, etc) is imaginable. To avoid users spending their time for implementation of such simple data access monitoring-based operation schemes, this makes DAMON to handle such schemes directly. With this change, users can simply specify their desired schemes to DAMON. Then, DAMON will automatically apply the schemes to the user-specified target processes. Each of the schemes is composed with conditions for filtering of the target memory regions and desired memory management action for the target. Specifically, the format is:: The filtering conditions are size of memory region, number of accesses to the region monitored by DAMON, and the age of the region. The age of region is incremented periodically but reset when its addresses or access frequency has significantly changed or the action of a scheme was applied. For the action, current implementation supports a few of madvise()-like hints, ``WILLNEED``, ``COLD``, ``PAGEOUT``, ``HUGEPAGE``, and ``NOHUGEPAGE``. Because DAMON supports various address spaces and application of the actions to a monitoring target region is dependent to the type of the target address space, the application code should be implemented by each primitives and registered to the framework. Note that this only implements the framework part. Following commit will implement the action applications for virtual address spaces primitives. Link: https://lkml.kernel.org/r/20211001125604.29660-3-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rienjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 1f366e421c8f69583ed37b56d86e3747331869c3) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Iae8c0d0ade588de0720140fcf6f97a1873f896a0 --- include/linux/damon.h | 66 +++++++++++++++++++++++++ mm/damon/core.c | 109 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 3e8215debbd4..dbe18b0fb795 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -69,6 +69,48 @@ struct damon_target { struct list_head list; }; +/** + * enum damos_action - Represents an action of a Data Access Monitoring-based + * Operation Scheme. + * + * @DAMOS_WILLNEED: Call ``madvise()`` for the region with MADV_WILLNEED. + * @DAMOS_COLD: Call ``madvise()`` for the region with MADV_COLD. + * @DAMOS_PAGEOUT: Call ``madvise()`` for the region with MADV_PAGEOUT. + * @DAMOS_HUGEPAGE: Call ``madvise()`` for the region with MADV_HUGEPAGE. + * @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE. + */ +enum damos_action { + DAMOS_WILLNEED, + DAMOS_COLD, + DAMOS_PAGEOUT, + DAMOS_HUGEPAGE, + DAMOS_NOHUGEPAGE, +}; + +/** + * struct damos - Represents a Data Access Monitoring-based Operation Scheme. + * @min_sz_region: Minimum size of target regions. + * @max_sz_region: Maximum size of target regions. + * @min_nr_accesses: Minimum ``->nr_accesses`` of target regions. + * @max_nr_accesses: Maximum ``->nr_accesses`` of target regions. + * @min_age_region: Minimum age of target regions. + * @max_age_region: Maximum age of target regions. + * @action: &damo_action to be applied to the target regions. + * @list: List head for siblings. + * + * Note that both the minimums and the maximums are inclusive. + */ +struct damos { + unsigned long min_sz_region; + unsigned long max_sz_region; + unsigned int min_nr_accesses; + unsigned int max_nr_accesses; + unsigned int min_age_region; + unsigned int max_age_region; + enum damos_action action; + struct list_head list; +}; + struct damon_ctx; /** @@ -79,6 +121,7 @@ struct damon_ctx; * @prepare_access_checks: Prepare next access check of target regions. * @check_accesses: Check the accesses to target regions. * @reset_aggregated: Reset aggregated accesses monitoring results. + * @apply_scheme: Apply a DAMON-based operation scheme. * @target_valid: Determine if the target is valid. * @cleanup: Clean up the context. * @@ -104,6 +147,9 @@ struct damon_ctx; * of its update. The value will be used for regions adjustment threshold. * @reset_aggregated should reset the access monitoring results that aggregated * by @check_accesses. + * @apply_scheme is called from @kdamond when a region for user provided + * DAMON-based operation scheme is found. It should apply the scheme's action + * to the region. This is not used for &DAMON_ARBITRARY_TARGET case. * @target_valid should check whether the target is still valid for the * monitoring. * @cleanup is called from @kdamond just before its termination. @@ -114,6 +160,8 @@ struct damon_primitive { void (*prepare_access_checks)(struct damon_ctx *context); unsigned int (*check_accesses)(struct damon_ctx *context); void (*reset_aggregated)(struct damon_ctx *context); + int (*apply_scheme)(struct damon_ctx *context, struct damon_target *t, + struct damon_region *r, struct damos *scheme); bool (*target_valid)(void *target); void (*cleanup)(struct damon_ctx *context); }; @@ -192,6 +240,7 @@ struct damon_callback { * @min_nr_regions: The minimum number of adaptive monitoring regions. * @max_nr_regions: The maximum number of adaptive monitoring regions. * @adaptive_targets: Head of monitoring targets (&damon_target) list. + * @schemes: Head of schemes (&damos) list. */ struct damon_ctx { unsigned long sample_interval; @@ -213,6 +262,7 @@ struct damon_ctx { unsigned long min_nr_regions; unsigned long max_nr_regions; struct list_head adaptive_targets; + struct list_head schemes; }; #define damon_next_region(r) \ @@ -233,6 +283,12 @@ struct damon_ctx { #define damon_for_each_target_safe(t, next, ctx) \ list_for_each_entry_safe(t, next, &(ctx)->adaptive_targets, list) +#define damon_for_each_scheme(s, ctx) \ + list_for_each_entry(s, &(ctx)->schemes, list) + +#define damon_for_each_scheme_safe(s, next, ctx) \ + list_for_each_entry_safe(s, next, &(ctx)->schemes, list) + #ifdef CONFIG_DAMON struct damon_region *damon_new_region(unsigned long start, unsigned long end); @@ -242,6 +298,14 @@ inline void damon_insert_region(struct damon_region *r, void damon_add_region(struct damon_region *r, struct damon_target *t); void damon_destroy_region(struct damon_region *r, struct damon_target *t); +struct damos *damon_new_scheme( + unsigned long min_sz_region, unsigned long max_sz_region, + unsigned int min_nr_accesses, unsigned int max_nr_accesses, + unsigned int min_age_region, unsigned int max_age_region, + enum damos_action action); +void damon_add_scheme(struct damon_ctx *ctx, struct damos *s); +void damon_destroy_scheme(struct damos *s); + struct damon_target *damon_new_target(unsigned long id); void damon_add_target(struct damon_ctx *ctx, struct damon_target *t); void damon_free_target(struct damon_target *t); @@ -255,6 +319,8 @@ int damon_set_targets(struct damon_ctx *ctx, int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, unsigned long aggr_int, unsigned long primitive_upd_int, unsigned long min_nr_reg, unsigned long max_nr_reg); +int damon_set_schemes(struct damon_ctx *ctx, + struct damos **schemes, ssize_t nr_schemes); int damon_nr_running_ctxs(void); int damon_start(struct damon_ctx **ctxs, int nr_ctxs); diff --git a/mm/damon/core.c b/mm/damon/core.c index 3efbe80779db..0ed97b21cbb6 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -85,6 +85,50 @@ void damon_destroy_region(struct damon_region *r, struct damon_target *t) damon_free_region(r); } +struct damos *damon_new_scheme( + unsigned long min_sz_region, unsigned long max_sz_region, + unsigned int min_nr_accesses, unsigned int max_nr_accesses, + unsigned int min_age_region, unsigned int max_age_region, + enum damos_action action) +{ + struct damos *scheme; + + scheme = kmalloc(sizeof(*scheme), GFP_KERNEL); + if (!scheme) + return NULL; + scheme->min_sz_region = min_sz_region; + scheme->max_sz_region = max_sz_region; + scheme->min_nr_accesses = min_nr_accesses; + scheme->max_nr_accesses = max_nr_accesses; + scheme->min_age_region = min_age_region; + scheme->max_age_region = max_age_region; + scheme->action = action; + INIT_LIST_HEAD(&scheme->list); + + return scheme; +} + +void damon_add_scheme(struct damon_ctx *ctx, struct damos *s) +{ + list_add_tail(&s->list, &ctx->schemes); +} + +static void damon_del_scheme(struct damos *s) +{ + list_del(&s->list); +} + +static void damon_free_scheme(struct damos *s) +{ + kfree(s); +} + +void damon_destroy_scheme(struct damos *s) +{ + damon_del_scheme(s); + damon_free_scheme(s); +} + /* * Construct a damon_target struct * @@ -156,6 +200,7 @@ struct damon_ctx *damon_new_ctx(void) ctx->max_nr_regions = 1000; INIT_LIST_HEAD(&ctx->adaptive_targets); + INIT_LIST_HEAD(&ctx->schemes); return ctx; } @@ -175,7 +220,13 @@ static void damon_destroy_targets(struct damon_ctx *ctx) void damon_destroy_ctx(struct damon_ctx *ctx) { + struct damos *s, *next_s; + damon_destroy_targets(ctx); + + damon_for_each_scheme_safe(s, next_s, ctx) + damon_destroy_scheme(s); + kfree(ctx); } @@ -250,6 +301,30 @@ int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, return 0; } +/** + * damon_set_schemes() - Set data access monitoring based operation schemes. + * @ctx: monitoring context + * @schemes: array of the schemes + * @nr_schemes: number of entries in @schemes + * + * This function should not be called while the kdamond of the context is + * running. + * + * Return: 0 if success, or negative error code otherwise. + */ +int damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, + ssize_t nr_schemes) +{ + struct damos *s, *next; + ssize_t i; + + damon_for_each_scheme_safe(s, next, ctx) + damon_destroy_scheme(s); + for (i = 0; i < nr_schemes; i++) + damon_add_scheme(ctx, schemes[i]); + return 0; +} + /** * damon_nr_running_ctxs() - Return number of currently running contexts. */ @@ -453,6 +528,39 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) } } +static void damon_do_apply_schemes(struct damon_ctx *c, + struct damon_target *t, + struct damon_region *r) +{ + struct damos *s; + unsigned long sz; + + damon_for_each_scheme(s, c) { + sz = r->ar.end - r->ar.start; + if (sz < s->min_sz_region || s->max_sz_region < sz) + continue; + if (r->nr_accesses < s->min_nr_accesses || + s->max_nr_accesses < r->nr_accesses) + continue; + if (r->age < s->min_age_region || s->max_age_region < r->age) + continue; + if (c->primitive.apply_scheme) + c->primitive.apply_scheme(c, t, r, s); + r->age = 0; + } +} + +static void kdamond_apply_schemes(struct damon_ctx *c) +{ + struct damon_target *t; + struct damon_region *r; + + damon_for_each_target(t, c) { + damon_for_each_region(r, t) + damon_do_apply_schemes(c, t, r); + } +} + #define sz_damon_region(r) (r->ar.end - r->ar.start) /* @@ -693,6 +801,7 @@ static int kdamond_fn(void *data) if (ctx->callback.after_aggregation && ctx->callback.after_aggregation(ctx)) set_kdamond_stop(ctx); + kdamond_apply_schemes(ctx); kdamond_reset_aggregated(ctx); kdamond_split_regions(ctx); if (ctx->primitive.reset_aggregated) From cad23cd779bf120b432c74fa6ad9d6679db68c08 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:25 -0700 Subject: [PATCH 067/156] UPSTREAM: mm/damon/vaddr: support DAMON-based Operation Schemes This makes DAMON's default primitives for virtual address spaces to support DAMON-based Operation Schemes (DAMOS) by implementing actions application functions and registering it to the monitoring context. The implementation simply links 'madvise()' for related DAMOS actions. That is, 'madvise(MADV_WILLNEED)' is called for 'WILLNEED' DAMOS action and similar for other actions ('COLD', 'PAGEOUT', 'HUGEPAGE', 'NOHUGEPAGE'). So, the kernel space DAMON users can now use the DAMON-based optimizations with only small amount of code. Link: https://lkml.kernel.org/r/20211001125604.29660-4-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rienjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 6dea8add4d2875b80843e4a4c8acd334a4db8c8f) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I791dacf0f965deaed4a9fca155aa376764927b46 --- include/linux/damon.h | 2 ++ mm/damon/vaddr.c | 56 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index dbe18b0fb795..be6b6e81e8ee 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -337,6 +337,8 @@ void damon_va_prepare_access_checks(struct damon_ctx *ctx); unsigned int damon_va_check_accesses(struct damon_ctx *ctx); bool damon_va_target_valid(void *t); void damon_va_cleanup(struct damon_ctx *ctx); +int damon_va_apply_scheme(struct damon_ctx *context, struct damon_target *t, + struct damon_region *r, struct damos *scheme); void damon_va_set_primitives(struct damon_ctx *ctx); #endif /* CONFIG_DAMON_VADDR */ diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 58c1fb2aafa9..3e1c74d36bab 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) "damon-va: " fmt +#include #include #include #include @@ -658,6 +659,60 @@ bool damon_va_target_valid(void *target) return false; } +#ifndef CONFIG_ADVISE_SYSCALLS +static int damos_madvise(struct damon_target *target, struct damon_region *r, + int behavior) +{ + return -EINVAL; +} +#else +static int damos_madvise(struct damon_target *target, struct damon_region *r, + int behavior) +{ + struct mm_struct *mm; + int ret = -ENOMEM; + + mm = damon_get_mm(target); + if (!mm) + goto out; + + ret = do_madvise(mm, PAGE_ALIGN(r->ar.start), + PAGE_ALIGN(r->ar.end - r->ar.start), behavior); + mmput(mm); +out: + return ret; +} +#endif /* CONFIG_ADVISE_SYSCALLS */ + +int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, + struct damon_region *r, struct damos *scheme) +{ + int madv_action; + + switch (scheme->action) { + case DAMOS_WILLNEED: + madv_action = MADV_WILLNEED; + break; + case DAMOS_COLD: + madv_action = MADV_COLD; + break; + case DAMOS_PAGEOUT: + madv_action = MADV_PAGEOUT; + break; + case DAMOS_HUGEPAGE: + madv_action = MADV_HUGEPAGE; + break; + case DAMOS_NOHUGEPAGE: + madv_action = MADV_NOHUGEPAGE; + break; + default: + pr_warn("Wrong action %d\n", scheme->action); + return -EINVAL; + } + + return damos_madvise(t, r, madv_action); +} + void damon_va_set_primitives(struct damon_ctx *ctx) { ctx->primitive.init = damon_va_init; @@ -667,6 +722,7 @@ void damon_va_set_primitives(struct damon_ctx *ctx) ctx->primitive.reset_aggregated = NULL; ctx->primitive.target_valid = damon_va_target_valid; ctx->primitive.cleanup = NULL; + ctx->primitive.apply_scheme = damon_va_apply_scheme; } #include "vaddr-test.h" From 5203491dbb7586279cce096cf6db64845e4ec69c Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:29 -0700 Subject: [PATCH 068/156] UPSTREAM: mm/damon/dbgfs: support DAMON-based Operation Schemes This makes 'damon-dbgfs' to support the data access monitoring oriented memory management schemes. Users can read and update the schemes using ``/damon/schemes`` file. The format is:: Link: https://lkml.kernel.org/r/20211001125604.29660-5-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rienjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit af122dd8f3c0099349bc98ff69f0d90efd8b149f) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I0ef1d7cc491f93ae6baaefbf9dc47ee342807069 --- mm/damon/dbgfs.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 162 insertions(+), 3 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index faee070977d8..78b7a04490c5 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -98,6 +98,159 @@ out: return ret; } +static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) +{ + struct damos *s; + int written = 0; + int rc; + + damon_for_each_scheme(s, c) { + rc = scnprintf(&buf[written], len - written, + "%lu %lu %u %u %u %u %d\n", + s->min_sz_region, s->max_sz_region, + s->min_nr_accesses, s->max_nr_accesses, + s->min_age_region, s->max_age_region, + s->action); + if (!rc) + return -ENOMEM; + + written += rc; + } + return written; +} + +static ssize_t dbgfs_schemes_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char *kbuf; + ssize_t len; + + kbuf = kmalloc(count, GFP_KERNEL); + if (!kbuf) + return -ENOMEM; + + mutex_lock(&ctx->kdamond_lock); + len = sprint_schemes(ctx, kbuf, count); + mutex_unlock(&ctx->kdamond_lock); + if (len < 0) + goto out; + len = simple_read_from_buffer(buf, count, ppos, kbuf, len); + +out: + kfree(kbuf); + return len; +} + +static void free_schemes_arr(struct damos **schemes, ssize_t nr_schemes) +{ + ssize_t i; + + for (i = 0; i < nr_schemes; i++) + kfree(schemes[i]); + kfree(schemes); +} + +static bool damos_action_valid(int action) +{ + switch (action) { + case DAMOS_WILLNEED: + case DAMOS_COLD: + case DAMOS_PAGEOUT: + case DAMOS_HUGEPAGE: + case DAMOS_NOHUGEPAGE: + return true; + default: + return false; + } +} + +/* + * Converts a string into an array of struct damos pointers + * + * Returns an array of struct damos pointers that converted if the conversion + * success, or NULL otherwise. + */ +static struct damos **str_to_schemes(const char *str, ssize_t len, + ssize_t *nr_schemes) +{ + struct damos *scheme, **schemes; + const int max_nr_schemes = 256; + int pos = 0, parsed, ret; + unsigned long min_sz, max_sz; + unsigned int min_nr_a, max_nr_a, min_age, max_age; + unsigned int action; + + schemes = kmalloc_array(max_nr_schemes, sizeof(scheme), + GFP_KERNEL); + if (!schemes) + return NULL; + + *nr_schemes = 0; + while (pos < len && *nr_schemes < max_nr_schemes) { + ret = sscanf(&str[pos], "%lu %lu %u %u %u %u %u%n", + &min_sz, &max_sz, &min_nr_a, &max_nr_a, + &min_age, &max_age, &action, &parsed); + if (ret != 7) + break; + if (!damos_action_valid(action)) { + pr_err("wrong action %d\n", action); + goto fail; + } + + pos += parsed; + scheme = damon_new_scheme(min_sz, max_sz, min_nr_a, max_nr_a, + min_age, max_age, action); + if (!scheme) + goto fail; + + schemes[*nr_schemes] = scheme; + *nr_schemes += 1; + } + return schemes; +fail: + free_schemes_arr(schemes, *nr_schemes); + return NULL; +} + +static ssize_t dbgfs_schemes_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char *kbuf; + struct damos **schemes; + ssize_t nr_schemes = 0, ret = count; + int err; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + schemes = str_to_schemes(kbuf, ret, &nr_schemes); + if (!schemes) { + ret = -EINVAL; + goto out; + } + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + ret = -EBUSY; + goto unlock_out; + } + + err = damon_set_schemes(ctx, schemes, nr_schemes); + if (err) + ret = err; + else + nr_schemes = 0; +unlock_out: + mutex_unlock(&ctx->kdamond_lock); + free_schemes_arr(schemes, nr_schemes); +out: + kfree(kbuf); + return ret; +} + static inline bool targetid_is_pid(const struct damon_ctx *ctx) { return ctx->primitive.target_valid == damon_va_target_valid; @@ -279,6 +432,12 @@ static const struct file_operations attrs_fops = { .write = dbgfs_attrs_write, }; +static const struct file_operations schemes_fops = { + .open = damon_dbgfs_open, + .read = dbgfs_schemes_read, + .write = dbgfs_schemes_write, +}; + static const struct file_operations target_ids_fops = { .open = damon_dbgfs_open, .read = dbgfs_target_ids_read, @@ -292,10 +451,10 @@ static const struct file_operations kdamond_pid_fops = { static void dbgfs_fill_ctx_dir(struct dentry *dir, struct damon_ctx *ctx) { - const char * const file_names[] = {"attrs", "target_ids", + const char * const file_names[] = {"attrs", "schemes", "target_ids", "kdamond_pid"}; - const struct file_operations *fops[] = {&attrs_fops, &target_ids_fops, - &kdamond_pid_fops}; + const struct file_operations *fops[] = {&attrs_fops, &schemes_fops, + &target_ids_fops, &kdamond_pid_fops}; int i; for (i = 0; i < ARRAY_SIZE(file_names); i++) From ef678357b3d3b9cb17474c9104152356e817b70f Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:32 -0700 Subject: [PATCH 069/156] UPSTREAM: mm/damon/schemes: implement statistics feature To tune the DAMON-based operation schemes, knowing how many and how large regions are affected by each of the schemes will be helful. Those stats could be used for not only the tuning, but also monitoring of the working set size and the number of regions, if the scheme does not change the program behavior too much. For the reason, this implements the statistics for the schemes. The total number and size of the regions that each scheme is applied are exported to users via '->stat_count' and '->stat_sz' of 'struct damos'. Admins can also check the number by reading 'schemes' debugfs file. The last two integers now represents the stats. To allow collecting the stats without changing the program behavior, this also adds new scheme action, 'DAMOS_STAT'. Note that 'DAMOS_STAT' is not only making no memory operation actions, but also does not reset the age of regions. Link: https://lkml.kernel.org/r/20211001125604.29660-6-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rienjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 2f0b548c9f03a78f4ce6ab48986e3108028936a6) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Id485ee13922bd769075a77e7263380db32a15544 --- include/linux/damon.h | 10 +++++++++- mm/damon/core.c | 7 ++++++- mm/damon/dbgfs.c | 5 +++-- mm/damon/vaddr.c | 2 ++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index be6b6e81e8ee..f301bb53381c 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -78,6 +78,7 @@ struct damon_target { * @DAMOS_PAGEOUT: Call ``madvise()`` for the region with MADV_PAGEOUT. * @DAMOS_HUGEPAGE: Call ``madvise()`` for the region with MADV_HUGEPAGE. * @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE. + * @DAMOS_STAT: Do nothing but count the stat. */ enum damos_action { DAMOS_WILLNEED, @@ -85,6 +86,7 @@ enum damos_action { DAMOS_PAGEOUT, DAMOS_HUGEPAGE, DAMOS_NOHUGEPAGE, + DAMOS_STAT, /* Do nothing but only record the stat */ }; /** @@ -96,9 +98,13 @@ enum damos_action { * @min_age_region: Minimum age of target regions. * @max_age_region: Maximum age of target regions. * @action: &damo_action to be applied to the target regions. + * @stat_count: Total number of regions that this scheme is applied. + * @stat_sz: Total size of regions that this scheme is applied. * @list: List head for siblings. * - * Note that both the minimums and the maximums are inclusive. + * For each aggregation interval, DAMON applies @action to monitoring target + * regions fit in the condition and updates the statistics. Note that both + * the minimums and the maximums are inclusive. */ struct damos { unsigned long min_sz_region; @@ -108,6 +114,8 @@ struct damos { unsigned int min_age_region; unsigned int max_age_region; enum damos_action action; + unsigned long stat_count; + unsigned long stat_sz; struct list_head list; }; diff --git a/mm/damon/core.c b/mm/damon/core.c index 0ed97b21cbb6..2f6785737902 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -103,6 +103,8 @@ struct damos *damon_new_scheme( scheme->min_age_region = min_age_region; scheme->max_age_region = max_age_region; scheme->action = action; + scheme->stat_count = 0; + scheme->stat_sz = 0; INIT_LIST_HEAD(&scheme->list); return scheme; @@ -544,9 +546,12 @@ static void damon_do_apply_schemes(struct damon_ctx *c, continue; if (r->age < s->min_age_region || s->max_age_region < r->age) continue; + s->stat_count++; + s->stat_sz += sz; if (c->primitive.apply_scheme) c->primitive.apply_scheme(c, t, r, s); - r->age = 0; + if (s->action != DAMOS_STAT) + r->age = 0; } } diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 78b7a04490c5..28d6abf27763 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -106,11 +106,11 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) damon_for_each_scheme(s, c) { rc = scnprintf(&buf[written], len - written, - "%lu %lu %u %u %u %u %d\n", + "%lu %lu %u %u %u %u %d %lu %lu\n", s->min_sz_region, s->max_sz_region, s->min_nr_accesses, s->max_nr_accesses, s->min_age_region, s->max_age_region, - s->action); + s->action, s->stat_count, s->stat_sz); if (!rc) return -ENOMEM; @@ -159,6 +159,7 @@ static bool damos_action_valid(int action) case DAMOS_PAGEOUT: case DAMOS_HUGEPAGE: case DAMOS_NOHUGEPAGE: + case DAMOS_STAT: return true; default: return false; diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 3e1c74d36bab..953c145b4f08 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -705,6 +705,8 @@ int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, case DAMOS_NOHUGEPAGE: madv_action = MADV_NOHUGEPAGE; break; + case DAMOS_STAT: + return 0; default: pr_warn("Wrong action %d\n", scheme->action); return -EINVAL; From cc2e33ff7c34ef661aad7e61aa9679d8f1bfa5b6 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:36 -0700 Subject: [PATCH 070/156] UPSTREAM: selftests/damon: add 'schemes' debugfs tests This adds simple selftets for 'schemes' debugfs file of DAMON. Link: https://lkml.kernel.org/r/20211001125604.29660-7-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rienjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 8d5d4c6359054f3e680e1a2caca50e9b6d688b7d) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I5858f06af5169aca20bab3d871e4a0dd532b0a86 --- tools/testing/selftests/damon/debugfs_attrs.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/testing/selftests/damon/debugfs_attrs.sh b/tools/testing/selftests/damon/debugfs_attrs.sh index bfabb19dc0d3..639cfb6a1f65 100644 --- a/tools/testing/selftests/damon/debugfs_attrs.sh +++ b/tools/testing/selftests/damon/debugfs_attrs.sh @@ -57,6 +57,19 @@ test_write_fail "$file" "1 2 3 5 4" "$orig_content" \ test_content "$file" "$orig_content" "1 2 3 4 5" "successfully written" echo "$orig_content" > "$file" +# Test schemes file +# ================= + +file="$DBGFS/schemes" +orig_content=$(cat "$file") + +test_write_succ "$file" "1 2 3 4 5 6 4" \ + "$orig_content" "valid input" +test_write_fail "$file" "1 2 +3 4 5 6 3" "$orig_content" "multi lines" +test_write_succ "$file" "" "$orig_content" "disabling" +echo "$orig_content" > "$file" + # Test target_ids file # ==================== From 27b2b8d2550280bd1bb11d60ab5e446dbd17a73d Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:42 -0700 Subject: [PATCH 071/156] UPSTREAM: mm/damon/dbgfs: allow users to set initial monitoring target regions Patch series "DAMON: Support Physical Memory Address Space Monitoring:. DAMON currently supports only virtual address spaces monitoring. It can be easily extended for various use cases and address spaces by configuring its monitoring primitives layer to use appropriate primitives implementations, though. This patchset implements monitoring primitives for the physical address space monitoring using the structure. The first 3 patches allow the user space users manually set the monitoring regions. The 1st patch implements the feature in the 'damon-dbgfs'. Then, patches for adding a unit tests (the 2nd patch) and updating the documentation (the 3rd patch) follow. Following 4 patches implement the physical address space monitoring primitives. The 4th patch makes some primitive functions for the virtual address spaces primitives reusable. The 5th patch implements the physical address space monitoring primitives. The 6th patch links the primitives to the 'damon-dbgfs'. Finally, 7th patch documents this new features. This patch (of 7): Some 'damon-dbgfs' users would want to monitor only a part of the entire virtual memory address space. The program interface users in the kernel space could use '->before_start()' callback or set the regions inside the context struct as they want, but 'damon-dbgfs' users cannot. For that reason, this introduces a new debugfs file called 'init_region'. 'damon-dbgfs' users can specify which initial monitoring target address regions they want by writing special input to the file. The input should describe each region in each line in the below form: Note that the regions will be updated to cover entire memory mapped regions after a 'regions update interval' is passed. If you want the regions to not be updated after the initial setting, you could set the interval as a very long time, say, a few decades. Link: https://lkml.kernel.org/r/20211012205711.29216-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211012205711.29216-2-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Cameron Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Jonathan Corbet Cc: David Hildenbrand Cc: David Woodhouse Cc: Marco Elver Cc: Leonard Foerster Cc: Greg Thelen Cc: Markus Boehme Cc: David Rienjes Cc: Shakeel Butt Cc: Shuah Khan Cc: Brendan Higgins Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 90bebce9fcd6488ba6b010af3a16a0a0d7e44cb6) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Idb8961fbe0d851f9b4a1da6b42dfff291d86eae2 --- mm/damon/dbgfs.c | 156 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 154 insertions(+), 2 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 28d6abf27763..1cce53cd241d 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -394,6 +394,152 @@ out: return ret; } +static ssize_t sprint_init_regions(struct damon_ctx *c, char *buf, ssize_t len) +{ + struct damon_target *t; + struct damon_region *r; + int written = 0; + int rc; + + damon_for_each_target(t, c) { + damon_for_each_region(r, t) { + rc = scnprintf(&buf[written], len - written, + "%lu %lu %lu\n", + t->id, r->ar.start, r->ar.end); + if (!rc) + return -ENOMEM; + written += rc; + } + } + return written; +} + +static ssize_t dbgfs_init_regions_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char *kbuf; + ssize_t len; + + kbuf = kmalloc(count, GFP_KERNEL); + if (!kbuf) + return -ENOMEM; + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + mutex_unlock(&ctx->kdamond_lock); + len = -EBUSY; + goto out; + } + + len = sprint_init_regions(ctx, kbuf, count); + mutex_unlock(&ctx->kdamond_lock); + if (len < 0) + goto out; + len = simple_read_from_buffer(buf, count, ppos, kbuf, len); + +out: + kfree(kbuf); + return len; +} + +static int add_init_region(struct damon_ctx *c, + unsigned long target_id, struct damon_addr_range *ar) +{ + struct damon_target *t; + struct damon_region *r, *prev; + unsigned long id; + int rc = -EINVAL; + + if (ar->start >= ar->end) + return -EINVAL; + + damon_for_each_target(t, c) { + id = t->id; + if (targetid_is_pid(c)) + id = (unsigned long)pid_vnr((struct pid *)id); + if (id == target_id) { + r = damon_new_region(ar->start, ar->end); + if (!r) + return -ENOMEM; + damon_add_region(r, t); + if (damon_nr_regions(t) > 1) { + prev = damon_prev_region(r); + if (prev->ar.end > r->ar.start) { + damon_destroy_region(r, t); + return -EINVAL; + } + } + rc = 0; + } + } + return rc; +} + +static int set_init_regions(struct damon_ctx *c, const char *str, ssize_t len) +{ + struct damon_target *t; + struct damon_region *r, *next; + int pos = 0, parsed, ret; + unsigned long target_id; + struct damon_addr_range ar; + int err; + + damon_for_each_target(t, c) { + damon_for_each_region_safe(r, next, t) + damon_destroy_region(r, t); + } + + while (pos < len) { + ret = sscanf(&str[pos], "%lu %lu %lu%n", + &target_id, &ar.start, &ar.end, &parsed); + if (ret != 3) + break; + err = add_init_region(c, target_id, &ar); + if (err) + goto fail; + pos += parsed; + } + + return 0; + +fail: + damon_for_each_target(t, c) { + damon_for_each_region_safe(r, next, t) + damon_destroy_region(r, t); + } + return err; +} + +static ssize_t dbgfs_init_regions_write(struct file *file, + const char __user *buf, size_t count, + loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char *kbuf; + ssize_t ret = count; + int err; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + ret = -EBUSY; + goto unlock_out; + } + + err = set_init_regions(ctx, kbuf, ret); + if (err) + ret = err; + +unlock_out: + mutex_unlock(&ctx->kdamond_lock); + kfree(kbuf); + return ret; +} + static ssize_t dbgfs_kdamond_pid_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { @@ -445,6 +591,12 @@ static const struct file_operations target_ids_fops = { .write = dbgfs_target_ids_write, }; +static const struct file_operations init_regions_fops = { + .open = damon_dbgfs_open, + .read = dbgfs_init_regions_read, + .write = dbgfs_init_regions_write, +}; + static const struct file_operations kdamond_pid_fops = { .open = damon_dbgfs_open, .read = dbgfs_kdamond_pid_read, @@ -453,9 +605,9 @@ static const struct file_operations kdamond_pid_fops = { static void dbgfs_fill_ctx_dir(struct dentry *dir, struct damon_ctx *ctx) { const char * const file_names[] = {"attrs", "schemes", "target_ids", - "kdamond_pid"}; + "init_regions", "kdamond_pid"}; const struct file_operations *fops[] = {&attrs_fops, &schemes_fops, - &target_ids_fops, &kdamond_pid_fops}; + &target_ids_fops, &init_regions_fops, &kdamond_pid_fops}; int i; for (i = 0; i < ARRAY_SIZE(file_names); i++) From c7f64c7f78e2e70fd0878c2a976b43a3788e6e01 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:46 -0700 Subject: [PATCH 072/156] UPSTREAM: mm/damon/dbgfs-test: add a unit test case for 'init_regions' This adds another test case for the new feature, 'init_regions'. Link: https://lkml.kernel.org/r/20211012205711.29216-3-sj@kernel.org Signed-off-by: SeongJae Park Reviewed-by: Brendan Higgins Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rienjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 1c2e11bfa649cc07e6322b0e5ea3cdbada9c43c3) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I99ea9f6aa40f3b241bedd0a8434d41a49ac8fea5 --- mm/damon/dbgfs-test.h | 54 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/mm/damon/dbgfs-test.h b/mm/damon/dbgfs-test.h index 4eddcfa73996..104b22957616 100644 --- a/mm/damon/dbgfs-test.h +++ b/mm/damon/dbgfs-test.h @@ -109,9 +109,63 @@ static void damon_dbgfs_test_set_targets(struct kunit *test) dbgfs_destroy_ctx(ctx); } +static void damon_dbgfs_test_set_init_regions(struct kunit *test) +{ + struct damon_ctx *ctx = damon_new_ctx(); + unsigned long ids[] = {1, 2, 3}; + /* Each line represents one region in `` `` */ + char * const valid_inputs[] = {"2 10 20\n 2 20 30\n2 35 45", + "2 10 20\n", + "2 10 20\n1 39 59\n1 70 134\n 2 20 25\n", + ""}; + /* Reading the file again will show sorted, clean output */ + char * const valid_expects[] = {"2 10 20\n2 20 30\n2 35 45\n", + "2 10 20\n", + "1 39 59\n1 70 134\n2 10 20\n2 20 25\n", + ""}; + char * const invalid_inputs[] = {"4 10 20\n", /* target not exists */ + "2 10 20\n 2 14 26\n", /* regions overlap */ + "1 10 20\n2 30 40\n 1 5 8"}; /* not sorted by address */ + char *input, *expect; + int i, rc; + char buf[256]; + + damon_set_targets(ctx, ids, 3); + + /* Put valid inputs and check the results */ + for (i = 0; i < ARRAY_SIZE(valid_inputs); i++) { + input = valid_inputs[i]; + expect = valid_expects[i]; + + rc = set_init_regions(ctx, input, strnlen(input, 256)); + KUNIT_EXPECT_EQ(test, rc, 0); + + memset(buf, 0, 256); + sprint_init_regions(ctx, buf, 256); + + KUNIT_EXPECT_STREQ(test, (char *)buf, expect); + } + /* Put invlid inputs and check the return error code */ + for (i = 0; i < ARRAY_SIZE(invalid_inputs); i++) { + input = invalid_inputs[i]; + pr_info("input: %s\n", input); + rc = set_init_regions(ctx, input, strnlen(input, 256)); + KUNIT_EXPECT_EQ(test, rc, -EINVAL); + + memset(buf, 0, 256); + sprint_init_regions(ctx, buf, 256); + + KUNIT_EXPECT_STREQ(test, (char *)buf, ""); + } + + damon_set_targets(ctx, NULL, 0); + damon_destroy_ctx(ctx); +} + static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_dbgfs_test_str_to_target_ids), KUNIT_CASE(damon_dbgfs_test_set_targets), + KUNIT_CASE(damon_dbgfs_test_set_init_regions), {}, }; From 3c4a2c142846e16575a55cf74deee6013217df0e Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:53 -0700 Subject: [PATCH 073/156] UPSTREAM: mm/damon/vaddr: separate commonly usable functions This moves functions in the default virtual address spaces monitoring primitives that commonly usable from other address spaces like physical address space into a header file. Those will be reused by the physical address space monitoring primitives which will be implemented by the following commit. [sj@kernel.org: include 'highmem.h' to fix a build failure] Link: https://lkml.kernel.org/r/20211014110848.5204-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211012205711.29216-5-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rienjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 46c3a0accdc48c86928157fd073e66807f338485) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I6f69ed866410db882171c4318f5ee799fbc4eb98 --- mm/damon/Makefile | 2 +- mm/damon/prmtv-common.c | 87 +++++++++++++++++++++++++++++++++++++++ mm/damon/prmtv-common.h | 17 ++++++++ mm/damon/vaddr.c | 90 ++--------------------------------------- 4 files changed, 109 insertions(+), 87 deletions(-) create mode 100644 mm/damon/prmtv-common.c create mode 100644 mm/damon/prmtv-common.h diff --git a/mm/damon/Makefile b/mm/damon/Makefile index fed4be3bace3..99b1bfe01ff5 100644 --- a/mm/damon/Makefile +++ b/mm/damon/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DAMON) := core.o -obj-$(CONFIG_DAMON_VADDR) += vaddr.o +obj-$(CONFIG_DAMON_VADDR) += prmtv-common.o vaddr.o obj-$(CONFIG_DAMON_DBGFS) += dbgfs.o diff --git a/mm/damon/prmtv-common.c b/mm/damon/prmtv-common.c new file mode 100644 index 000000000000..7e62ee54fb54 --- /dev/null +++ b/mm/damon/prmtv-common.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Common Primitives for Data Access Monitoring + * + * Author: SeongJae Park + */ + +#include +#include +#include +#include + +#include "prmtv-common.h" + +/* + * Get an online page for a pfn if it's in the LRU list. Otherwise, returns + * NULL. + * + * The body of this function is stolen from the 'page_idle_get_page()'. We + * steal rather than reuse it because the code is quite simple. + */ +struct page *damon_get_page(unsigned long pfn) +{ + struct page *page = pfn_to_online_page(pfn); + + if (!page || !PageLRU(page) || !get_page_unless_zero(page)) + return NULL; + + if (unlikely(!PageLRU(page))) { + put_page(page); + page = NULL; + } + return page; +} + +void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr) +{ + bool referenced = false; + struct page *page = damon_get_page(pte_pfn(*pte)); + + if (!page) + return; + + if (pte_young(*pte)) { + referenced = true; + *pte = pte_mkold(*pte); + } + +#ifdef CONFIG_MMU_NOTIFIER + if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE)) + referenced = true; +#endif /* CONFIG_MMU_NOTIFIER */ + + if (referenced) + set_page_young(page); + + set_page_idle(page); + put_page(page); +} + +void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + bool referenced = false; + struct page *page = damon_get_page(pmd_pfn(*pmd)); + + if (!page) + return; + + if (pmd_young(*pmd)) { + referenced = true; + *pmd = pmd_mkold(*pmd); + } + +#ifdef CONFIG_MMU_NOTIFIER + if (mmu_notifier_clear_young(mm, addr, + addr + ((1UL) << HPAGE_PMD_SHIFT))) + referenced = true; +#endif /* CONFIG_MMU_NOTIFIER */ + + if (referenced) + set_page_young(page); + + set_page_idle(page); + put_page(page); +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +} diff --git a/mm/damon/prmtv-common.h b/mm/damon/prmtv-common.h new file mode 100644 index 000000000000..7093d19e5d42 --- /dev/null +++ b/mm/damon/prmtv-common.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common Primitives for Data Access Monitoring + * + * Author: SeongJae Park + */ + +#include +#include + +/* Get a random number in [l, r) */ +#define damon_rand(l, r) (l + prandom_u32_max(r - l)) + +struct page *damon_get_page(unsigned long pfn); + +void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr); +void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr); diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 953c145b4f08..f0aef35602da 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -8,25 +8,19 @@ #define pr_fmt(fmt) "damon-va: " fmt #include -#include -#include -#include -#include #include +#include +#include #include #include -#include -#include -#include + +#include "prmtv-common.h" #ifdef CONFIG_DAMON_VADDR_KUNIT_TEST #undef DAMON_MIN_REGION #define DAMON_MIN_REGION 1 #endif -/* Get a random number in [l, r) */ -#define damon_rand(l, r) (l + prandom_u32_max(r - l)) - /* * 't->id' should be the pointer to the relevant 'struct pid' having reference * count. Caller must put the returned task, unless it is NULL. @@ -373,82 +367,6 @@ void damon_va_update(struct damon_ctx *ctx) } } -/* - * Get an online page for a pfn if it's in the LRU list. Otherwise, returns - * NULL. - * - * The body of this function is stolen from the 'page_idle_get_page()'. We - * steal rather than reuse it because the code is quite simple. - */ -static struct page *damon_get_page(unsigned long pfn) -{ - struct page *page = pfn_to_online_page(pfn); - - if (!page || !PageLRU(page) || !get_page_unless_zero(page)) - return NULL; - - if (unlikely(!PageLRU(page))) { - put_page(page); - page = NULL; - } - return page; -} - -static void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, - unsigned long addr) -{ - bool referenced = false; - struct page *page = damon_get_page(pte_pfn(*pte)); - - if (!page) - return; - - if (pte_young(*pte)) { - referenced = true; - *pte = pte_mkold(*pte); - } - -#ifdef CONFIG_MMU_NOTIFIER - if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE)) - referenced = true; -#endif /* CONFIG_MMU_NOTIFIER */ - - if (referenced) - set_page_young(page); - - set_page_idle(page); - put_page(page); -} - -static void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, - unsigned long addr) -{ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - bool referenced = false; - struct page *page = damon_get_page(pmd_pfn(*pmd)); - - if (!page) - return; - - if (pmd_young(*pmd)) { - referenced = true; - *pmd = pmd_mkold(*pmd); - } - -#ifdef CONFIG_MMU_NOTIFIER - if (mmu_notifier_clear_young(mm, addr, - addr + ((1UL) << HPAGE_PMD_SHIFT))) - referenced = true; -#endif /* CONFIG_MMU_NOTIFIER */ - - if (referenced) - set_page_young(page); - - set_page_idle(page); - put_page(page); -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -} - static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long next, struct mm_walk *walk) { From f348ba2256b86a457909680c42bd8b0572749ef4 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:46:56 -0700 Subject: [PATCH 074/156] UPSTREAM: mm/damon: implement primitives for physical address space monitoring This implements the monitoring primitives for the physical memory address space. Internally, it uses the PTE Accessed bit, similar to that of the virtual address spaces monitoring primitives. It supports only user memory pages, as idle pages tracking does. If the monitoring target physical memory address range contains non-user memory pages, access check of the pages will do nothing but simply treat the pages as not accessed. Link: https://lkml.kernel.org/r/20211012205711.29216-6-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rienjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit a28397beb55b68bd0f15c6778540e8ae1bc26d21) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ib36d7b5d2256fbc49013b394071d884b5f64e2ce --- include/linux/damon.h | 10 ++ mm/damon/Kconfig | 8 ++ mm/damon/Makefile | 1 + mm/damon/paddr.c | 224 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 243 insertions(+) create mode 100644 mm/damon/paddr.c diff --git a/include/linux/damon.h b/include/linux/damon.h index f301bb53381c..715dadd21f7c 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -351,4 +351,14 @@ void damon_va_set_primitives(struct damon_ctx *ctx); #endif /* CONFIG_DAMON_VADDR */ +#ifdef CONFIG_DAMON_PADDR + +/* Monitoring primitives for the physical memory address space */ +void damon_pa_prepare_access_checks(struct damon_ctx *ctx); +unsigned int damon_pa_check_accesses(struct damon_ctx *ctx); +bool damon_pa_target_valid(void *t); +void damon_pa_set_primitives(struct damon_ctx *ctx); + +#endif /* CONFIG_DAMON_PADDR */ + #endif /* _DAMON_H */ diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig index ba8898c7eb8e..2a5923be631e 100644 --- a/mm/damon/Kconfig +++ b/mm/damon/Kconfig @@ -32,6 +32,14 @@ config DAMON_VADDR This builds the default data access monitoring primitives for DAMON that work for virtual address spaces. +config DAMON_PADDR + bool "Data access monitoring primitives for the physical address space" + depends on DAMON && MMU + select PAGE_IDLE_FLAG + help + This builds the default data access monitoring primitives for DAMON + that works for the physical address space. + config DAMON_VADDR_KUNIT_TEST bool "Test for DAMON primitives" if !KUNIT_ALL_TESTS depends on DAMON_VADDR && KUNIT=y diff --git a/mm/damon/Makefile b/mm/damon/Makefile index 99b1bfe01ff5..8d9b0df79702 100644 --- a/mm/damon/Makefile +++ b/mm/damon/Makefile @@ -2,4 +2,5 @@ obj-$(CONFIG_DAMON) := core.o obj-$(CONFIG_DAMON_VADDR) += prmtv-common.o vaddr.o +obj-$(CONFIG_DAMON_PADDR) += prmtv-common.o paddr.o obj-$(CONFIG_DAMON_DBGFS) += dbgfs.o diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c new file mode 100644 index 000000000000..d7a2ecd09ed0 --- /dev/null +++ b/mm/damon/paddr.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DAMON Primitives for The Physical Address Space + * + * Author: SeongJae Park + */ + +#define pr_fmt(fmt) "damon-pa: " fmt + +#include +#include +#include +#include + +#include "prmtv-common.h" + +static bool __damon_pa_mkold(struct page *page, struct vm_area_struct *vma, + unsigned long addr, void *arg) +{ + struct page_vma_mapped_walk pvmw = { + .page = page, + .vma = vma, + .address = addr, + }; + + while (page_vma_mapped_walk(&pvmw)) { + addr = pvmw.address; + if (pvmw.pte) + damon_ptep_mkold(pvmw.pte, vma->vm_mm, addr); + else + damon_pmdp_mkold(pvmw.pmd, vma->vm_mm, addr); + } + return true; +} + +static void damon_pa_mkold(unsigned long paddr) +{ + struct page *page = damon_get_page(PHYS_PFN(paddr)); + struct rmap_walk_control rwc = { + .rmap_one = __damon_pa_mkold, + .anon_lock = page_lock_anon_vma_read, + }; + bool need_lock; + + if (!page) + return; + + if (!page_mapped(page) || !page_rmapping(page)) { + set_page_idle(page); + goto out; + } + + need_lock = !PageAnon(page) || PageKsm(page); + if (need_lock && !trylock_page(page)) + goto out; + + rmap_walk(page, &rwc); + + if (need_lock) + unlock_page(page); + +out: + put_page(page); +} + +static void __damon_pa_prepare_access_check(struct damon_ctx *ctx, + struct damon_region *r) +{ + r->sampling_addr = damon_rand(r->ar.start, r->ar.end); + + damon_pa_mkold(r->sampling_addr); +} + +void damon_pa_prepare_access_checks(struct damon_ctx *ctx) +{ + struct damon_target *t; + struct damon_region *r; + + damon_for_each_target(t, ctx) { + damon_for_each_region(r, t) + __damon_pa_prepare_access_check(ctx, r); + } +} + +struct damon_pa_access_chk_result { + unsigned long page_sz; + bool accessed; +}; + +static bool __damon_pa_young(struct page *page, struct vm_area_struct *vma, + unsigned long addr, void *arg) +{ + struct damon_pa_access_chk_result *result = arg; + struct page_vma_mapped_walk pvmw = { + .page = page, + .vma = vma, + .address = addr, + }; + + result->accessed = false; + result->page_sz = PAGE_SIZE; + while (page_vma_mapped_walk(&pvmw)) { + addr = pvmw.address; + if (pvmw.pte) { + result->accessed = pte_young(*pvmw.pte) || + !page_is_idle(page) || + mmu_notifier_test_young(vma->vm_mm, addr); + } else { +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + result->accessed = pmd_young(*pvmw.pmd) || + !page_is_idle(page) || + mmu_notifier_test_young(vma->vm_mm, addr); + result->page_sz = ((1UL) << HPAGE_PMD_SHIFT); +#else + WARN_ON_ONCE(1); +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + } + if (result->accessed) { + page_vma_mapped_walk_done(&pvmw); + break; + } + } + + /* If accessed, stop walking */ + return !result->accessed; +} + +static bool damon_pa_young(unsigned long paddr, unsigned long *page_sz) +{ + struct page *page = damon_get_page(PHYS_PFN(paddr)); + struct damon_pa_access_chk_result result = { + .page_sz = PAGE_SIZE, + .accessed = false, + }; + struct rmap_walk_control rwc = { + .arg = &result, + .rmap_one = __damon_pa_young, + .anon_lock = page_lock_anon_vma_read, + }; + bool need_lock; + + if (!page) + return false; + + if (!page_mapped(page) || !page_rmapping(page)) { + if (page_is_idle(page)) + result.accessed = false; + else + result.accessed = true; + put_page(page); + goto out; + } + + need_lock = !PageAnon(page) || PageKsm(page); + if (need_lock && !trylock_page(page)) { + put_page(page); + return NULL; + } + + rmap_walk(page, &rwc); + + if (need_lock) + unlock_page(page); + put_page(page); + +out: + *page_sz = result.page_sz; + return result.accessed; +} + +static void __damon_pa_check_access(struct damon_ctx *ctx, + struct damon_region *r) +{ + static unsigned long last_addr; + static unsigned long last_page_sz = PAGE_SIZE; + static bool last_accessed; + + /* If the region is in the last checked page, reuse the result */ + if (ALIGN_DOWN(last_addr, last_page_sz) == + ALIGN_DOWN(r->sampling_addr, last_page_sz)) { + if (last_accessed) + r->nr_accesses++; + return; + } + + last_accessed = damon_pa_young(r->sampling_addr, &last_page_sz); + if (last_accessed) + r->nr_accesses++; + + last_addr = r->sampling_addr; +} + +unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) +{ + struct damon_target *t; + struct damon_region *r; + unsigned int max_nr_accesses = 0; + + damon_for_each_target(t, ctx) { + damon_for_each_region(r, t) { + __damon_pa_check_access(ctx, r); + max_nr_accesses = max(r->nr_accesses, max_nr_accesses); + } + } + + return max_nr_accesses; +} + +bool damon_pa_target_valid(void *t) +{ + return true; +} + +void damon_pa_set_primitives(struct damon_ctx *ctx) +{ + ctx->primitive.init = NULL; + ctx->primitive.update = NULL; + ctx->primitive.prepare_access_checks = damon_pa_prepare_access_checks; + ctx->primitive.check_accesses = damon_pa_check_accesses; + ctx->primitive.reset_aggregated = NULL; + ctx->primitive.target_valid = damon_pa_target_valid; + ctx->primitive.cleanup = NULL; + ctx->primitive.apply_scheme = NULL; +} From 932c8c61e19e45295ce52b9f16cdbba47b2cac84 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:00 -0700 Subject: [PATCH 075/156] UPSTREAM: mm/damon/dbgfs: support physical memory monitoring This makes the 'damon-dbgfs' to support the physical memory monitoring, in addition to the virtual memory monitoring. Users can do the physical memory monitoring by writing a special keyword, 'paddr' to the 'target_ids' debugfs file. Then, DAMON will check the special keyword and configure the monitoring context to run with the primitives for the physical address space. Unlike the virtual memory monitoring, the monitoring target region will not be automatically set. Therefore, users should also set the monitoring target address region using the 'init_regions' debugfs file. Also, note that the physical memory monitoring will not automatically terminated. The user should explicitly turn off the monitoring by writing 'off' to the 'monitor_on' debugfs file. Link: https://lkml.kernel.org/r/20211012205711.29216-7-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Brendan Higgins Cc: David Hildenbrand Cc: David Rienjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit c026291ab88f02247999959d01182cb8eb6e6a5b) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I8d90c01714b5cc05ebaaea34d957686f5fafc868 --- mm/damon/Kconfig | 2 +- mm/damon/dbgfs.c | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig index 2a5923be631e..ca33b289ebbe 100644 --- a/mm/damon/Kconfig +++ b/mm/damon/Kconfig @@ -54,7 +54,7 @@ config DAMON_VADDR_KUNIT_TEST config DAMON_DBGFS bool "DAMON debugfs interface" - depends on DAMON_VADDR && DEBUG_FS + depends on DAMON_VADDR && DAMON_PADDR && DEBUG_FS help This builds the debugfs interface for DAMON. The user space admins can use the interface for arbitrary data access monitoring. diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 1cce53cd241d..38188347d8ab 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -339,6 +339,7 @@ static ssize_t dbgfs_target_ids_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { struct damon_ctx *ctx = file->private_data; + bool id_is_pid = true; char *kbuf, *nrs; unsigned long *targets; ssize_t nr_targets; @@ -351,6 +352,11 @@ static ssize_t dbgfs_target_ids_write(struct file *file, return PTR_ERR(kbuf); nrs = kbuf; + if (!strncmp(kbuf, "paddr\n", count)) { + id_is_pid = false; + /* target id is meaningless here, but we set it just for fun */ + scnprintf(kbuf, count, "42 "); + } targets = str_to_target_ids(nrs, ret, &nr_targets); if (!targets) { @@ -358,7 +364,7 @@ static ssize_t dbgfs_target_ids_write(struct file *file, goto out; } - if (targetid_is_pid(ctx)) { + if (id_is_pid) { for (i = 0; i < nr_targets; i++) { targets[i] = (unsigned long)find_get_pid( (int)targets[i]); @@ -372,15 +378,24 @@ static ssize_t dbgfs_target_ids_write(struct file *file, mutex_lock(&ctx->kdamond_lock); if (ctx->kdamond) { - if (targetid_is_pid(ctx)) + if (id_is_pid) dbgfs_put_pids(targets, nr_targets); ret = -EBUSY; goto unlock_out; } + /* remove targets with previously-set primitive */ + damon_set_targets(ctx, NULL, 0); + + /* Configure the context for the address space type */ + if (id_is_pid) + damon_va_set_primitives(ctx); + else + damon_pa_set_primitives(ctx); + err = damon_set_targets(ctx, targets, nr_targets); if (err) { - if (targetid_is_pid(ctx)) + if (id_is_pid) dbgfs_put_pids(targets, nr_targets); ret = err; } From 1d68b96800efb58f1cec547fc3a2e07539ef951e Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Fri, 5 Nov 2021 13:47:07 -0700 Subject: [PATCH 076/156] UPSTREAM: mm/damon/vaddr: constify static mm_walk_ops The only usage of these structs is to pass their addresses to walk_page_range(), which takes a pointer to const mm_walk_ops as argument. Make them const to allow the compiler to put them in read-only memory. Link: https://lkml.kernel.org/r/20211014075042.17174-2-rikard.falkeborn@gmail.com Signed-off-by: Rikard Falkeborn Reviewed-by: SeongJae Park Reviewed-by: Anshuman Khandual Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 199b50f4c9485c46c2403d8b3e0eca90ec401ed6) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ib72d1a17a17377357a53b80cbafc13c456e71683 --- mm/damon/vaddr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index f0aef35602da..758501b8d97d 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -394,7 +394,7 @@ out: return 0; } -static struct mm_walk_ops damon_mkold_ops = { +static const struct mm_walk_ops damon_mkold_ops = { .pmd_entry = damon_mkold_pmd_entry, }; @@ -490,7 +490,7 @@ out: return 0; } -static struct mm_walk_ops damon_young_ops = { +static const struct mm_walk_ops damon_young_ops = { .pmd_entry = damon_young_pmd_entry, }; From 71a23818ca3b152df168586dca12bcb10c198656 Mon Sep 17 00:00:00 2001 From: Rongwei Wang Date: Fri, 5 Nov 2021 13:47:09 -0700 Subject: [PATCH 077/156] UPSTREAM: mm/damon/dbgfs: remove unnecessary variables In some functions, it's unnecessary to declare 'err' and 'ret' variables at the same time. This patch mainly to simplify the issue of such declarations by reusing one variable. Link: https://lkml.kernel.org/r/20211014073014.35754-1-sj@kernel.org Signed-off-by: Rongwei Wang Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 9210622ab81f7e722da7563166d93b2a028a79d4) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Iac1dfa450c23f09ced9b4edc9e8e42b6cb01b1c4 --- mm/damon/dbgfs.c | 66 +++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 38188347d8ab..c90988a20fa4 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -69,8 +69,7 @@ static ssize_t dbgfs_attrs_write(struct file *file, struct damon_ctx *ctx = file->private_data; unsigned long s, a, r, minr, maxr; char *kbuf; - ssize_t ret = count; - int err; + ssize_t ret; kbuf = user_input_str(buf, count, ppos); if (IS_ERR(kbuf)) @@ -88,9 +87,9 @@ static ssize_t dbgfs_attrs_write(struct file *file, goto unlock_out; } - err = damon_set_attrs(ctx, s, a, r, minr, maxr); - if (err) - ret = err; + ret = damon_set_attrs(ctx, s, a, r, minr, maxr); + if (!ret) + ret = count; unlock_out: mutex_unlock(&ctx->kdamond_lock); out: @@ -220,14 +219,13 @@ static ssize_t dbgfs_schemes_write(struct file *file, const char __user *buf, struct damon_ctx *ctx = file->private_data; char *kbuf; struct damos **schemes; - ssize_t nr_schemes = 0, ret = count; - int err; + ssize_t nr_schemes = 0, ret; kbuf = user_input_str(buf, count, ppos); if (IS_ERR(kbuf)) return PTR_ERR(kbuf); - schemes = str_to_schemes(kbuf, ret, &nr_schemes); + schemes = str_to_schemes(kbuf, count, &nr_schemes); if (!schemes) { ret = -EINVAL; goto out; @@ -239,11 +237,12 @@ static ssize_t dbgfs_schemes_write(struct file *file, const char __user *buf, goto unlock_out; } - err = damon_set_schemes(ctx, schemes, nr_schemes); - if (err) - ret = err; - else + ret = damon_set_schemes(ctx, schemes, nr_schemes); + if (!ret) { + ret = count; nr_schemes = 0; + } + unlock_out: mutex_unlock(&ctx->kdamond_lock); free_schemes_arr(schemes, nr_schemes); @@ -343,9 +342,8 @@ static ssize_t dbgfs_target_ids_write(struct file *file, char *kbuf, *nrs; unsigned long *targets; ssize_t nr_targets; - ssize_t ret = count; + ssize_t ret; int i; - int err; kbuf = user_input_str(buf, count, ppos); if (IS_ERR(kbuf)) @@ -358,7 +356,7 @@ static ssize_t dbgfs_target_ids_write(struct file *file, scnprintf(kbuf, count, "42 "); } - targets = str_to_target_ids(nrs, ret, &nr_targets); + targets = str_to_target_ids(nrs, count, &nr_targets); if (!targets) { ret = -ENOMEM; goto out; @@ -393,11 +391,12 @@ static ssize_t dbgfs_target_ids_write(struct file *file, else damon_pa_set_primitives(ctx); - err = damon_set_targets(ctx, targets, nr_targets); - if (err) { + ret = damon_set_targets(ctx, targets, nr_targets); + if (ret) { if (id_is_pid) dbgfs_put_pids(targets, nr_targets); - ret = err; + } else { + ret = count; } unlock_out: @@ -715,8 +714,7 @@ static ssize_t dbgfs_mk_context_write(struct file *file, { char *kbuf; char *ctx_name; - ssize_t ret = count; - int err; + ssize_t ret; kbuf = user_input_str(buf, count, ppos); if (IS_ERR(kbuf)) @@ -734,9 +732,9 @@ static ssize_t dbgfs_mk_context_write(struct file *file, } mutex_lock(&damon_dbgfs_lock); - err = dbgfs_mk_context(ctx_name); - if (err) - ret = err; + ret = dbgfs_mk_context(ctx_name); + if (!ret) + ret = count; mutex_unlock(&damon_dbgfs_lock); out: @@ -805,8 +803,7 @@ static ssize_t dbgfs_rm_context_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { char *kbuf; - ssize_t ret = count; - int err; + ssize_t ret; char *ctx_name; kbuf = user_input_str(buf, count, ppos); @@ -825,9 +822,9 @@ static ssize_t dbgfs_rm_context_write(struct file *file, } mutex_lock(&damon_dbgfs_lock); - err = dbgfs_rm_context(ctx_name); - if (err) - ret = err; + ret = dbgfs_rm_context(ctx_name); + if (!ret) + ret = count; mutex_unlock(&damon_dbgfs_lock); out: @@ -851,9 +848,8 @@ static ssize_t dbgfs_monitor_on_read(struct file *file, static ssize_t dbgfs_monitor_on_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - ssize_t ret = count; + ssize_t ret; char *kbuf; - int err; kbuf = user_input_str(buf, count, ppos); if (IS_ERR(kbuf)) @@ -866,14 +862,14 @@ static ssize_t dbgfs_monitor_on_write(struct file *file, } if (!strncmp(kbuf, "on", count)) - err = damon_start(dbgfs_ctxs, dbgfs_nr_ctxs); + ret = damon_start(dbgfs_ctxs, dbgfs_nr_ctxs); else if (!strncmp(kbuf, "off", count)) - err = damon_stop(dbgfs_ctxs, dbgfs_nr_ctxs); + ret = damon_stop(dbgfs_ctxs, dbgfs_nr_ctxs); else - err = -EINVAL; + ret = -EINVAL; - if (err) - ret = err; + if (!ret) + ret = count; kfree(kbuf); return ret; } From 780cd6f93052f93fa9acfc28ea215f50d43c373c Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:13 -0700 Subject: [PATCH 078/156] UPSTREAM: mm/damon/paddr: support the pageout scheme Introduction ============ This patchset 1) makes the engine for general data access pattern-oriented memory management (DAMOS) be more useful for production environments, and 2) implements a static kernel module for lightweight proactive reclamation using the engine. Proactive Reclamation --------------------- On general memory over-committed systems, proactively reclaiming cold pages helps saving memory and reducing latency spikes that incurred by the direct reclaim or the CPU consumption of kswapd, while incurring only minimal performance degradation[2]. A Free Pages Reporting[8] based memory over-commit virtualization system would be one more specific use case. In the system, the guest VMs reports their free memory to host, and the host reallocates the reported memory to other guests. As a result, the system's memory utilization can be maximized. However, the guests could be not so memory-frugal, because some kernel subsystems and user-space applications are designed to use as much memory as available. Then, guests would report only small amount of free memory to host, results in poor memory utilization. Running the proactive reclamation in such guests could help mitigating this problem. Google has also implemented this idea and using it in their data center. They further proposed upstreaming it in LSFMM'19, and "the general consensus was that, while this sort of proactive reclaim would be useful for a number of users, the cost of this particular solution was too high to consider merging it upstream"[3]. The cost mainly comes from the coldness tracking. Roughly speaking, the implementation periodically scans the 'Accessed' bit of each page. For the reason, the overhead linearly increases as the size of the memory and the scanning frequency grows. As a result, Google is known to dedicating one CPU for the work. That's a reasonable option to someone like Google, but it wouldn't be so to some others. DAMON and DAMOS: An engine for data access pattern-oriented memory management ----------------------------------------------------------------------------- DAMON[4] is a framework for general data access monitoring. Its adaptive monitoring overhead control feature minimizes its monitoring overhead. It also let the upper-bound of the overhead be configurable by clients, regardless of the size of the monitoring target memory. While monitoring 70 GiB memory of a production system every 5 milliseconds, it consumes less than 1% single CPU time. For this, it could sacrify some of the quality of the monitoring results. Nevertheless, the lower-bound of the quality is configurable, and it uses a best-effort algorithm for better quality. Our test results[5] show the quality is practical enough. From the production system monitoring, we were able to find a 4 KiB region in the 70 GiB memory that shows highest access frequency. We normally don't monitor the data access pattern just for fun but to improve something like memory management. Proactive reclamation is one such usage. For such general cases, DAMON provides a feature called DAMon-based Operation Schemes (DAMOS)[6]. It makes DAMON an engine for general data access pattern oriented memory management. Using this, clients can ask DAMON to find memory regions of specific data access pattern and apply some memory management action (e.g., page out, move to head of the LRU list, use huge page, ...). We call the request 'scheme'. Proactive Reclamation on top of DAMON/DAMOS ------------------------------------------- Therefore, by using DAMON for the cold pages detection, the proactive reclamation's monitoring overhead issue can be solved. Actually, we previously implemented a version of proactive reclamation using DAMOS and achieved noticeable improvements with our evaluation setup[5]. Nevertheless, it more for a proof-of-concept, rather than production uses. It supports only virtual address spaces of processes, and require additional tuning efforts for given workloads and the hardware. For the tuning, we introduced a simple auto-tuning user space tool[8]. Google is also known to using a ML-based similar approach for their fleets[2]. But, making it just works with intuitive knobs in the kernel would be helpful for general users. To this end, this patchset improves DAMOS to be ready for such production usages, and implements another version of the proactive reclamation, namely DAMON_RECLAIM, on top of it. DAMOS Improvements: Aggressiveness Control, Prioritization, and Watermarks -------------------------------------------------------------------------- First of all, the current version of DAMOS supports only virtual address spaces. This patchset makes it supports the physical address space for the page out action. Next major problem of the current version of DAMOS is the lack of the aggressiveness control, which can results in arbitrary overhead. For example, if huge memory regions having the data access pattern of interest are found, applying the requested action to all of the regions could incur significant overhead. It can be controlled by tuning the target data access pattern with manual or automated approaches[2,7]. But, some people would prefer the kernel to just work with only intuitive tuning or default values. For such cases, this patchset implements a safeguard, namely time/size quota. Using this, the clients can specify up to how much time can be used for applying the action, and/or up to how much memory regions the action can be applied within a user-specified time duration. A followup question is, to which memory regions should the action applied within the limits? We implement a simple regions prioritization mechanism for each action and make DAMOS to apply the action to high priority regions first. It also allows clients tune the prioritization mechanism to use different weights for size, access frequency, and age of memory regions. This means we could use not only LRU but also LFU or some fancy algorithms like CAR[9] with lightweight overhead. Though DAMON is lightweight, someone would want to remove even the cold pages monitoring overhead when it is unnecessary. Currently, it should manually turned on and off by clients, but some clients would simply want to turn it on and off based on some metrics like free memory ratio or memory fragmentation. For such cases, this patchset implements a watermarks-based automatic activation feature. It allows the clients configure the metric of their interest, and three watermarks of the metric. If the metric is higher than the high watermark or lower than the low watermark, the scheme is deactivated. If the metric is lower than the mid watermark but higher than the low watermark, the scheme is activated. DAMON-based Reclaim ------------------- Using the improved version of DAMOS, this patchset implements a static kernel module called 'damon_reclaim'. It finds memory regions that didn't accessed for specific time duration and page out. Consuming too much CPU for the paging out operations, or doing pageout too frequently can be critical for systems configuring their swap devices with software-defined in-memory block devices like zram/zswap or total number of writes limited devices like SSDs, respectively. To avoid the problems, the time/size quotas can be configured. Under the quotas, it pages out memory regions that didn't accessed longer first. Also, to remove the monitoring overhead under peaceful situation, and to fall back to the LRU-list based page granularity reclamation when it doesn't make progress, the three watermarks based activation mechanism is used, with the free memory ratio as the watermark metric. For convenient configurations, it provides several module parameters. Using these, sysadmins can enable/disable it, and tune its parameters including the coldness identification time threshold, the time/size quotas and the three watermarks. Evaluation ========== In short, DAMON_RECLAIM with 50ms/s time quota and regions prioritization on v5.15-rc5 Linux kernel with ZRAM swap device achieves 38.58% memory saving with only 1.94% runtime overhead. For this, DAMON_RECLAIM consumes only 4.97% of single CPU time. Setup ----- We evaluate DAMON_RECLAIM to show how each of the DAMOS improvements make effect. For this, we measure DAMON_RECLAIM's CPU consumption, entire system memory footprint, total number of major page faults, and runtime of 24 realistic workloads in PARSEC3 and SPLASH-2X benchmark suites on my QEMU/KVM based virtual machine. The virtual machine runs on an i3.metal AWS instance, has 130GiB memory, and runs a linux kernel built on latest -mm tree[1] plus this patchset. It also utilizes a 4 GiB ZRAM swap device. We repeats the measurement 5 times and use averages. [1] https://github.com/hnaz/linux-mm/tree/v5.15-rc5-mmots-2021-10-13-19-55 Detailed Results ---------------- The results are summarized in the below table. With coldness identification threshold of 5 seconds, DAMON_RECLAIM without the time quota-based speed limit achieves 47.21% memory saving, but incur 4.59% runtime slowdown to the workloads on average. For this, DAMON_RECLAIM consumes about 11.28% single CPU time. Applying time quotas of 200ms/s, 50ms/s, and 10ms/s without the regions prioritization reduces the slowdown to 4.89%, 2.65%, and 1.5%, respectively. Time quota of 200ms/s (20%) makes no real change compared to the quota unapplied version, because the quota unapplied version consumes only 11.28% CPU time. DAMON_RECLAIM's CPU utilization also similarly reduced: 11.24%, 5.51%, and 2.01% of single CPU time. That is, the overhead is proportional to the speed limit. Nevertheless, it also reduces the memory saving because it becomes less aggressive. In detail, the three variants show 48.76%, 37.83%, and 7.85% memory saving, respectively. Applying the regions prioritization (page out regions that not accessed longer first within the time quota) further reduces the performance degradation. Runtime slowdowns and total number of major page faults increase has been 4.89%/218,690% -> 4.39%/166,136% (200ms/s), 2.65%/111,886% -> 1.94%/59,053% (50ms/s), and 1.5%/34,973.40% -> 2.08%/8,781.75% (10ms/s). The runtime under 10ms/s time quota has increased with prioritization, but apparently that's under the margin of error. time quota prioritization memory_saving cpu_util slowdown pgmajfaults overhead N N 47.21% 11.28% 4.59% 194,802% 200ms/s N 48.76% 11.24% 4.89% 218,690% 50ms/s N 37.83% 5.51% 2.65% 111,886% 10ms/s N 7.85% 2.01% 1.5% 34,793.40% 200ms/s Y 50.08% 10.38% 4.39% 166,136% 50ms/s Y 38.58% 4.97% 1.94% 59,053% 10ms/s Y 3.63% 1.73% 2.08% 8,781.75% Baseline and Complete Git Trees =============================== The patches are based on the latest -mm tree (v5.15-rc5-mmots-2021-10-13-19-55). You can also clone the complete git tree from: $ git clone git://github.com/sjp38/linux -b damon_reclaim/patches/v1 The web is also available: https://git.kernel.org/pub/scm/linux/kernel/git/sj/linux.git/tag/?h=damon_reclaim/patches/v1 Sequence Of Patches =================== The first patch makes DAMOS support the physical address space for the page out action. Following five patches (patches 2-6) implement the time/size quotas. Next four patches (patches 7-10) implement the memory regions prioritization within the limit. Then, three following patches (patches 11-13) implement the watermarks-based schemes activation. Finally, the last two patches (patches 14-15) implement and document the DAMON-based reclamation using the advanced DAMOS. [1] https://www.kernel.org/doc/html/v5.15-rc1/vm/damon/index.html [2] https://research.google/pubs/pub48551/ [3] https://lwn.net/Articles/787611/ [4] https://damonitor.github.io [5] https://damonitor.github.io/doc/html/latest/vm/damon/eval.html [6] https://lore.kernel.org/linux-mm/20211001125604.29660-1-sj@kernel.org/ [7] https://github.com/awslabs/damoos [8] https://www.kernel.org/doc/html/latest/vm/free_page_reporting.html [9] https://www.usenix.org/conference/fast-04/car-clock-adaptive-replacement This patch (of 15): This makes the DAMON primitives for physical address space support the pageout action for DAMON-based Operation Schemes. With this commit, hence, users can easily implement system-level data access-aware reclamations using DAMOS. [sj@kernel.org: fix missing-prototype build warning] Link: https://lkml.kernel.org/r/20211025064220.13904-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211019150731.16699-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211019150731.16699-2-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Cameron Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: Jonathan Corbet Cc: David Hildenbrand Cc: David Woodhouse Cc: Marco Elver Cc: Leonard Foerster Cc: Greg Thelen Cc: Markus Boehme Cc: David Rientjes Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 57223ac295845b1d72ec1bd02b5fab992b77a021) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I56dbce96f1d8152cac49ef0d11cb81a342bfa89d --- include/linux/damon.h | 2 ++ mm/damon/paddr.c | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 715dadd21f7c..9a327bc787b5 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -357,6 +357,8 @@ void damon_va_set_primitives(struct damon_ctx *ctx); void damon_pa_prepare_access_checks(struct damon_ctx *ctx); unsigned int damon_pa_check_accesses(struct damon_ctx *ctx); bool damon_pa_target_valid(void *t); +int damon_pa_apply_scheme(struct damon_ctx *context, struct damon_target *t, + struct damon_region *r, struct damos *scheme); void damon_pa_set_primitives(struct damon_ctx *ctx); #endif /* CONFIG_DAMON_PADDR */ diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index d7a2ecd09ed0..957ada55de77 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -11,7 +11,9 @@ #include #include #include +#include +#include "../internal.h" #include "prmtv-common.h" static bool __damon_pa_mkold(struct page *page, struct vm_area_struct *vma, @@ -211,6 +213,39 @@ bool damon_pa_target_valid(void *t) return true; } +int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, + struct damon_region *r, struct damos *scheme) +{ + unsigned long addr; + LIST_HEAD(page_list); + + if (scheme->action != DAMOS_PAGEOUT) + return -EINVAL; + + for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { + struct page *page = damon_get_page(PHYS_PFN(addr)); + + if (!page) + continue; + + ClearPageReferenced(page); + test_and_clear_page_young(page); + if (isolate_lru_page(page)) { + put_page(page); + continue; + } + if (PageUnevictable(page)) { + putback_lru_page(page); + } else { + list_add(&page->lru, &page_list); + put_page(page); + } + } + reclaim_pages(&page_list); + cond_resched(); + return 0; +} + void damon_pa_set_primitives(struct damon_ctx *ctx) { ctx->primitive.init = NULL; @@ -220,5 +255,5 @@ void damon_pa_set_primitives(struct damon_ctx *ctx) ctx->primitive.reset_aggregated = NULL; ctx->primitive.target_valid = damon_pa_target_valid; ctx->primitive.cleanup = NULL; - ctx->primitive.apply_scheme = NULL; + ctx->primitive.apply_scheme = damon_pa_apply_scheme; } From a9af0008bedd426baaa16db789c3a77c9b3852cb Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:16 -0700 Subject: [PATCH 079/156] UPSTREAM: mm/damon/schemes: implement size quota for schemes application speed control There could be arbitrarily large memory regions fulfilling the target data access pattern of a DAMON-based operation scheme. In the case, applying the action of the scheme could incur too high overhead. To provide an intuitive way for avoiding it, this implements a feature called size quota. If the quota is set, DAMON tries to apply the action only up to the given amount of memory regions within a given time window. Link: https://lkml.kernel.org/r/20211019150731.16699-3-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 2b8a248d5873343aa16f6c5ede30517693995f13) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I14a2cddcf6d9fa313ab16150a8d580ee60288b9e --- include/linux/damon.h | 36 +++++++++++++++++++++++--- mm/damon/core.c | 60 +++++++++++++++++++++++++++++++++++++------ mm/damon/dbgfs.c | 4 ++- 3 files changed, 87 insertions(+), 13 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 9a327bc787b5..3a1ce9d9921c 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -89,6 +89,26 @@ enum damos_action { DAMOS_STAT, /* Do nothing but only record the stat */ }; +/** + * struct damos_quota - Controls the aggressiveness of the given scheme. + * @sz: Maximum bytes of memory that the action can be applied. + * @reset_interval: Charge reset interval in milliseconds. + * + * To avoid consuming too much CPU time or IO resources for applying the + * &struct damos->action to large memory, DAMON allows users to set a size + * quota. The quota can be set by writing non-zero values to &sz. If the size + * quota is set, DAMON tries to apply the action only up to &sz bytes within + * &reset_interval. + */ +struct damos_quota { + unsigned long sz; + unsigned long reset_interval; + +/* private: For charging the quota */ + unsigned long charged_sz; + unsigned long charged_from; +}; + /** * struct damos - Represents a Data Access Monitoring-based Operation Scheme. * @min_sz_region: Minimum size of target regions. @@ -98,13 +118,20 @@ enum damos_action { * @min_age_region: Minimum age of target regions. * @max_age_region: Maximum age of target regions. * @action: &damo_action to be applied to the target regions. + * @quota: Control the aggressiveness of this scheme. * @stat_count: Total number of regions that this scheme is applied. * @stat_sz: Total size of regions that this scheme is applied. * @list: List head for siblings. * - * For each aggregation interval, DAMON applies @action to monitoring target - * regions fit in the condition and updates the statistics. Note that both - * the minimums and the maximums are inclusive. + * For each aggregation interval, DAMON finds regions which fit in the + * condition (&min_sz_region, &max_sz_region, &min_nr_accesses, + * &max_nr_accesses, &min_age_region, &max_age_region) and applies &action to + * those. To avoid consuming too much CPU time or IO resources for the + * &action, "a is used. + * + * After applying the &action to each region, &stat_count and &stat_sz is + * updated to reflect the number of regions and total size of regions that the + * &action is applied. */ struct damos { unsigned long min_sz_region; @@ -114,6 +141,7 @@ struct damos { unsigned int min_age_region; unsigned int max_age_region; enum damos_action action; + struct damos_quota quota; unsigned long stat_count; unsigned long stat_sz; struct list_head list; @@ -310,7 +338,7 @@ struct damos *damon_new_scheme( unsigned long min_sz_region, unsigned long max_sz_region, unsigned int min_nr_accesses, unsigned int max_nr_accesses, unsigned int min_age_region, unsigned int max_age_region, - enum damos_action action); + enum damos_action action, struct damos_quota *quota); void damon_add_scheme(struct damon_ctx *ctx, struct damos *s); void damon_destroy_scheme(struct damos *s); diff --git a/mm/damon/core.c b/mm/damon/core.c index 2f6785737902..cce14a0d5c72 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -89,7 +89,7 @@ struct damos *damon_new_scheme( unsigned long min_sz_region, unsigned long max_sz_region, unsigned int min_nr_accesses, unsigned int max_nr_accesses, unsigned int min_age_region, unsigned int max_age_region, - enum damos_action action) + enum damos_action action, struct damos_quota *quota) { struct damos *scheme; @@ -107,6 +107,11 @@ struct damos *damon_new_scheme( scheme->stat_sz = 0; INIT_LIST_HEAD(&scheme->list); + scheme->quota.sz = quota->sz; + scheme->quota.reset_interval = quota->reset_interval; + scheme->quota.charged_sz = 0; + scheme->quota.charged_from = 0; + return scheme; } @@ -530,15 +535,25 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) } } +static void damon_split_region_at(struct damon_ctx *ctx, + struct damon_target *t, struct damon_region *r, + unsigned long sz_r); + static void damon_do_apply_schemes(struct damon_ctx *c, struct damon_target *t, struct damon_region *r) { struct damos *s; - unsigned long sz; damon_for_each_scheme(s, c) { - sz = r->ar.end - r->ar.start; + struct damos_quota *quota = &s->quota; + unsigned long sz = r->ar.end - r->ar.start; + + /* Check the quota */ + if (quota->sz && quota->charged_sz >= quota->sz) + continue; + + /* Check the target regions condition */ if (sz < s->min_sz_region || s->max_sz_region < sz) continue; if (r->nr_accesses < s->min_nr_accesses || @@ -546,22 +561,51 @@ static void damon_do_apply_schemes(struct damon_ctx *c, continue; if (r->age < s->min_age_region || s->max_age_region < r->age) continue; - s->stat_count++; - s->stat_sz += sz; - if (c->primitive.apply_scheme) + + /* Apply the scheme */ + if (c->primitive.apply_scheme) { + if (quota->sz && quota->charged_sz + sz > quota->sz) { + sz = ALIGN_DOWN(quota->sz - quota->charged_sz, + DAMON_MIN_REGION); + if (!sz) + goto update_stat; + damon_split_region_at(c, t, r, sz); + } c->primitive.apply_scheme(c, t, r, s); + quota->charged_sz += sz; + } if (s->action != DAMOS_STAT) r->age = 0; + +update_stat: + s->stat_count++; + s->stat_sz += sz; } } static void kdamond_apply_schemes(struct damon_ctx *c) { struct damon_target *t; - struct damon_region *r; + struct damon_region *r, *next_r; + struct damos *s; + + damon_for_each_scheme(s, c) { + struct damos_quota *quota = &s->quota; + + if (!quota->sz) + continue; + + /* New charge window starts */ + if (time_after_eq(jiffies, quota->charged_from + + msecs_to_jiffies( + quota->reset_interval))) { + quota->charged_from = jiffies; + quota->charged_sz = 0; + } + } damon_for_each_target(t, c) { - damon_for_each_region(r, t) + damon_for_each_region_safe(r, next_r, t) damon_do_apply_schemes(c, t, r); } } diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index c90988a20fa4..a04bd50cc4c4 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -188,6 +188,8 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, *nr_schemes = 0; while (pos < len && *nr_schemes < max_nr_schemes) { + struct damos_quota quota = {}; + ret = sscanf(&str[pos], "%lu %lu %u %u %u %u %u%n", &min_sz, &max_sz, &min_nr_a, &max_nr_a, &min_age, &max_age, &action, &parsed); @@ -200,7 +202,7 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, pos += parsed; scheme = damon_new_scheme(min_sz, max_sz, min_nr_a, max_nr_a, - min_age, max_age, action); + min_age, max_age, action, "a); if (!scheme) goto fail; From 8c0c30e2f039b2075f9b5788332c52eacc0c3f50 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:20 -0700 Subject: [PATCH 080/156] UPSTREAM: mm/damon/schemes: skip already charged targets and regions If DAMOS has stopped applying action in the middle of a group of memory regions due to its size quota, it starts the work again from the beginning of the address space in the next charge window. If there is a huge memory region at the beginning of the address space and it fulfills the scheme's target data access pattern always, the action will applied to only the region. This mitigates the case by skipping memory regions that charged in current charge window at the beginning of next charge window. Link: https://lkml.kernel.org/r/20211019150731.16699-4-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 50585192bc2ef9309d32dabdbb5e735679f4f128) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I73d1b16851b2dbe57927d9685b53e7ab3ad22a9c --- include/linux/damon.h | 5 +++++ mm/damon/core.c | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 3a1ce9d9921c..585d985768fd 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -107,6 +107,8 @@ struct damos_quota { /* private: For charging the quota */ unsigned long charged_sz; unsigned long charged_from; + struct damon_target *charge_target_from; + unsigned long charge_addr_from; }; /** @@ -307,6 +309,9 @@ struct damon_ctx { #define damon_prev_region(r) \ (container_of(r->list.prev, struct damon_region, list)) +#define damon_last_region(t) \ + (list_last_entry(&t->regions_list, struct damon_region, list)) + #define damon_for_each_region(r, t) \ list_for_each_entry(r, &t->regions_list, list) diff --git a/mm/damon/core.c b/mm/damon/core.c index cce14a0d5c72..693b75bc3450 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -111,6 +111,8 @@ struct damos *damon_new_scheme( scheme->quota.reset_interval = quota->reset_interval; scheme->quota.charged_sz = 0; scheme->quota.charged_from = 0; + scheme->quota.charge_target_from = NULL; + scheme->quota.charge_addr_from = 0; return scheme; } @@ -553,6 +555,37 @@ static void damon_do_apply_schemes(struct damon_ctx *c, if (quota->sz && quota->charged_sz >= quota->sz) continue; + /* Skip previously charged regions */ + if (quota->charge_target_from) { + if (t != quota->charge_target_from) + continue; + if (r == damon_last_region(t)) { + quota->charge_target_from = NULL; + quota->charge_addr_from = 0; + continue; + } + if (quota->charge_addr_from && + r->ar.end <= quota->charge_addr_from) + continue; + + if (quota->charge_addr_from && r->ar.start < + quota->charge_addr_from) { + sz = ALIGN_DOWN(quota->charge_addr_from - + r->ar.start, DAMON_MIN_REGION); + if (!sz) { + if (r->ar.end - r->ar.start <= + DAMON_MIN_REGION) + continue; + sz = DAMON_MIN_REGION; + } + damon_split_region_at(c, t, r, sz); + r = damon_next_region(r); + sz = r->ar.end - r->ar.start; + } + quota->charge_target_from = NULL; + quota->charge_addr_from = 0; + } + /* Check the target regions condition */ if (sz < s->min_sz_region || s->max_sz_region < sz) continue; @@ -573,6 +606,10 @@ static void damon_do_apply_schemes(struct damon_ctx *c, } c->primitive.apply_scheme(c, t, r, s); quota->charged_sz += sz; + if (quota->sz && quota->charged_sz >= quota->sz) { + quota->charge_target_from = t; + quota->charge_addr_from = r->ar.end + 1; + } } if (s->action != DAMOS_STAT) r->age = 0; From 2c090a6bc61a3d48afecad6b75cb9413c3e8fde4 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:23 -0700 Subject: [PATCH 081/156] UPSTREAM: mm/damon/schemes: implement time quota The size quota feature of DAMOS is useful for IO resource-critical systems, but not so intuitive for CPU time-critical systems. Systems using zram or zswap-like swap device would be examples. To provide another intuitive ways for such systems, this implements time-based quota for DAMON-based Operation Schemes. If the quota is set, DAMOS tries to use only up to the user-defined quota of CPU time within a given time window. Link: https://lkml.kernel.org/r/20211019150731.16699-5-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 1cd2430300594a230dba9178ac9e286d868d9da2) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I5ee77a0882cb67d76f7f24ce7742414718ba028c --- include/linux/damon.h | 25 +++++++++++++++++++----- mm/damon/core.c | 45 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 585d985768fd..1e7671bf3d23 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -91,20 +91,35 @@ enum damos_action { /** * struct damos_quota - Controls the aggressiveness of the given scheme. + * @ms: Maximum milliseconds that the scheme can use. * @sz: Maximum bytes of memory that the action can be applied. * @reset_interval: Charge reset interval in milliseconds. * * To avoid consuming too much CPU time or IO resources for applying the - * &struct damos->action to large memory, DAMON allows users to set a size - * quota. The quota can be set by writing non-zero values to &sz. If the size - * quota is set, DAMON tries to apply the action only up to &sz bytes within - * &reset_interval. + * &struct damos->action to large memory, DAMON allows users to set time and/or + * size quotas. The quotas can be set by writing non-zero values to &ms and + * &sz, respectively. If the time quota is set, DAMON tries to use only up to + * &ms milliseconds within &reset_interval for applying the action. If the + * size quota is set, DAMON tries to apply the action only up to &sz bytes + * within &reset_interval. + * + * Internally, the time quota is transformed to a size quota using estimated + * throughput of the scheme's action. DAMON then compares it against &sz and + * uses smaller one as the effective quota. */ struct damos_quota { + unsigned long ms; unsigned long sz; unsigned long reset_interval; -/* private: For charging the quota */ +/* private: */ + /* For throughput estimation */ + unsigned long total_charged_sz; + unsigned long total_charged_ns; + + unsigned long esz; /* Effective size quota in bytes */ + + /* For charging the quota */ unsigned long charged_sz; unsigned long charged_from; struct damon_target *charge_target_from; diff --git a/mm/damon/core.c b/mm/damon/core.c index 693b75bc3450..d1da4bef96ed 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -107,8 +107,12 @@ struct damos *damon_new_scheme( scheme->stat_sz = 0; INIT_LIST_HEAD(&scheme->list); + scheme->quota.ms = quota->ms; scheme->quota.sz = quota->sz; scheme->quota.reset_interval = quota->reset_interval; + scheme->quota.total_charged_sz = 0; + scheme->quota.total_charged_ns = 0; + scheme->quota.esz = 0; scheme->quota.charged_sz = 0; scheme->quota.charged_from = 0; scheme->quota.charge_target_from = NULL; @@ -550,9 +554,10 @@ static void damon_do_apply_schemes(struct damon_ctx *c, damon_for_each_scheme(s, c) { struct damos_quota *quota = &s->quota; unsigned long sz = r->ar.end - r->ar.start; + struct timespec64 begin, end; /* Check the quota */ - if (quota->sz && quota->charged_sz >= quota->sz) + if (quota->esz && quota->charged_sz >= quota->esz) continue; /* Skip previously charged regions */ @@ -597,16 +602,21 @@ static void damon_do_apply_schemes(struct damon_ctx *c, /* Apply the scheme */ if (c->primitive.apply_scheme) { - if (quota->sz && quota->charged_sz + sz > quota->sz) { - sz = ALIGN_DOWN(quota->sz - quota->charged_sz, + if (quota->esz && + quota->charged_sz + sz > quota->esz) { + sz = ALIGN_DOWN(quota->esz - quota->charged_sz, DAMON_MIN_REGION); if (!sz) goto update_stat; damon_split_region_at(c, t, r, sz); } + ktime_get_coarse_ts64(&begin); c->primitive.apply_scheme(c, t, r, s); + ktime_get_coarse_ts64(&end); + quota->total_charged_ns += timespec64_to_ns(&end) - + timespec64_to_ns(&begin); quota->charged_sz += sz; - if (quota->sz && quota->charged_sz >= quota->sz) { + if (quota->esz && quota->charged_sz >= quota->esz) { quota->charge_target_from = t; quota->charge_addr_from = r->ar.end + 1; } @@ -620,6 +630,29 @@ update_stat: } } +/* Shouldn't be called if quota->ms and quota->sz are zero */ +static void damos_set_effective_quota(struct damos_quota *quota) +{ + unsigned long throughput; + unsigned long esz; + + if (!quota->ms) { + quota->esz = quota->sz; + return; + } + + if (quota->total_charged_ns) + throughput = quota->total_charged_sz * 1000000 / + quota->total_charged_ns; + else + throughput = PAGE_SIZE * 1024; + esz = throughput * quota->ms; + + if (quota->sz && quota->sz < esz) + esz = quota->sz; + quota->esz = esz; +} + static void kdamond_apply_schemes(struct damon_ctx *c) { struct damon_target *t; @@ -629,15 +662,17 @@ static void kdamond_apply_schemes(struct damon_ctx *c) damon_for_each_scheme(s, c) { struct damos_quota *quota = &s->quota; - if (!quota->sz) + if (!quota->ms && !quota->sz) continue; /* New charge window starts */ if (time_after_eq(jiffies, quota->charged_from + msecs_to_jiffies( quota->reset_interval))) { + quota->total_charged_sz += quota->charged_sz; quota->charged_from = jiffies; quota->charged_sz = 0; + damos_set_effective_quota(quota); } } From 8c491daa0f31d6d324ccd282b493fb239ac1986d Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:27 -0700 Subject: [PATCH 082/156] UPSTREAM: mm/damon/dbgfs: support quotas of schemes This makes the debugfs interface of DAMON support the scheme quotas by chaning the format of the input for the schemes file. Link: https://lkml.kernel.org/r/20211019150731.16699-6-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit d7d0ec85e983945079364db3c3d2d80cc795a48c) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I2260814672b0b9db7f0b416505bfa767fa0fa00b --- mm/damon/dbgfs.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index a04bd50cc4c4..097e6745ba75 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -105,11 +105,14 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) damon_for_each_scheme(s, c) { rc = scnprintf(&buf[written], len - written, - "%lu %lu %u %u %u %u %d %lu %lu\n", + "%lu %lu %u %u %u %u %d %lu %lu %lu %lu %lu\n", s->min_sz_region, s->max_sz_region, s->min_nr_accesses, s->max_nr_accesses, s->min_age_region, s->max_age_region, - s->action, s->stat_count, s->stat_sz); + s->action, + s->quota.ms, s->quota.sz, + s->quota.reset_interval, + s->stat_count, s->stat_sz); if (!rc) return -ENOMEM; @@ -190,10 +193,11 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, while (pos < len && *nr_schemes < max_nr_schemes) { struct damos_quota quota = {}; - ret = sscanf(&str[pos], "%lu %lu %u %u %u %u %u%n", + ret = sscanf(&str[pos], "%lu %lu %u %u %u %u %u %lu %lu %lu%n", &min_sz, &max_sz, &min_nr_a, &max_nr_a, - &min_age, &max_age, &action, &parsed); - if (ret != 7) + &min_age, &max_age, &action, "a.ms, + "a.sz, "a.reset_interval, &parsed); + if (ret != 10) break; if (!damos_action_valid(action)) { pr_err("wrong action %d\n", action); From 1990bcb74640522a12e33290054165d576045d0e Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:30 -0700 Subject: [PATCH 083/156] UPSTREAM: mm/damon/selftests: support schemes quotas This updates DAMON selftests to support updated schemes debugfs file format for the quotas. Link: https://lkml.kernel.org/r/20211019150731.16699-7-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit a2cb4dd0d40d3dcb7288a963d0f66271934417b6) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I6e8ea947a6f999daef8ee52cd882d17dda6ae098 --- tools/testing/selftests/damon/debugfs_attrs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/damon/debugfs_attrs.sh b/tools/testing/selftests/damon/debugfs_attrs.sh index 639cfb6a1f65..8e33a7b584e7 100644 --- a/tools/testing/selftests/damon/debugfs_attrs.sh +++ b/tools/testing/selftests/damon/debugfs_attrs.sh @@ -63,10 +63,10 @@ echo "$orig_content" > "$file" file="$DBGFS/schemes" orig_content=$(cat "$file") -test_write_succ "$file" "1 2 3 4 5 6 4" \ +test_write_succ "$file" "1 2 3 4 5 6 4 0 0 0" \ "$orig_content" "valid input" test_write_fail "$file" "1 2 -3 4 5 6 3" "$orig_content" "multi lines" +3 4 5 6 3 0 0 0" "$orig_content" "multi lines" test_write_succ "$file" "" "$orig_content" "disabling" echo "$orig_content" > "$file" From dd1947047e43e81505f2b7cffe8b130b7f20adeb Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:33 -0700 Subject: [PATCH 084/156] UPSTREAM: mm/damon/schemes: prioritize regions within the quotas This makes DAMON apply schemes to regions having higher priority first, if it cannot apply schemes to all regions due to the quotas. The prioritization function should be implemented in the monitoring primitives. Those would commonly calculate the priority of the region using attributes of regions, namely 'size', 'nr_accesses', and 'age'. For example, some primitive would calculate the priority of each region using a weighted sum of 'nr_accesses' and 'age' of the region. The optimal weights would depend on give environments, so this makes those customizable. Nevertheless, the score calculation functions are only encouraged to respect the weights, not mandated. Link: https://lkml.kernel.org/r/20211019150731.16699-8-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 38683e003153f7abfa612d7b7fe147efa4624af2) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I7c96b116f7daf1d33df427b023aba3664951e2cb --- include/linux/damon.h | 26 ++++++++++++++++++ mm/damon/core.c | 62 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 1e7671bf3d23..5d47ad9e3911 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -14,6 +14,8 @@ /* Minimal region size. Every damon_region is aligned by this. */ #define DAMON_MIN_REGION PAGE_SIZE +/* Max priority score for DAMON-based operation schemes */ +#define DAMOS_MAX_SCORE (99) /** * struct damon_addr_range - Represents an address region of [@start, @end). @@ -95,6 +97,10 @@ enum damos_action { * @sz: Maximum bytes of memory that the action can be applied. * @reset_interval: Charge reset interval in milliseconds. * + * @weight_sz: Weight of the region's size for prioritization. + * @weight_nr_accesses: Weight of the region's nr_accesses for prioritization. + * @weight_age: Weight of the region's age for prioritization. + * * To avoid consuming too much CPU time or IO resources for applying the * &struct damos->action to large memory, DAMON allows users to set time and/or * size quotas. The quotas can be set by writing non-zero values to &ms and @@ -106,12 +112,22 @@ enum damos_action { * Internally, the time quota is transformed to a size quota using estimated * throughput of the scheme's action. DAMON then compares it against &sz and * uses smaller one as the effective quota. + * + * For selecting regions within the quota, DAMON prioritizes current scheme's + * target memory regions using the &struct damon_primitive->get_scheme_score. + * You could customize the prioritization logic by setting &weight_sz, + * &weight_nr_accesses, and &weight_age, because monitoring primitives are + * encouraged to respect those. */ struct damos_quota { unsigned long ms; unsigned long sz; unsigned long reset_interval; + unsigned int weight_sz; + unsigned int weight_nr_accesses; + unsigned int weight_age; + /* private: */ /* For throughput estimation */ unsigned long total_charged_sz; @@ -124,6 +140,10 @@ struct damos_quota { unsigned long charged_from; struct damon_target *charge_target_from; unsigned long charge_addr_from; + + /* For prioritization */ + unsigned long histogram[DAMOS_MAX_SCORE + 1]; + unsigned int min_score; }; /** @@ -174,6 +194,7 @@ struct damon_ctx; * @prepare_access_checks: Prepare next access check of target regions. * @check_accesses: Check the accesses to target regions. * @reset_aggregated: Reset aggregated accesses monitoring results. + * @get_scheme_score: Get the score of a region for a scheme. * @apply_scheme: Apply a DAMON-based operation scheme. * @target_valid: Determine if the target is valid. * @cleanup: Clean up the context. @@ -200,6 +221,8 @@ struct damon_ctx; * of its update. The value will be used for regions adjustment threshold. * @reset_aggregated should reset the access monitoring results that aggregated * by @check_accesses. + * @get_scheme_score should return the priority score of a region for a scheme + * as an integer in [0, &DAMOS_MAX_SCORE]. * @apply_scheme is called from @kdamond when a region for user provided * DAMON-based operation scheme is found. It should apply the scheme's action * to the region. This is not used for &DAMON_ARBITRARY_TARGET case. @@ -213,6 +236,9 @@ struct damon_primitive { void (*prepare_access_checks)(struct damon_ctx *context); unsigned int (*check_accesses)(struct damon_ctx *context); void (*reset_aggregated)(struct damon_ctx *context); + int (*get_scheme_score)(struct damon_ctx *context, + struct damon_target *t, struct damon_region *r, + struct damos *scheme); int (*apply_scheme)(struct damon_ctx *context, struct damon_target *t, struct damon_region *r, struct damos *scheme); bool (*target_valid)(void *target); diff --git a/mm/damon/core.c b/mm/damon/core.c index d1da4bef96ed..fad25778e2ec 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -12,6 +12,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -110,6 +111,9 @@ struct damos *damon_new_scheme( scheme->quota.ms = quota->ms; scheme->quota.sz = quota->sz; scheme->quota.reset_interval = quota->reset_interval; + scheme->quota.weight_sz = quota->weight_sz; + scheme->quota.weight_nr_accesses = quota->weight_nr_accesses; + scheme->quota.weight_age = quota->weight_age; scheme->quota.total_charged_sz = 0; scheme->quota.total_charged_ns = 0; scheme->quota.esz = 0; @@ -545,6 +549,28 @@ static void damon_split_region_at(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, unsigned long sz_r); +static bool __damos_valid_target(struct damon_region *r, struct damos *s) +{ + unsigned long sz; + + sz = r->ar.end - r->ar.start; + return s->min_sz_region <= sz && sz <= s->max_sz_region && + s->min_nr_accesses <= r->nr_accesses && + r->nr_accesses <= s->max_nr_accesses && + s->min_age_region <= r->age && r->age <= s->max_age_region; +} + +static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t, + struct damon_region *r, struct damos *s) +{ + bool ret = __damos_valid_target(r, s); + + if (!ret || !s->quota.esz || !c->primitive.get_scheme_score) + return ret; + + return c->primitive.get_scheme_score(c, t, r, s) >= s->quota.min_score; +} + static void damon_do_apply_schemes(struct damon_ctx *c, struct damon_target *t, struct damon_region *r) @@ -591,13 +617,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, quota->charge_addr_from = 0; } - /* Check the target regions condition */ - if (sz < s->min_sz_region || s->max_sz_region < sz) - continue; - if (r->nr_accesses < s->min_nr_accesses || - s->max_nr_accesses < r->nr_accesses) - continue; - if (r->age < s->min_age_region || s->max_age_region < r->age) + if (!damos_valid_target(c, t, r, s)) continue; /* Apply the scheme */ @@ -661,6 +681,8 @@ static void kdamond_apply_schemes(struct damon_ctx *c) damon_for_each_scheme(s, c) { struct damos_quota *quota = &s->quota; + unsigned long cumulated_sz; + unsigned int score, max_score = 0; if (!quota->ms && !quota->sz) continue; @@ -674,6 +696,32 @@ static void kdamond_apply_schemes(struct damon_ctx *c) quota->charged_sz = 0; damos_set_effective_quota(quota); } + + if (!c->primitive.get_scheme_score) + continue; + + /* Fill up the score histogram */ + memset(quota->histogram, 0, sizeof(quota->histogram)); + damon_for_each_target(t, c) { + damon_for_each_region(r, t) { + if (!__damos_valid_target(r, s)) + continue; + score = c->primitive.get_scheme_score( + c, t, r, s); + quota->histogram[score] += + r->ar.end - r->ar.start; + if (score > max_score) + max_score = score; + } + } + + /* Set the min score limit */ + for (cumulated_sz = 0, score = max_score; ; score--) { + cumulated_sz += quota->histogram[score]; + if (cumulated_sz >= quota->esz || !score) + break; + } + quota->min_score = score; } damon_for_each_target(t, c) { From 798e889699417520783dd427a4357285b1b2f69f Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:37 -0700 Subject: [PATCH 085/156] UPSTREAM: mm/damon/vaddr,paddr: support pageout prioritization This makes the default monitoring primitives for virtual address spaces and the physical address sapce to support memory regions prioritization for 'PAGEOUT' DAMOS action. It calculates hotness of each region as weighted sum of 'nr_accesses' and 'age' of the region and get the priority score as reverse of the hotness, so that cold regions can be paged out first. Link: https://lkml.kernel.org/r/20211019150731.16699-9-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 198f0f4c58b9f481e4e51c8c70a6ab9852bbab7f) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I5f8f8d5e3b0669ec48fa6d698b963091c892106d --- include/linux/damon.h | 4 ++++ mm/damon/paddr.c | 14 +++++++++++++ mm/damon/prmtv-common.c | 46 +++++++++++++++++++++++++++++++++++++++++ mm/damon/prmtv-common.h | 3 +++ mm/damon/vaddr.c | 15 ++++++++++++++ 5 files changed, 82 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 5d47ad9e3911..1217566a0ebc 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -421,6 +421,8 @@ bool damon_va_target_valid(void *t); void damon_va_cleanup(struct damon_ctx *ctx); int damon_va_apply_scheme(struct damon_ctx *context, struct damon_target *t, struct damon_region *r, struct damos *scheme); +int damon_va_scheme_score(struct damon_ctx *context, struct damon_target *t, + struct damon_region *r, struct damos *scheme); void damon_va_set_primitives(struct damon_ctx *ctx); #endif /* CONFIG_DAMON_VADDR */ @@ -433,6 +435,8 @@ unsigned int damon_pa_check_accesses(struct damon_ctx *ctx); bool damon_pa_target_valid(void *t); int damon_pa_apply_scheme(struct damon_ctx *context, struct damon_target *t, struct damon_region *r, struct damos *scheme); +int damon_pa_scheme_score(struct damon_ctx *context, struct damon_target *t, + struct damon_region *r, struct damos *scheme); void damon_pa_set_primitives(struct damon_ctx *ctx); #endif /* CONFIG_DAMON_PADDR */ diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 957ada55de77..a496d6f203d6 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -246,6 +246,19 @@ int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, return 0; } +int damon_pa_scheme_score(struct damon_ctx *context, struct damon_target *t, + struct damon_region *r, struct damos *scheme) +{ + switch (scheme->action) { + case DAMOS_PAGEOUT: + return damon_pageout_score(context, r, scheme); + default: + break; + } + + return DAMOS_MAX_SCORE; +} + void damon_pa_set_primitives(struct damon_ctx *ctx) { ctx->primitive.init = NULL; @@ -256,4 +269,5 @@ void damon_pa_set_primitives(struct damon_ctx *ctx) ctx->primitive.target_valid = damon_pa_target_valid; ctx->primitive.cleanup = NULL; ctx->primitive.apply_scheme = damon_pa_apply_scheme; + ctx->primitive.get_scheme_score = damon_pa_scheme_score; } diff --git a/mm/damon/prmtv-common.c b/mm/damon/prmtv-common.c index 7e62ee54fb54..92a04f5831d6 100644 --- a/mm/damon/prmtv-common.c +++ b/mm/damon/prmtv-common.c @@ -85,3 +85,49 @@ void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr) put_page(page); #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ } + +#define DAMON_MAX_SUBSCORE (100) +#define DAMON_MAX_AGE_IN_LOG (32) + +int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, + struct damos *s) +{ + unsigned int max_nr_accesses; + int freq_subscore; + unsigned int age_in_sec; + int age_in_log, age_subscore; + unsigned int freq_weight = s->quota.weight_nr_accesses; + unsigned int age_weight = s->quota.weight_age; + int hotness; + + max_nr_accesses = c->aggr_interval / c->sample_interval; + freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses; + + age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000; + for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec; + age_in_log++, age_in_sec >>= 1) + ; + + /* If frequency is 0, higher age means it's colder */ + if (freq_subscore == 0) + age_in_log *= -1; + + /* + * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG]. + * Scale it to be in [0, 100] and set it as age subscore. + */ + age_in_log += DAMON_MAX_AGE_IN_LOG; + age_subscore = age_in_log * DAMON_MAX_SUBSCORE / + DAMON_MAX_AGE_IN_LOG / 2; + + hotness = (freq_weight * freq_subscore + age_weight * age_subscore); + if (freq_weight + age_weight) + hotness /= freq_weight + age_weight; + /* + * Transform it to fit in [0, DAMOS_MAX_SCORE] + */ + hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; + + /* Return coldness of the region */ + return DAMOS_MAX_SCORE - hotness; +} diff --git a/mm/damon/prmtv-common.h b/mm/damon/prmtv-common.h index 7093d19e5d42..61f27037603e 100644 --- a/mm/damon/prmtv-common.h +++ b/mm/damon/prmtv-common.h @@ -15,3 +15,6 @@ struct page *damon_get_page(unsigned long pfn); void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr); void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr); + +int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, + struct damos *s); diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 758501b8d97d..675cd8c7df9b 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -633,6 +633,20 @@ int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, return damos_madvise(t, r, madv_action); } +int damon_va_scheme_score(struct damon_ctx *context, struct damon_target *t, + struct damon_region *r, struct damos *scheme) +{ + + switch (scheme->action) { + case DAMOS_PAGEOUT: + return damon_pageout_score(context, r, scheme); + default: + break; + } + + return DAMOS_MAX_SCORE; +} + void damon_va_set_primitives(struct damon_ctx *ctx) { ctx->primitive.init = damon_va_init; @@ -643,6 +657,7 @@ void damon_va_set_primitives(struct damon_ctx *ctx) ctx->primitive.target_valid = damon_va_target_valid; ctx->primitive.cleanup = NULL; ctx->primitive.apply_scheme = damon_va_apply_scheme; + ctx->primitive.get_scheme_score = damon_va_scheme_score; } #include "vaddr-test.h" From eb9cf87aa874fede150019598a822c6544845815 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:40 -0700 Subject: [PATCH 086/156] UPSTREAM: mm/damon/dbgfs: support prioritization weights This allows DAMON debugfs interface users set the prioritization weights by putting three more numbers to the 'schemes' file. Link: https://lkml.kernel.org/r/20211019150731.16699-10-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit f4a68b4a04e6db9397f7776c51d0f9715bd1a60e) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ia47dfebe9817da3e0d669ea996c0d3b44f980936 --- mm/damon/dbgfs.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 097e6745ba75..20c4feb8b918 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -105,13 +105,16 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) damon_for_each_scheme(s, c) { rc = scnprintf(&buf[written], len - written, - "%lu %lu %u %u %u %u %d %lu %lu %lu %lu %lu\n", + "%lu %lu %u %u %u %u %d %lu %lu %lu %u %u %u %lu %lu\n", s->min_sz_region, s->max_sz_region, s->min_nr_accesses, s->max_nr_accesses, s->min_age_region, s->max_age_region, s->action, s->quota.ms, s->quota.sz, s->quota.reset_interval, + s->quota.weight_sz, + s->quota.weight_nr_accesses, + s->quota.weight_age, s->stat_count, s->stat_sz); if (!rc) return -ENOMEM; @@ -193,11 +196,14 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, while (pos < len && *nr_schemes < max_nr_schemes) { struct damos_quota quota = {}; - ret = sscanf(&str[pos], "%lu %lu %u %u %u %u %u %lu %lu %lu%n", + ret = sscanf(&str[pos], + "%lu %lu %u %u %u %u %u %lu %lu %lu %u %u %u%n", &min_sz, &max_sz, &min_nr_a, &max_nr_a, &min_age, &max_age, &action, "a.ms, - "a.sz, "a.reset_interval, &parsed); - if (ret != 10) + "a.sz, "a.reset_interval, + "a.weight_sz, "a.weight_nr_accesses, + "a.weight_age, &parsed); + if (ret != 13) break; if (!damos_action_valid(action)) { pr_err("wrong action %d\n", action); From d4ba5298bef5cf6ae18c53f801d8c3e3535ecf16 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:44 -0700 Subject: [PATCH 087/156] UPSTREAM: tools/selftests/damon: update for regions prioritization of schemes This updates the DAMON selftests for 'schemes' debugfs file, as the file format is updated. Link: https://lkml.kernel.org/r/20211019150731.16699-11-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 5a0d6a08b81162fbe1e207f02571ace6d888f8b0) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I3a70ee8105555cc65987914c2c43973e5d3f9fd3 --- tools/testing/selftests/damon/debugfs_attrs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/damon/debugfs_attrs.sh b/tools/testing/selftests/damon/debugfs_attrs.sh index 8e33a7b584e7..466dbeb37e31 100644 --- a/tools/testing/selftests/damon/debugfs_attrs.sh +++ b/tools/testing/selftests/damon/debugfs_attrs.sh @@ -63,10 +63,10 @@ echo "$orig_content" > "$file" file="$DBGFS/schemes" orig_content=$(cat "$file") -test_write_succ "$file" "1 2 3 4 5 6 4 0 0 0" \ +test_write_succ "$file" "1 2 3 4 5 6 4 0 0 0 1 2 3" \ "$orig_content" "valid input" test_write_fail "$file" "1 2 -3 4 5 6 3 0 0 0" "$orig_content" "multi lines" +3 4 5 6 3 0 0 0 1 2 3" "$orig_content" "multi lines" test_write_succ "$file" "" "$orig_content" "disabling" echo "$orig_content" > "$file" From fb7d5f3b1a1a7f151efbc7a73d8b1939f5a8f5e9 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:47 -0700 Subject: [PATCH 088/156] UPSTREAM: mm/damon/schemes: activate schemes based on a watermarks mechanism DAMON-based operation schemes need to be manually turned on and off. In some use cases, however, the condition for turning a scheme on and off would depend on the system's situation. For example, schemes for proactive pages reclamation would need to be turned on when some memory pressure is detected, and turned off when the system has enough free memory. For easier control of schemes activation based on the system situation, this introduces a watermarks-based mechanism. The client can describe the watermark metric (e.g., amount of free memory in the system), watermark check interval, and three watermarks, namely high, mid, and low. If the scheme is deactivated, it only gets the metric and compare that to the three watermarks for every check interval. If the metric is higher than the high watermark, the scheme is deactivated. If the metric is between the mid watermark and the low watermark, the scheme is activated. If the metric is lower than the low watermark, the scheme is deactivated again. This is to allow users fall back to traditional page-granularity mechanisms. Link: https://lkml.kernel.org/r/20211019150731.16699-12-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit ee801b7dd7822a82fd7663048ad649545fac6df3) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I857d0aeab8aa58929ea71c8e9d42b3b93966fc51 --- include/linux/damon.h | 52 ++++++++++++++++++++++- mm/damon/core.c | 97 ++++++++++++++++++++++++++++++++++++++++++- mm/damon/dbgfs.c | 5 ++- 3 files changed, 151 insertions(+), 3 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 1217566a0ebc..c93325efddd7 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -146,6 +146,45 @@ struct damos_quota { unsigned int min_score; }; +/** + * enum damos_wmark_metric - Represents the watermark metric. + * + * @DAMOS_WMARK_NONE: Ignore the watermarks of the given scheme. + * @DAMOS_WMARK_FREE_MEM_RATE: Free memory rate of the system in [0,1000]. + */ +enum damos_wmark_metric { + DAMOS_WMARK_NONE, + DAMOS_WMARK_FREE_MEM_RATE, +}; + +/** + * struct damos_watermarks - Controls when a given scheme should be activated. + * @metric: Metric for the watermarks. + * @interval: Watermarks check time interval in microseconds. + * @high: High watermark. + * @mid: Middle watermark. + * @low: Low watermark. + * + * If &metric is &DAMOS_WMARK_NONE, the scheme is always active. Being active + * means DAMON does monitoring and applying the action of the scheme to + * appropriate memory regions. Else, DAMON checks &metric of the system for at + * least every &interval microseconds and works as below. + * + * If &metric is higher than &high, the scheme is inactivated. If &metric is + * between &mid and &low, the scheme is activated. If &metric is lower than + * &low, the scheme is inactivated. + */ +struct damos_watermarks { + enum damos_wmark_metric metric; + unsigned long interval; + unsigned long high; + unsigned long mid; + unsigned long low; + +/* private: */ + bool activated; +}; + /** * struct damos - Represents a Data Access Monitoring-based Operation Scheme. * @min_sz_region: Minimum size of target regions. @@ -156,6 +195,7 @@ struct damos_quota { * @max_age_region: Maximum age of target regions. * @action: &damo_action to be applied to the target regions. * @quota: Control the aggressiveness of this scheme. + * @wmarks: Watermarks for automated (in)activation of this scheme. * @stat_count: Total number of regions that this scheme is applied. * @stat_sz: Total size of regions that this scheme is applied. * @list: List head for siblings. @@ -166,6 +206,14 @@ struct damos_quota { * those. To avoid consuming too much CPU time or IO resources for the * &action, "a is used. * + * To do the work only when needed, schemes can be activated for specific + * system situations using &wmarks. If all schemes that registered to the + * monitoring context are inactive, DAMON stops monitoring either, and just + * repeatedly checks the watermarks. + * + * If all schemes that registered to a &struct damon_ctx are inactive, DAMON + * stops monitoring and just repeatedly checks the watermarks. + * * After applying the &action to each region, &stat_count and &stat_sz is * updated to reflect the number of regions and total size of regions that the * &action is applied. @@ -179,6 +227,7 @@ struct damos { unsigned int max_age_region; enum damos_action action; struct damos_quota quota; + struct damos_watermarks wmarks; unsigned long stat_count; unsigned long stat_sz; struct list_head list; @@ -384,7 +433,8 @@ struct damos *damon_new_scheme( unsigned long min_sz_region, unsigned long max_sz_region, unsigned int min_nr_accesses, unsigned int max_nr_accesses, unsigned int min_age_region, unsigned int max_age_region, - enum damos_action action, struct damos_quota *quota); + enum damos_action action, struct damos_quota *quota, + struct damos_watermarks *wmarks); void damon_add_scheme(struct damon_ctx *ctx, struct damos *s); void damon_destroy_scheme(struct damos *s); diff --git a/mm/damon/core.c b/mm/damon/core.c index fad25778e2ec..6993c60ae31c 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -90,7 +91,8 @@ struct damos *damon_new_scheme( unsigned long min_sz_region, unsigned long max_sz_region, unsigned int min_nr_accesses, unsigned int max_nr_accesses, unsigned int min_age_region, unsigned int max_age_region, - enum damos_action action, struct damos_quota *quota) + enum damos_action action, struct damos_quota *quota, + struct damos_watermarks *wmarks) { struct damos *scheme; @@ -122,6 +124,13 @@ struct damos *damon_new_scheme( scheme->quota.charge_target_from = NULL; scheme->quota.charge_addr_from = 0; + scheme->wmarks.metric = wmarks->metric; + scheme->wmarks.interval = wmarks->interval; + scheme->wmarks.high = wmarks->high; + scheme->wmarks.mid = wmarks->mid; + scheme->wmarks.low = wmarks->low; + scheme->wmarks.activated = true; + return scheme; } @@ -582,6 +591,9 @@ static void damon_do_apply_schemes(struct damon_ctx *c, unsigned long sz = r->ar.end - r->ar.start; struct timespec64 begin, end; + if (!s->wmarks.activated) + continue; + /* Check the quota */ if (quota->esz && quota->charged_sz >= quota->esz) continue; @@ -684,6 +696,9 @@ static void kdamond_apply_schemes(struct damon_ctx *c) unsigned long cumulated_sz; unsigned int score, max_score = 0; + if (!s->wmarks.activated) + continue; + if (!quota->ms && !quota->sz) continue; @@ -924,6 +939,83 @@ static bool kdamond_need_stop(struct damon_ctx *ctx) return true; } +static unsigned long damos_wmark_metric_value(enum damos_wmark_metric metric) +{ + struct sysinfo i; + + switch (metric) { + case DAMOS_WMARK_FREE_MEM_RATE: + si_meminfo(&i); + return i.freeram * 1000 / i.totalram; + default: + break; + } + return -EINVAL; +} + +/* + * Returns zero if the scheme is active. Else, returns time to wait for next + * watermark check in micro-seconds. + */ +static unsigned long damos_wmark_wait_us(struct damos *scheme) +{ + unsigned long metric; + + if (scheme->wmarks.metric == DAMOS_WMARK_NONE) + return 0; + + metric = damos_wmark_metric_value(scheme->wmarks.metric); + /* higher than high watermark or lower than low watermark */ + if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) { + if (scheme->wmarks.activated) + pr_debug("inactivate a scheme (%d) for %s wmark\n", + scheme->action, + metric > scheme->wmarks.high ? + "high" : "low"); + scheme->wmarks.activated = false; + return scheme->wmarks.interval; + } + + /* inactive and higher than middle watermark */ + if ((scheme->wmarks.high >= metric && metric >= scheme->wmarks.mid) && + !scheme->wmarks.activated) + return scheme->wmarks.interval; + + if (!scheme->wmarks.activated) + pr_debug("activate a scheme (%d)\n", scheme->action); + scheme->wmarks.activated = true; + return 0; +} + +static void kdamond_usleep(unsigned long usecs) +{ + if (usecs > 100 * 1000) + schedule_timeout_interruptible(usecs_to_jiffies(usecs)); + else + usleep_range(usecs, usecs + 1); +} + +/* Returns negative error code if it's not activated but should return */ +static int kdamond_wait_activation(struct damon_ctx *ctx) +{ + struct damos *s; + unsigned long wait_time; + unsigned long min_wait_time = 0; + + while (!kdamond_need_stop(ctx)) { + damon_for_each_scheme(s, ctx) { + wait_time = damos_wmark_wait_us(s); + if (!min_wait_time || wait_time < min_wait_time) + min_wait_time = wait_time; + } + if (!min_wait_time) + return 0; + + kdamond_usleep(min_wait_time); + } + return -EBUSY; +} + static void set_kdamond_stop(struct damon_ctx *ctx) { mutex_lock(&ctx->kdamond_lock); @@ -952,6 +1044,9 @@ static int kdamond_fn(void *data) sz_limit = damon_region_sz_limit(ctx); while (!kdamond_need_stop(ctx)) { + if (kdamond_wait_activation(ctx)) + continue; + if (ctx->primitive.prepare_access_checks) ctx->primitive.prepare_access_checks(ctx); if (ctx->callback.after_sampling && diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 20c4feb8b918..9f13060d1058 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -195,6 +195,9 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, *nr_schemes = 0; while (pos < len && *nr_schemes < max_nr_schemes) { struct damos_quota quota = {}; + struct damos_watermarks wmarks = { + .metric = DAMOS_WMARK_NONE, + }; ret = sscanf(&str[pos], "%lu %lu %u %u %u %u %u %lu %lu %lu %u %u %u%n", @@ -212,7 +215,7 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, pos += parsed; scheme = damon_new_scheme(min_sz, max_sz, min_nr_a, max_nr_a, - min_age, max_age, action, "a); + min_age, max_age, action, "a, &wmarks); if (!scheme) goto fail; From 88d44101df95beb06167ca50ea68b1ac61c35785 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:50 -0700 Subject: [PATCH 089/156] UPSTREAM: mm/damon/dbgfs: support watermarks This updates DAMON debugfs interface to support the watermarks based schemes activation. For this, now 'schemes' file receives five more values. Link: https://lkml.kernel.org/r/20211019150731.16699-13-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit ae666a6dddfd119da55cc1bad54f7cbd8b2ef54c) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I9b1b7c285ea04bd4dc9beefcc6a4e5368f093745 --- mm/damon/dbgfs.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 9f13060d1058..6828e463348b 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -105,7 +105,7 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) damon_for_each_scheme(s, c) { rc = scnprintf(&buf[written], len - written, - "%lu %lu %u %u %u %u %d %lu %lu %lu %u %u %u %lu %lu\n", + "%lu %lu %u %u %u %u %d %lu %lu %lu %u %u %u %d %lu %lu %lu %lu %lu %lu\n", s->min_sz_region, s->max_sz_region, s->min_nr_accesses, s->max_nr_accesses, s->min_age_region, s->max_age_region, @@ -115,6 +115,8 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) s->quota.weight_sz, s->quota.weight_nr_accesses, s->quota.weight_age, + s->wmarks.metric, s->wmarks.interval, + s->wmarks.high, s->wmarks.mid, s->wmarks.low, s->stat_count, s->stat_sz); if (!rc) return -ENOMEM; @@ -195,18 +197,18 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, *nr_schemes = 0; while (pos < len && *nr_schemes < max_nr_schemes) { struct damos_quota quota = {}; - struct damos_watermarks wmarks = { - .metric = DAMOS_WMARK_NONE, - }; + struct damos_watermarks wmarks; ret = sscanf(&str[pos], - "%lu %lu %u %u %u %u %u %lu %lu %lu %u %u %u%n", + "%lu %lu %u %u %u %u %u %lu %lu %lu %u %u %u %u %lu %lu %lu %lu%n", &min_sz, &max_sz, &min_nr_a, &max_nr_a, &min_age, &max_age, &action, "a.ms, "a.sz, "a.reset_interval, "a.weight_sz, "a.weight_nr_accesses, - "a.weight_age, &parsed); - if (ret != 13) + "a.weight_age, &wmarks.metric, + &wmarks.interval, &wmarks.high, &wmarks.mid, + &wmarks.low, &parsed); + if (ret != 18) break; if (!damos_action_valid(action)) { pr_err("wrong action %d\n", action); From 57b0fb6229606046473a27ae3d9e5036a0b1e278 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:54 -0700 Subject: [PATCH 090/156] UPSTREAM: selftests/damon: support watermarks This updates DAMON selftests for 'schemes' debugfs file to reflect the changes in the format. Link: https://lkml.kernel.org/r/20211019150731.16699-14-sj@kernel.org Signed-off-by: SeongJae Park Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 1dc90ccd15c55cc3edec508466db9248a841acad) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ic66b3aa8db37ba276624d80278ca8e35f2c07963 --- tools/testing/selftests/damon/debugfs_attrs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/damon/debugfs_attrs.sh b/tools/testing/selftests/damon/debugfs_attrs.sh index 466dbeb37e31..196b6640bf37 100644 --- a/tools/testing/selftests/damon/debugfs_attrs.sh +++ b/tools/testing/selftests/damon/debugfs_attrs.sh @@ -63,10 +63,10 @@ echo "$orig_content" > "$file" file="$DBGFS/schemes" orig_content=$(cat "$file") -test_write_succ "$file" "1 2 3 4 5 6 4 0 0 0 1 2 3" \ +test_write_succ "$file" "1 2 3 4 5 6 4 0 0 0 1 2 3 1 100 3 2 1" \ "$orig_content" "valid input" test_write_fail "$file" "1 2 -3 4 5 6 3 0 0 0 1 2 3" "$orig_content" "multi lines" +3 4 5 6 3 0 0 0 1 2 3 1 100 3 2 1" "$orig_content" "multi lines" test_write_succ "$file" "" "$orig_content" "disabling" echo "$orig_content" > "$file" From 4d321b786fc023f2c68c3179dfa4b1229d5d29c9 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 5 Nov 2021 13:47:57 -0700 Subject: [PATCH 091/156] UPSTREAM: mm/damon: introduce DAMON-based Reclamation (DAMON_RECLAIM) This implements a new kernel subsystem that finds cold memory regions using DAMON and reclaims those immediately. It is intended to be used as proactive lightweigh reclamation logic for light memory pressure. For heavy memory pressure, it could be inactivated and fall back to the traditional page-scanning based reclamation. It's implemented on top of DAMON framework to use the DAMON-based Operation Schemes (DAMOS) feature. It utilizes all the DAMOS features including speed limit, prioritization, and watermarks. It could be enabled and tuned in boot time via the kernel boot parameter, and in run time via its module parameters ('/sys/module/damon_reclaim/parameters/') interface. [yangyingliang@huawei.com: fix error return code in damon_reclaim_turn()] Link: https://lkml.kernel.org/r/20211025124500.2758060-1-yangyingliang@huawei.com Link: https://lkml.kernel.org/r/20211019150731.16699-15-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Yang Yingliang Cc: Amit Shah Cc: Benjamin Herrenschmidt Cc: David Hildenbrand Cc: David Rientjes Cc: David Woodhouse Cc: Greg Thelen Cc: Jonathan Cameron Cc: Jonathan Corbet Cc: Leonard Foerster Cc: Marco Elver Cc: Markus Boehme Cc: Shakeel Butt Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 43b0536cb4710e7bb591edfda7e68a1c327a3409) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Id57795ee37e5db046f782d2bffcd0533fe476558 --- mm/damon/Kconfig | 12 ++ mm/damon/Makefile | 1 + mm/damon/reclaim.c | 356 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 369 insertions(+) create mode 100644 mm/damon/reclaim.c diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig index ca33b289ebbe..5bcf05851ad0 100644 --- a/mm/damon/Kconfig +++ b/mm/damon/Kconfig @@ -73,4 +73,16 @@ config DAMON_DBGFS_KUNIT_TEST If unsure, say N. +config DAMON_RECLAIM + bool "Build DAMON-based reclaim (DAMON_RECLAIM)" + depends on DAMON_PADDR + help + This builds the DAMON-based reclamation subsystem. It finds pages + that not accessed for a long time (cold) using DAMON and reclaim + those. + + This is suggested to be used as a proactive and lightweight + reclamation under light memory pressure, while the traditional page + scanning-based reclamation is used for heavy pressure. + endmenu diff --git a/mm/damon/Makefile b/mm/damon/Makefile index 8d9b0df79702..f7d5ac377a2b 100644 --- a/mm/damon/Makefile +++ b/mm/damon/Makefile @@ -4,3 +4,4 @@ obj-$(CONFIG_DAMON) := core.o obj-$(CONFIG_DAMON_VADDR) += prmtv-common.o vaddr.o obj-$(CONFIG_DAMON_PADDR) += prmtv-common.o paddr.o obj-$(CONFIG_DAMON_DBGFS) += dbgfs.o +obj-$(CONFIG_DAMON_RECLAIM) += reclaim.o diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c new file mode 100644 index 000000000000..dc1485044eaf --- /dev/null +++ b/mm/damon/reclaim.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DAMON-based page reclamation + * + * Author: SeongJae Park + */ + +#define pr_fmt(fmt) "damon-reclaim: " fmt + +#include +#include +#include +#include +#include + +#ifdef MODULE_PARAM_PREFIX +#undef MODULE_PARAM_PREFIX +#endif +#define MODULE_PARAM_PREFIX "damon_reclaim." + +/* + * Enable or disable DAMON_RECLAIM. + * + * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``. + * Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could + * do no real monitoring and reclamation due to the watermarks-based activation + * condition. Refer to below descriptions for the watermarks parameter for + * this. + */ +static bool enabled __read_mostly; +module_param(enabled, bool, 0600); + +/* + * Time threshold for cold memory regions identification in microseconds. + * + * If a memory region is not accessed for this or longer time, DAMON_RECLAIM + * identifies the region as cold, and reclaims. 120 seconds by default. + */ +static unsigned long min_age __read_mostly = 120000000; +module_param(min_age, ulong, 0600); + +/* + * Limit of time for trying the reclamation in milliseconds. + * + * DAMON_RECLAIM tries to use only up to this time within a time window + * (quota_reset_interval_ms) for trying reclamation of cold pages. This can be + * used for limiting CPU consumption of DAMON_RECLAIM. If the value is zero, + * the limit is disabled. + * + * 10 ms by default. + */ +static unsigned long quota_ms __read_mostly = 10; +module_param(quota_ms, ulong, 0600); + +/* + * Limit of size of memory for the reclamation in bytes. + * + * DAMON_RECLAIM charges amount of memory which it tried to reclaim within a + * time window (quota_reset_interval_ms) and makes no more than this limit is + * tried. This can be used for limiting consumption of CPU and IO. If this + * value is zero, the limit is disabled. + * + * 128 MiB by default. + */ +static unsigned long quota_sz __read_mostly = 128 * 1024 * 1024; +module_param(quota_sz, ulong, 0600); + +/* + * The time/size quota charge reset interval in milliseconds. + * + * The charge reset interval for the quota of time (quota_ms) and size + * (quota_sz). That is, DAMON_RECLAIM does not try reclamation for more than + * quota_ms milliseconds or quota_sz bytes within quota_reset_interval_ms + * milliseconds. + * + * 1 second by default. + */ +static unsigned long quota_reset_interval_ms __read_mostly = 1000; +module_param(quota_reset_interval_ms, ulong, 0600); + +/* + * The watermarks check time interval in microseconds. + * + * Minimal time to wait before checking the watermarks, when DAMON_RECLAIM is + * enabled but inactive due to its watermarks rule. 5 seconds by default. + */ +static unsigned long wmarks_interval __read_mostly = 5000000; +module_param(wmarks_interval, ulong, 0600); + +/* + * Free memory rate (per thousand) for the high watermark. + * + * If free memory of the system in bytes per thousand bytes is higher than + * this, DAMON_RECLAIM becomes inactive, so it does nothing but periodically + * checks the watermarks. 500 (50%) by default. + */ +static unsigned long wmarks_high __read_mostly = 500; +module_param(wmarks_high, ulong, 0600); + +/* + * Free memory rate (per thousand) for the middle watermark. + * + * If free memory of the system in bytes per thousand bytes is between this and + * the low watermark, DAMON_RECLAIM becomes active, so starts the monitoring + * and the reclaiming. 400 (40%) by default. + */ +static unsigned long wmarks_mid __read_mostly = 400; +module_param(wmarks_mid, ulong, 0600); + +/* + * Free memory rate (per thousand) for the low watermark. + * + * If free memory of the system in bytes per thousand bytes is lower than this, + * DAMON_RECLAIM becomes inactive, so it does nothing but periodically checks + * the watermarks. In the case, the system falls back to the LRU-based page + * granularity reclamation logic. 200 (20%) by default. + */ +static unsigned long wmarks_low __read_mostly = 200; +module_param(wmarks_low, ulong, 0600); + +/* + * Sampling interval for the monitoring in microseconds. + * + * The sampling interval of DAMON for the cold memory monitoring. Please refer + * to the DAMON documentation for more detail. 5 ms by default. + */ +static unsigned long sample_interval __read_mostly = 5000; +module_param(sample_interval, ulong, 0600); + +/* + * Aggregation interval for the monitoring in microseconds. + * + * The aggregation interval of DAMON for the cold memory monitoring. Please + * refer to the DAMON documentation for more detail. 100 ms by default. + */ +static unsigned long aggr_interval __read_mostly = 100000; +module_param(aggr_interval, ulong, 0600); + +/* + * Minimum number of monitoring regions. + * + * The minimal number of monitoring regions of DAMON for the cold memory + * monitoring. This can be used to set lower-bound of the monitoring quality. + * But, setting this too high could result in increased monitoring overhead. + * Please refer to the DAMON documentation for more detail. 10 by default. + */ +static unsigned long min_nr_regions __read_mostly = 10; +module_param(min_nr_regions, ulong, 0600); + +/* + * Maximum number of monitoring regions. + * + * The maximum number of monitoring regions of DAMON for the cold memory + * monitoring. This can be used to set upper-bound of the monitoring overhead. + * However, setting this too low could result in bad monitoring quality. + * Please refer to the DAMON documentation for more detail. 1000 by default. + */ +static unsigned long max_nr_regions __read_mostly = 1000; +module_param(max_nr_regions, ulong, 0600); + +/* + * Start of the target memory region in physical address. + * + * The start physical address of memory region that DAMON_RECLAIM will do work + * against. By default, biggest System RAM is used as the region. + */ +static unsigned long monitor_region_start __read_mostly; +module_param(monitor_region_start, ulong, 0600); + +/* + * End of the target memory region in physical address. + * + * The end physical address of memory region that DAMON_RECLAIM will do work + * against. By default, biggest System RAM is used as the region. + */ +static unsigned long monitor_region_end __read_mostly; +module_param(monitor_region_end, ulong, 0600); + +/* + * PID of the DAMON thread + * + * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. + * Else, -1. + */ +static int kdamond_pid __read_mostly = -1; +module_param(kdamond_pid, int, 0400); + +static struct damon_ctx *ctx; +static struct damon_target *target; + +struct damon_reclaim_ram_walk_arg { + unsigned long start; + unsigned long end; +}; + +static int walk_system_ram(struct resource *res, void *arg) +{ + struct damon_reclaim_ram_walk_arg *a = arg; + + if (a->end - a->start < res->end - res->start) { + a->start = res->start; + a->end = res->end; + } + return 0; +} + +/* + * Find biggest 'System RAM' resource and store its start and end address in + * @start and @end, respectively. If no System RAM is found, returns false. + */ +static bool get_monitoring_region(unsigned long *start, unsigned long *end) +{ + struct damon_reclaim_ram_walk_arg arg = {}; + + walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram); + if (arg.end <= arg.start) + return false; + + *start = arg.start; + *end = arg.end; + return true; +} + +static struct damos *damon_reclaim_new_scheme(void) +{ + struct damos_watermarks wmarks = { + .metric = DAMOS_WMARK_FREE_MEM_RATE, + .interval = wmarks_interval, + .high = wmarks_high, + .mid = wmarks_mid, + .low = wmarks_low, + }; + struct damos_quota quota = { + /* + * Do not try reclamation for more than quota_ms milliseconds + * or quota_sz bytes within quota_reset_interval_ms. + */ + .ms = quota_ms, + .sz = quota_sz, + .reset_interval = quota_reset_interval_ms, + /* Within the quota, page out older regions first. */ + .weight_sz = 0, + .weight_nr_accesses = 0, + .weight_age = 1 + }; + struct damos *scheme = damon_new_scheme( + /* Find regions having PAGE_SIZE or larger size */ + PAGE_SIZE, ULONG_MAX, + /* and not accessed at all */ + 0, 0, + /* for min_age or more micro-seconds, and */ + min_age / aggr_interval, UINT_MAX, + /* page out those, as soon as found */ + DAMOS_PAGEOUT, + /* under the quota. */ + "a, + /* (De)activate this according to the watermarks. */ + &wmarks); + + return scheme; +} + +static int damon_reclaim_turn(bool on) +{ + struct damon_region *region; + struct damos *scheme; + int err; + + if (!on) { + err = damon_stop(&ctx, 1); + if (!err) + kdamond_pid = -1; + return err; + } + + err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0, + min_nr_regions, max_nr_regions); + if (err) + return err; + + if (monitor_region_start > monitor_region_end) + return -EINVAL; + if (!monitor_region_start && !monitor_region_end && + !get_monitoring_region(&monitor_region_start, + &monitor_region_end)) + return -EINVAL; + /* DAMON will free this on its own when finish monitoring */ + region = damon_new_region(monitor_region_start, monitor_region_end); + if (!region) + return -ENOMEM; + damon_add_region(region, target); + + /* Will be freed by 'damon_set_schemes()' below */ + scheme = damon_reclaim_new_scheme(); + if (!scheme) { + err = -ENOMEM; + goto free_region_out; + } + err = damon_set_schemes(ctx, &scheme, 1); + if (err) + goto free_scheme_out; + + err = damon_start(&ctx, 1); + if (!err) { + kdamond_pid = ctx->kdamond->pid; + return 0; + } + +free_scheme_out: + damon_destroy_scheme(scheme); +free_region_out: + damon_destroy_region(region, target); + return err; +} + +#define ENABLE_CHECK_INTERVAL_MS 1000 +static struct delayed_work damon_reclaim_timer; +static void damon_reclaim_timer_fn(struct work_struct *work) +{ + static bool last_enabled; + bool now_enabled; + + now_enabled = enabled; + if (last_enabled != now_enabled) { + if (!damon_reclaim_turn(now_enabled)) + last_enabled = now_enabled; + else + enabled = last_enabled; + } + + schedule_delayed_work(&damon_reclaim_timer, + msecs_to_jiffies(ENABLE_CHECK_INTERVAL_MS)); +} +static DECLARE_DELAYED_WORK(damon_reclaim_timer, damon_reclaim_timer_fn); + +static int __init damon_reclaim_init(void) +{ + ctx = damon_new_ctx(); + if (!ctx) + return -ENOMEM; + + damon_pa_set_primitives(ctx); + + /* 4242 means nothing but fun */ + target = damon_new_target(4242); + if (!target) { + damon_destroy_ctx(ctx); + return -ENOMEM; + } + damon_add_target(ctx, target); + + schedule_delayed_work(&damon_reclaim_timer, 0); + return 0; +} + +module_init(damon_reclaim_init); From 8c2db14f3f6a2470b1f8c113c7fa79222d36fe37 Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 5 Nov 2021 13:48:04 -0700 Subject: [PATCH 092/156] UPSTREAM: mm/damon: remove unnecessary variable initialization Patch series "mm/damon: Fix some small bugs", v4. This patch (of 2): In 'damon_va_apply_three_regions' there is no need to set variable 'i' to zero. Link: https://lkml.kernel.org/r/b7df8d3dad0943a37e01f60c441b1968b2b20354.1634720326.git.xhao@linux.alibaba.com Link: https://lkml.kernel.org/r/cover.1634720326.git.xhao@linux.alibaba.com Signed-off-by: Xin Hao Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit a460a36034bad4403c2c62e04a521bc6987ae5db) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I81c05842e2372f3ab7a3d74df5358fbfb1c96342 --- mm/damon/vaddr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 675cd8c7df9b..35fe49080ee9 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -306,7 +306,7 @@ static void damon_va_apply_three_regions(struct damon_target *t, struct damon_addr_range bregions[3]) { struct damon_region *r, *next; - unsigned int i = 0; + unsigned int i; /* Remove regions which are not in the three big regions now */ damon_for_each_region_safe(r, next, t) { From f1456aa48d1e439ecdcd07961bc56474c5bb31df Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 5 Nov 2021 13:48:07 -0700 Subject: [PATCH 093/156] UPSTREAM: mm/damon/dbgfs: add adaptive_targets list check before enable monitor_on When the ctx->adaptive_targets list is empty, I did some test on monitor_on interface like this. # cat /sys/kernel/debug/damon/target_ids # # echo on > /sys/kernel/debug/damon/monitor_on # damon: kdamond (5390) starts Though the ctx->adaptive_targets list is empty, but the kthread_run still be called, and the kdamond.x thread still be created, this is meaningless. So there adds a judgment in 'dbgfs_monitor_on_write', if the ctx->adaptive_targets list is empty, return -EINVAL. Link: https://lkml.kernel.org/r/0a60a6e8ec9d71989e0848a4dc3311996ca3b5d4.1634720326.git.xhao@linux.alibaba.com Signed-off-by: Xin Hao Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit b5ca3e83ddb05342b1b30700b999cb9b107511f6) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I768592c5fded26ead98ba94a46d43fd64118f589 --- include/linux/damon.h | 1 + mm/damon/core.c | 5 +++++ mm/damon/dbgfs.c | 15 ++++++++++++--- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index c93325efddd7..fa7f32614b65 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -440,6 +440,7 @@ void damon_destroy_scheme(struct damos *s); struct damon_target *damon_new_target(unsigned long id); void damon_add_target(struct damon_ctx *ctx, struct damon_target *t); +bool damon_targets_empty(struct damon_ctx *ctx); void damon_free_target(struct damon_target *t); void damon_destroy_target(struct damon_target *t); unsigned int damon_nr_regions(struct damon_target *t); diff --git a/mm/damon/core.c b/mm/damon/core.c index 6993c60ae31c..46a6afea3030 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -180,6 +180,11 @@ void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) list_add_tail(&t->list, &ctx->adaptive_targets); } +bool damon_targets_empty(struct damon_ctx *ctx) +{ + return list_empty(&ctx->adaptive_targets); +} + static void damon_del_target(struct damon_target *t) { list_del(&t->list); diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 6828e463348b..befb27a29aab 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -878,12 +878,21 @@ static ssize_t dbgfs_monitor_on_write(struct file *file, return -EINVAL; } - if (!strncmp(kbuf, "on", count)) + if (!strncmp(kbuf, "on", count)) { + int i; + + for (i = 0; i < dbgfs_nr_ctxs; i++) { + if (damon_targets_empty(dbgfs_ctxs[i])) { + kfree(kbuf); + return -EINVAL; + } + } ret = damon_start(dbgfs_ctxs, dbgfs_nr_ctxs); - else if (!strncmp(kbuf, "off", count)) + } else if (!strncmp(kbuf, "off", count)) { ret = damon_stop(dbgfs_ctxs, dbgfs_nr_ctxs); - else + } else { ret = -EINVAL; + } if (!ret) ret = count; From bb15a0784236e06b698b2670b80d777517901706 Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Fri, 5 Nov 2021 13:48:22 -0700 Subject: [PATCH 094/156] UPSTREAM: mm/damon: simplify stop mechanism A kernel thread can exit gracefully with kthread_stop(). So we don't need a new flag 'kdamond_stop'. And to make sure the task struct is not freed when accessing it, get reference to it before termination. Link: https://lkml.kernel.org/r/20211027130517.4404-1-changbin.du@gmail.com Signed-off-by: Changbin Du Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 0f91d13366a402420bf98eaaf393db03946c13e0) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I7b959fc6cd114dc244c895cf19b9fd864b81cd12 --- include/linux/damon.h | 1 - mm/damon/core.c | 51 +++++++++++++------------------------------ 2 files changed, 15 insertions(+), 37 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index fa7f32614b65..321de9d72360 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -381,7 +381,6 @@ struct damon_ctx { /* public: */ struct task_struct *kdamond; - bool kdamond_stop; struct mutex kdamond_lock; struct damon_primitive primitive; diff --git a/mm/damon/core.c b/mm/damon/core.c index 46a6afea3030..f37c17b53814 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -390,17 +390,6 @@ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) return sz; } -static bool damon_kdamond_running(struct damon_ctx *ctx) -{ - bool running; - - mutex_lock(&ctx->kdamond_lock); - running = ctx->kdamond != NULL; - mutex_unlock(&ctx->kdamond_lock); - - return running; -} - static int kdamond_fn(void *data); /* @@ -418,7 +407,6 @@ static int __damon_start(struct damon_ctx *ctx) mutex_lock(&ctx->kdamond_lock); if (!ctx->kdamond) { err = 0; - ctx->kdamond_stop = false; ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d", nr_running_ctxs); if (IS_ERR(ctx->kdamond)) { @@ -474,13 +462,15 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs) */ static int __damon_stop(struct damon_ctx *ctx) { + struct task_struct *tsk; + mutex_lock(&ctx->kdamond_lock); - if (ctx->kdamond) { - ctx->kdamond_stop = true; + tsk = ctx->kdamond; + if (tsk) { + get_task_struct(tsk); mutex_unlock(&ctx->kdamond_lock); - while (damon_kdamond_running(ctx)) - usleep_range(ctx->sample_interval, - ctx->sample_interval * 2); + kthread_stop(tsk); + put_task_struct(tsk); return 0; } mutex_unlock(&ctx->kdamond_lock); @@ -925,12 +915,8 @@ static bool kdamond_need_update_primitive(struct damon_ctx *ctx) static bool kdamond_need_stop(struct damon_ctx *ctx) { struct damon_target *t; - bool stop; - mutex_lock(&ctx->kdamond_lock); - stop = ctx->kdamond_stop; - mutex_unlock(&ctx->kdamond_lock); - if (stop) + if (kthread_should_stop()) return true; if (!ctx->primitive.target_valid) @@ -1021,13 +1007,6 @@ static int kdamond_wait_activation(struct damon_ctx *ctx) return -EBUSY; } -static void set_kdamond_stop(struct damon_ctx *ctx) -{ - mutex_lock(&ctx->kdamond_lock); - ctx->kdamond_stop = true; - mutex_unlock(&ctx->kdamond_lock); -} - /* * The monitoring daemon that runs as a kernel thread */ @@ -1038,17 +1017,18 @@ static int kdamond_fn(void *data) struct damon_region *r, *next; unsigned int max_nr_accesses = 0; unsigned long sz_limit = 0; + bool done = false; pr_debug("kdamond (%d) starts\n", current->pid); if (ctx->primitive.init) ctx->primitive.init(ctx); if (ctx->callback.before_start && ctx->callback.before_start(ctx)) - set_kdamond_stop(ctx); + done = true; sz_limit = damon_region_sz_limit(ctx); - while (!kdamond_need_stop(ctx)) { + while (!kdamond_need_stop(ctx) && !done) { if (kdamond_wait_activation(ctx)) continue; @@ -1056,7 +1036,7 @@ static int kdamond_fn(void *data) ctx->primitive.prepare_access_checks(ctx); if (ctx->callback.after_sampling && ctx->callback.after_sampling(ctx)) - set_kdamond_stop(ctx); + done = true; usleep_range(ctx->sample_interval, ctx->sample_interval + 1); @@ -1069,7 +1049,7 @@ static int kdamond_fn(void *data) sz_limit); if (ctx->callback.after_aggregation && ctx->callback.after_aggregation(ctx)) - set_kdamond_stop(ctx); + done = true; kdamond_apply_schemes(ctx); kdamond_reset_aggregated(ctx); kdamond_split_regions(ctx); @@ -1088,9 +1068,8 @@ static int kdamond_fn(void *data) damon_destroy_region(r, t); } - if (ctx->callback.before_terminate && - ctx->callback.before_terminate(ctx)) - set_kdamond_stop(ctx); + if (ctx->callback.before_terminate) + ctx->callback.before_terminate(ctx); if (ctx->primitive.cleanup) ctx->primitive.cleanup(ctx); From b48f28f49c67fe598fea3de60e991a165ee38904 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 5 Nov 2021 13:48:24 -0700 Subject: [PATCH 095/156] UPSTREAM: mm/damon: fix a few spelling mistakes in comments and a pr_debug message There are a few spelling mistakes in the code. Fix these. Link: https://lkml.kernel.org/r/20211028184157.614544-1-colin.i.king@gmail.com Signed-off-by: Colin Ian King Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 0107865541961ee128149c9873996d32143a74d0) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I09c9421b1abeed6c002cf5fef07d229cf0f22ebd --- mm/damon/core.c | 2 +- mm/damon/dbgfs-test.h | 2 +- mm/damon/vaddr-test.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index f37c17b53814..c381b3c525d0 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -959,7 +959,7 @@ static unsigned long damos_wmark_wait_us(struct damos *scheme) /* higher than high watermark or lower than low watermark */ if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) { if (scheme->wmarks.activated) - pr_debug("inactivate a scheme (%d) for %s wmark\n", + pr_debug("deactivate a scheme (%d) for %s wmark\n", scheme->action, metric > scheme->wmarks.high ? "high" : "low"); diff --git a/mm/damon/dbgfs-test.h b/mm/damon/dbgfs-test.h index 104b22957616..86b9f9528231 100644 --- a/mm/damon/dbgfs-test.h +++ b/mm/damon/dbgfs-test.h @@ -145,7 +145,7 @@ static void damon_dbgfs_test_set_init_regions(struct kunit *test) KUNIT_EXPECT_STREQ(test, (char *)buf, expect); } - /* Put invlid inputs and check the return error code */ + /* Put invalid inputs and check the return error code */ for (i = 0; i < ARRAY_SIZE(invalid_inputs); i++) { input = invalid_inputs[i]; pr_info("input: %s\n", input); diff --git a/mm/damon/vaddr-test.h b/mm/damon/vaddr-test.h index 1f5c13257dba..ecfd0b2ed222 100644 --- a/mm/damon/vaddr-test.h +++ b/mm/damon/vaddr-test.h @@ -233,7 +233,7 @@ static void damon_test_apply_three_regions3(struct kunit *test) * and 70-100) has totally freed and mapped to different area (30-32 and * 65-68). The target regions which were in the old second and third big * regions should now be removed and new target regions covering the new second - * and third big regions should be crated. + * and third big regions should be created. */ static void damon_test_apply_three_regions4(struct kunit *test) { From 55a2d2f5a71a2cc6444a005195dd4c129084209f Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Fri, 5 Nov 2021 13:48:27 -0700 Subject: [PATCH 096/156] UPSTREAM: mm/damon: remove return value from before_terminate callback Since the return value of 'before_terminate' callback is never used, we make it have no return value. Link: https://lkml.kernel.org/r/20211029005023.8895-1-changbin.du@gmail.com Signed-off-by: Changbin Du Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 658f9ae761b5965893727dd4edcdad56e5a439bb) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ida44623827a7a667118a2af329f6e58a57941fc9 --- include/linux/damon.h | 2 +- mm/damon/dbgfs.c | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 321de9d72360..b4d4be3cc987 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -322,7 +322,7 @@ struct damon_callback { int (*before_start)(struct damon_ctx *context); int (*after_sampling)(struct damon_ctx *context); int (*after_aggregation)(struct damon_ctx *context); - int (*before_terminate)(struct damon_ctx *context); + void (*before_terminate)(struct damon_ctx *context); }; /** diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index befb27a29aab..eccc14b34901 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -645,18 +645,17 @@ static void dbgfs_fill_ctx_dir(struct dentry *dir, struct damon_ctx *ctx) debugfs_create_file(file_names[i], 0600, dir, ctx, fops[i]); } -static int dbgfs_before_terminate(struct damon_ctx *ctx) +static void dbgfs_before_terminate(struct damon_ctx *ctx) { struct damon_target *t, *next; if (!targetid_is_pid(ctx)) - return 0; + return; damon_for_each_target_safe(t, next, ctx) { put_pid((struct pid *)t->id); damon_destroy_target(t); } - return 0; } static struct damon_ctx *dbgfs_new_ctx(void) From dbbff9155c2f7726c4e79516c834a0818634d1ab Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 19 Nov 2021 16:43:49 -0800 Subject: [PATCH 097/156] UPSTREAM: mm/damon/dbgfs: use '__GFP_NOWARN' for user-specified size buffer allocation Patch series "DAMON fixes". This patch (of 2): DAMON users can trigger below warning in '__alloc_pages()' by invoking write() to some DAMON debugfs files with arbitrarily high count argument, because DAMON debugfs interface allocates some buffers based on the user-specified 'count'. if (unlikely(order >= MAX_ORDER)) { WARN_ON_ONCE(!(gfp & __GFP_NOWARN)); return NULL; } Because the DAMON debugfs interface code checks failure of the 'kmalloc()', this commit simply suppresses the warnings by adding '__GFP_NOWARN' flag. Link: https://lkml.kernel.org/r/20211110145758.16558-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211110145758.16558-2-sj@kernel.org Fixes: 4bc05954d007 ("mm/damon: implement a debugfs-based user space interface") Signed-off-by: SeongJae Park Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit db7a347b26fe05d2e8c115bb24dfd908d0252bc3) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ie04cc634d998260e5791b4daa4215b83a6af9071 --- mm/damon/dbgfs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index eccc14b34901..8ce1311ac533 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -32,7 +32,7 @@ static char *user_input_str(const char __user *buf, size_t count, loff_t *ppos) if (*ppos) return ERR_PTR(-EINVAL); - kbuf = kmalloc(count + 1, GFP_KERNEL); + kbuf = kmalloc(count + 1, GFP_KERNEL | __GFP_NOWARN); if (!kbuf) return ERR_PTR(-ENOMEM); @@ -133,7 +133,7 @@ static ssize_t dbgfs_schemes_read(struct file *file, char __user *buf, char *kbuf; ssize_t len; - kbuf = kmalloc(count, GFP_KERNEL); + kbuf = kmalloc(count, GFP_KERNEL | __GFP_NOWARN); if (!kbuf) return -ENOMEM; @@ -452,7 +452,7 @@ static ssize_t dbgfs_init_regions_read(struct file *file, char __user *buf, char *kbuf; ssize_t len; - kbuf = kmalloc(count, GFP_KERNEL); + kbuf = kmalloc(count, GFP_KERNEL | __GFP_NOWARN); if (!kbuf) return -ENOMEM; @@ -578,7 +578,7 @@ static ssize_t dbgfs_kdamond_pid_read(struct file *file, char *kbuf; ssize_t len; - kbuf = kmalloc(count, GFP_KERNEL); + kbuf = kmalloc(count, GFP_KERNEL | __GFP_NOWARN); if (!kbuf) return -ENOMEM; From 25814648677db065fcdcf60628f86ee52699c089 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 19 Nov 2021 16:43:52 -0800 Subject: [PATCH 098/156] UPSTREAM: mm/damon/dbgfs: fix missed use of damon_dbgfs_lock DAMON debugfs is supposed to protect dbgfs_ctxs, dbgfs_nr_ctxs, and dbgfs_dirs using damon_dbgfs_lock. However, some of the code is accessing the variables without the protection. This fixes it by protecting all such accesses. Link: https://lkml.kernel.org/r/20211110145758.16558-3-sj@kernel.org Fixes: 75c1c2b53c78 ("mm/damon/dbgfs: support multiple contexts") Signed-off-by: SeongJae Park Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit d78f3853f831eee46c6dbe726debf3be9e9c0d05) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ibdfcc75f9f219b69a4b4e6161af257c1d0eba891 --- mm/damon/dbgfs.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 8ce1311ac533..9b520bb4a3e7 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -877,12 +877,14 @@ static ssize_t dbgfs_monitor_on_write(struct file *file, return -EINVAL; } + mutex_lock(&damon_dbgfs_lock); if (!strncmp(kbuf, "on", count)) { int i; for (i = 0; i < dbgfs_nr_ctxs; i++) { if (damon_targets_empty(dbgfs_ctxs[i])) { kfree(kbuf); + mutex_unlock(&damon_dbgfs_lock); return -EINVAL; } } @@ -892,6 +894,7 @@ static ssize_t dbgfs_monitor_on_write(struct file *file, } else { ret = -EINVAL; } + mutex_unlock(&damon_dbgfs_lock); if (!ret) ret = count; @@ -944,15 +947,16 @@ static int __init __damon_dbgfs_init(void) static int __init damon_dbgfs_init(void) { - int rc; + int rc = -ENOMEM; + mutex_lock(&damon_dbgfs_lock); dbgfs_ctxs = kmalloc(sizeof(*dbgfs_ctxs), GFP_KERNEL); if (!dbgfs_ctxs) - return -ENOMEM; + goto out; dbgfs_ctxs[0] = dbgfs_new_ctx(); if (!dbgfs_ctxs[0]) { kfree(dbgfs_ctxs); - return -ENOMEM; + goto out; } dbgfs_nr_ctxs = 1; @@ -963,6 +967,8 @@ static int __init damon_dbgfs_init(void) pr_err("%s: dbgfs init failed\n", __func__); } +out: + mutex_unlock(&damon_dbgfs_lock); return rc; } From a9ec7ed936e9341de5d462c3630fec5c9c0f714b Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 10 Dec 2021 14:46:22 -0800 Subject: [PATCH 099/156] BACKPORT: timers: implement usleep_idle_range() Patch series "mm/damon: Fix fake /proc/loadavg reports", v3. This patchset fixes DAMON's fake load report issue. The first patch makes yet another variant of usleep_range() for this fix, and the second patch fixes the issue of DAMON by making it using the newly introduced function. This patch (of 2): Some kernel threads such as DAMON could need to repeatedly sleep in micro seconds level. Because usleep_range() sleeps in uninterruptible state, however, such threads would make /proc/loadavg reports fake load. To help such cases, this commit implements a variant of usleep_range() called usleep_idle_range(). It is same to usleep_range() but sets the state of the current task as TASK_IDLE while sleeping. Link: https://lkml.kernel.org/r/20211126145015.15862-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211126145015.15862-2-sj@kernel.org Signed-off-by: SeongJae Park Suggested-by: Andrew Morton Reviewed-by: Thomas Gleixner Tested-by: Oleksandr Natalenko Cc: John Stultz Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit e4779015fd5d2fb8390c258268addff24d6077c7) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ie590ba5fcff22c981d0a7ecae6d8e551160136f3 --- include/linux/delay.h | 8 ++++++++ kernel/time/timer.c | 36 +++++++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/include/linux/delay.h b/include/linux/delay.h index 1d0e2ce6b6d9..abecbccab6e4 100644 --- a/include/linux/delay.h +++ b/include/linux/delay.h @@ -20,6 +20,7 @@ */ #include +#include extern unsigned long loops_per_jiffy; @@ -58,8 +59,15 @@ void calibrate_delay(void); void __attribute__((weak)) calibration_delay_done(void); void msleep(unsigned int msecs); unsigned long msleep_interruptible(unsigned int msecs); +void usleep_range_state(unsigned long min, unsigned long max, + unsigned int state); void usleep_range(unsigned long min, unsigned long max); +static inline void usleep_idle_range(unsigned long min, unsigned long max) +{ + usleep_range_state(min, max, TASK_IDLE); +} + static inline void ssleep(unsigned int seconds) { msleep(seconds * 1000); diff --git a/kernel/time/timer.c b/kernel/time/timer.c index ed71c4141047..1535065a7d76 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -2052,6 +2052,32 @@ unsigned long msleep_interruptible(unsigned int msecs) EXPORT_SYMBOL(msleep_interruptible); +/** + * usleep_range_state - Sleep for an approximate time in a given state + * @min: Minimum time in usecs to sleep + * @max: Maximum time in usecs to sleep + * @state: State of the current task that will be while sleeping + * + * In non-atomic context where the exact wakeup time is flexible, use + * usleep_range_state() instead of udelay(). The sleep improves responsiveness + * by avoiding the CPU-hogging busy-wait of udelay(), and the range reduces + * power usage by allowing hrtimers to take advantage of an already- + * scheduled interrupt instead of scheduling a new one just for this sleep. + */ +void __sched usleep_range_state(unsigned long min, unsigned long max, + unsigned int state) +{ + ktime_t exp = ktime_add_us(ktime_get(), min); + u64 delta = (u64)(max - min) * NSEC_PER_USEC; + + for (;;) { + __set_current_state(state); + /* Do not return before the requested sleep time has elapsed */ + if (!schedule_hrtimeout_range(&exp, delta, HRTIMER_MODE_ABS)) + break; + } +} + /** * usleep_range - Sleep for an approximate time * @min: Minimum time in usecs to sleep @@ -2065,14 +2091,6 @@ EXPORT_SYMBOL(msleep_interruptible); */ void __sched usleep_range(unsigned long min, unsigned long max) { - ktime_t exp = ktime_add_us(ktime_get(), min); - u64 delta = (u64)(max - min) * NSEC_PER_USEC; - - for (;;) { - __set_current_state(TASK_UNINTERRUPTIBLE); - /* Do not return before the requested sleep time has elapsed */ - if (!schedule_hrtimeout_range(&exp, delta, HRTIMER_MODE_ABS)) - break; - } + usleep_range_state(min, max, TASK_UNINTERRUPTIBLE); } EXPORT_SYMBOL(usleep_range); From 63e8bc85e6c97888b99d4ff687f374a78309b776 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 10 Dec 2021 14:46:25 -0800 Subject: [PATCH 100/156] UPSTREAM: mm/damon/core: fix fake load reports due to uninterruptible sleeps Because DAMON sleeps in uninterruptible mode, /proc/loadavg reports fake load while DAMON is turned on, though it is doing nothing. This can confuse users[1]. To avoid the case, this commit makes DAMON sleeps in idle mode. [1] https://lore.kernel.org/all/11868371.O9o76ZdvQC@natalenko.name/ Link: https://lkml.kernel.org/r/20211126145015.15862-3-sj@kernel.org Fixes: 2224d8485492 ("mm: introduce Data Access MONitor (DAMON)") Reported-by: Oleksandr Natalenko Signed-off-by: SeongJae Park Tested-by: Oleksandr Natalenko Cc: John Stultz Cc: Thomas Gleixner Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 70e9274805fccfd175d0431a947bfd11ee7df40e) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Id3518e85d9b1db9f71b62932418993b4b9666326 --- mm/damon/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index c381b3c525d0..2daffd5820fe 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -981,9 +981,9 @@ static unsigned long damos_wmark_wait_us(struct damos *scheme) static void kdamond_usleep(unsigned long usecs) { if (usecs > 100 * 1000) - schedule_timeout_interruptible(usecs_to_jiffies(usecs)); + schedule_timeout_idle(usecs_to_jiffies(usecs)); else - usleep_range(usecs, usecs + 1); + usleep_idle_range(usecs, usecs + 1); } /* Returns negative error code if it's not activated but should return */ @@ -1038,7 +1038,7 @@ static int kdamond_fn(void *data) ctx->callback.after_sampling(ctx)) done = true; - usleep_range(ctx->sample_interval, ctx->sample_interval + 1); + kdamond_usleep(ctx->sample_interval); if (ctx->primitive.check_accesses) max_nr_accesses = ctx->primitive.check_accesses(ctx); From f37ab7f595356e673409201be6d80e104e2f1b21 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 10 Dec 2021 14:46:28 -0800 Subject: [PATCH 101/156] UPSTREAM: mm/damon/core: use better timer mechanisms selection threshold Patch series "mm/damon: Trivial fixups and improvements". This patchset contains trivial fixups and improvements for DAMON and its kunit/kselftest tests. This patch (of 11): DAMON is using hrtimer if requested sleep time is <=100ms, while the suggested threshold[1] is <=20ms. This commit applies the threshold. [1] Documentation/timers/timers-howto.rst Link: https://lkml.kernel.org/r/20211201150440.1088-2-sj@kernel.org Fixes: ee801b7dd7822 ("mm/damon/schemes: activate schemes based on a watermarks mechanism") Signed-off-by: SeongJae Park Cc: Shuah Khan Cc: Brendan Higgins Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 4de46a30b9929d3d1b29e481d48e9c25f8ac7919) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ieb5e3e196c96d8d54c83fecec53fdd13db763b62 --- mm/damon/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index 2daffd5820fe..eefb2ada67ca 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -980,7 +980,8 @@ static unsigned long damos_wmark_wait_us(struct damos *scheme) static void kdamond_usleep(unsigned long usecs) { - if (usecs > 100 * 1000) + /* See Documentation/timers/timers-howto.rst for the thresholds */ + if (usecs > 20 * USEC_PER_MSEC) schedule_timeout_idle(usecs_to_jiffies(usecs)); else usleep_idle_range(usecs, usecs + 1); From a7969dac5a2fd5783b7ac2a39a7b059cee71d4d9 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 10 Dec 2021 14:46:31 -0800 Subject: [PATCH 102/156] UPSTREAM: mm/damon/dbgfs: remove an unnecessary error message When wrong scheme action is requested via the debugfs interface, DAMON prints an error message. Because the function returns error code, this is not really needed. Because the code path is triggered by the user specified input, this can result in kernel log mistakenly being messy. To avoid the case, this commit removes the message. Link: https://lkml.kernel.org/r/20211201150440.1088-3-sj@kernel.org Fixes: af122dd8f3c0 ("mm/damon/dbgfs: support DAMON-based Operation Schemes") Signed-off-by: SeongJae Park Cc: Brendan Higgins Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 0bceffa236af401f5206feaf3538526cbc427209) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I6c6e4ea6891eee04cb934f237b7d1768a766f240 --- mm/damon/dbgfs.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 9b520bb4a3e7..1efac0022e9a 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -210,10 +210,8 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, &wmarks.low, &parsed); if (ret != 18) break; - if (!damos_action_valid(action)) { - pr_err("wrong action %d\n", action); + if (!damos_action_valid(action)) goto fail; - } pos += parsed; scheme = damon_new_scheme(min_sz, max_sz, min_nr_a, max_nr_a, From 4e19846848ed5a933b1df8e4ad6eeadecaa09970 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 10 Dec 2021 14:46:34 -0800 Subject: [PATCH 103/156] UPSTREAM: mm/damon/core: remove unnecessary error messages DAMON core prints error messages when damon_target object creation is failed or wrong monitoring attributes are given. Because appropriate error code is returned for each case, the messages are not essential. Also, because the code path can be triggered with user-specified input, this could result in kernel log mistakenly being messy. To avoid the case, this commit removes the messages. Link: https://lkml.kernel.org/r/20211201150440.1088-4-sj@kernel.org Fixes: 4bc05954d007 ("mm/damon: implement a debugfs-based user space interface") Fixes: b9a6ac4e4ede ("mm/damon: adaptively adjust regions") Signed-off-by: SeongJae Park Cc: Brendan Higgins Cc: kernel test robot Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 1afaf5cb687de85c5e00ac70f6eea5597077cbc5) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I571336c5d49998029d4727b523b7a0950c753b2f --- mm/damon/core.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index eefb2ada67ca..e92497895202 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -282,7 +282,6 @@ int damon_set_targets(struct damon_ctx *ctx, for (i = 0; i < nr_ids; i++) { t = damon_new_target(ids[i]); if (!t) { - pr_err("Failed to alloc damon_target\n"); /* The caller should do cleanup of the ids itself */ damon_for_each_target_safe(t, next, ctx) damon_destroy_target(t); @@ -312,16 +311,10 @@ int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, unsigned long aggr_int, unsigned long primitive_upd_int, unsigned long min_nr_reg, unsigned long max_nr_reg) { - if (min_nr_reg < 3) { - pr_err("min_nr_regions (%lu) must be at least 3\n", - min_nr_reg); + if (min_nr_reg < 3) return -EINVAL; - } - if (min_nr_reg > max_nr_reg) { - pr_err("invalid nr_regions. min (%lu) > max (%lu)\n", - min_nr_reg, max_nr_reg); + if (min_nr_reg > max_nr_reg) return -EINVAL; - } ctx->sample_interval = sample_int; ctx->aggr_interval = aggr_int; From 90af7e344b9b44b775d3c2ab0f1a28746471c1ca Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 10 Dec 2021 14:46:37 -0800 Subject: [PATCH 104/156] UPSTREAM: mm/damon/vaddr: remove an unnecessary warning message The DAMON virtual address space monitoring primitive prints a warning message for wrong DAMOS action. However, it is not essential as the code returns appropriate failure in the case. This commit removes the message to make the log clean. Link: https://lkml.kernel.org/r/20211201150440.1088-5-sj@kernel.org Fixes: 6dea8add4d28 ("mm/damon/vaddr: support DAMON-based Operation Schemes") Signed-off-by: SeongJae Park Reviewed-by: Muchun Song Cc: Brendan Higgins Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 09e12289cc044afa484e70c0b379d579d52caf9a) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I695d3e813f9449b388f2757c6e81cba76ee3e1bf --- mm/damon/vaddr.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 35fe49080ee9..5609d0ad4afd 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -626,7 +626,6 @@ int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, case DAMOS_STAT: return 0; default: - pr_warn("Wrong action %d\n", scheme->action); return -EINVAL; } From 4f0e48e5e94136923c0ff8288ae25f98170fd414 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 10 Dec 2021 14:46:40 -0800 Subject: [PATCH 105/156] UPSTREAM: mm/damon/vaddr-test: split a test function having >1024 bytes frame size On some configuration[1], 'damon_test_split_evenly()' kunit test function has >1024 bytes frame size, so below build warning is triggered: CC mm/damon/vaddr.o In file included from mm/damon/vaddr.c:672: mm/damon/vaddr-test.h: In function 'damon_test_split_evenly': mm/damon/vaddr-test.h:309:1: warning: the frame size of 1064 bytes is larger than 1024 bytes [-Wframe-larger-than=] 309 | } | ^ This commit fixes the warning by separating the common logic in the function. [1] https://lore.kernel.org/linux-mm/202111182146.OV3C4uGr-lkp@intel.com/ Link: https://lkml.kernel.org/r/20211201150440.1088-6-sj@kernel.org Fixes: 17ccae8bb5c9 ("mm/damon: add kunit tests") Signed-off-by: SeongJae Park Reported-by: kernel test robot Cc: Brendan Higgins Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 044cd9750fe010170f5dc812e4824d98f5ea928c) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I78226fdde7f992f5aa8209f0996561a3f7efce84 --- mm/damon/vaddr-test.h | 93 ++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/mm/damon/vaddr-test.h b/mm/damon/vaddr-test.h index ecfd0b2ed222..3097ef9c662a 100644 --- a/mm/damon/vaddr-test.h +++ b/mm/damon/vaddr-test.h @@ -252,59 +252,62 @@ static void damon_test_apply_three_regions4(struct kunit *test) new_three_regions, expected, ARRAY_SIZE(expected)); } +static void damon_test_split_evenly_fail(struct kunit *test, + unsigned long start, unsigned long end, unsigned int nr_pieces) +{ + struct damon_target *t = damon_new_target(42); + struct damon_region *r = damon_new_region(start, end); + + damon_add_region(r, t); + KUNIT_EXPECT_EQ(test, + damon_va_evenly_split_region(t, r, nr_pieces), -EINVAL); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1u); + + damon_for_each_region(r, t) { + KUNIT_EXPECT_EQ(test, r->ar.start, start); + KUNIT_EXPECT_EQ(test, r->ar.end, end); + } + + damon_free_target(t); +} + +static void damon_test_split_evenly_succ(struct kunit *test, + unsigned long start, unsigned long end, unsigned int nr_pieces) +{ + struct damon_target *t = damon_new_target(42); + struct damon_region *r = damon_new_region(start, end); + unsigned long expected_width = (end - start) / nr_pieces; + unsigned long i = 0; + + damon_add_region(r, t); + KUNIT_EXPECT_EQ(test, + damon_va_evenly_split_region(t, r, nr_pieces), 0); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), nr_pieces); + + damon_for_each_region(r, t) { + if (i == nr_pieces - 1) + break; + KUNIT_EXPECT_EQ(test, + r->ar.start, start + i++ * expected_width); + KUNIT_EXPECT_EQ(test, r->ar.end, start + i * expected_width); + } + KUNIT_EXPECT_EQ(test, r->ar.start, start + i * expected_width); + KUNIT_EXPECT_EQ(test, r->ar.end, end); + damon_free_target(t); +} + static void damon_test_split_evenly(struct kunit *test) { struct damon_ctx *c = damon_new_ctx(); - struct damon_target *t; - struct damon_region *r; - unsigned long i; KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(NULL, NULL, 5), -EINVAL); - t = damon_new_target(42); - r = damon_new_region(0, 100); - KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 0), -EINVAL); + damon_test_split_evenly_fail(test, 0, 100, 0); + damon_test_split_evenly_succ(test, 0, 100, 10); + damon_test_split_evenly_succ(test, 5, 59, 5); + damon_test_split_evenly_fail(test, 5, 6, 2); - damon_add_region(r, t); - KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 10), 0); - KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 10u); - - i = 0; - damon_for_each_region(r, t) { - KUNIT_EXPECT_EQ(test, r->ar.start, i++ * 10); - KUNIT_EXPECT_EQ(test, r->ar.end, i * 10); - } - damon_free_target(t); - - t = damon_new_target(42); - r = damon_new_region(5, 59); - damon_add_region(r, t); - KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 5), 0); - KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 5u); - - i = 0; - damon_for_each_region(r, t) { - if (i == 4) - break; - KUNIT_EXPECT_EQ(test, r->ar.start, 5 + 10 * i++); - KUNIT_EXPECT_EQ(test, r->ar.end, 5 + 10 * i); - } - KUNIT_EXPECT_EQ(test, r->ar.start, 5 + 10 * i); - KUNIT_EXPECT_EQ(test, r->ar.end, 59ul); - damon_free_target(t); - - t = damon_new_target(42); - r = damon_new_region(5, 6); - damon_add_region(r, t); - KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 2), -EINVAL); - KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1u); - - damon_for_each_region(r, t) { - KUNIT_EXPECT_EQ(test, r->ar.start, 5ul); - KUNIT_EXPECT_EQ(test, r->ar.end, 6ul); - } - damon_free_target(t); damon_destroy_ctx(c); } From 82bb332bf08bb228d62a75ba062abe2bc9275b0f Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 10 Dec 2021 14:46:43 -0800 Subject: [PATCH 106/156] UPSTREAM: mm/damon/vaddr-test: remove unnecessary variables A couple of test functions in DAMON virtual address space monitoring primitives implementation has unnecessary damon_ctx variables. This commit removes those. Link: https://lkml.kernel.org/r/20211201150440.1088-7-sj@kernel.org Signed-off-by: SeongJae Park Cc: Brendan Higgins Cc: Shuah Khan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 9f86d624292c238203b3687cdb870a2cde1a6f9b) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ic7543d67d15843d7c2110302de950958a3b46e04 --- mm/damon/vaddr-test.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mm/damon/vaddr-test.h b/mm/damon/vaddr-test.h index 3097ef9c662a..6a1b9272ea12 100644 --- a/mm/damon/vaddr-test.h +++ b/mm/damon/vaddr-test.h @@ -135,7 +135,6 @@ static void damon_do_test_apply_three_regions(struct kunit *test, struct damon_addr_range *three_regions, unsigned long *expected, int nr_expected) { - struct damon_ctx *ctx = damon_new_ctx(); struct damon_target *t; struct damon_region *r; int i; @@ -145,7 +144,6 @@ static void damon_do_test_apply_three_regions(struct kunit *test, r = damon_new_region(regions[i * 2], regions[i * 2 + 1]); damon_add_region(r, t); } - damon_add_target(ctx, t); damon_va_apply_three_regions(t, three_regions); @@ -154,8 +152,6 @@ static void damon_do_test_apply_three_regions(struct kunit *test, KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]); KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]); } - - damon_destroy_ctx(ctx); } /* @@ -298,8 +294,6 @@ static void damon_test_split_evenly_succ(struct kunit *test, static void damon_test_split_evenly(struct kunit *test) { - struct damon_ctx *c = damon_new_ctx(); - KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(NULL, NULL, 5), -EINVAL); @@ -307,8 +301,6 @@ static void damon_test_split_evenly(struct kunit *test) damon_test_split_evenly_succ(test, 0, 100, 10); damon_test_split_evenly_succ(test, 5, 59, 5); damon_test_split_evenly_fail(test, 5, 6, 2); - - damon_destroy_ctx(c); } static struct kunit_case damon_test_cases[] = { From c3939031fb31fc2a825346ffae1e8e908c44dae5 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 24 Dec 2021 21:12:54 -0800 Subject: [PATCH 107/156] UPSTREAM: mm/damon/dbgfs: protect targets destructions with kdamond_lock DAMON debugfs interface iterates current monitoring targets in 'dbgfs_target_ids_read()' while holding the corresponding 'kdamond_lock'. However, it also destructs the monitoring targets in 'dbgfs_before_terminate()' without holding the lock. This can result in a use_after_free bug. This commit avoids the race by protecting the destruction with the corresponding 'kdamond_lock'. Link: https://lkml.kernel.org/r/20211221094447.2241-1-sj@kernel.org Reported-by: Sangwoo Bae Fixes: 4bc05954d007 ("mm/damon: implement a debugfs-based user space interface") Signed-off-by: SeongJae Park Cc: [5.15.x] Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 34796417964b8d0aef45a99cf6c2d20cebe33733) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I27e10c0e1ccc6c28c5a948a246e45db7a0338bed --- mm/damon/dbgfs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 1efac0022e9a..4fbd729edc9e 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -650,10 +650,12 @@ static void dbgfs_before_terminate(struct damon_ctx *ctx) if (!targetid_is_pid(ctx)) return; + mutex_lock(&ctx->kdamond_lock); damon_for_each_target_safe(t, next, ctx) { put_pid((struct pid *)t->id); damon_destroy_target(t); } + mutex_unlock(&ctx->kdamond_lock); } static struct damon_ctx *dbgfs_new_ctx(void) From d4d20c7ef57893c4b168720fa77d2cbd38be9fd5 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Thu, 30 Dec 2021 20:12:34 -0800 Subject: [PATCH 108/156] UPSTREAM: mm/damon/dbgfs: fix 'struct pid' leaks in 'dbgfs_target_ids_write()' DAMON debugfs interface increases the reference counts of 'struct pid's for targets from the 'target_ids' file write callback ('dbgfs_target_ids_write()'), but decreases the counts only in DAMON monitoring termination callback ('dbgfs_before_terminate()'). Therefore, when 'target_ids' file is repeatedly written without DAMON monitoring start/termination, the reference count is not decreased and therefore memory for the 'struct pid' cannot be freed. This commit fixes this issue by decreasing the reference counts when 'target_ids' is written. Link: https://lkml.kernel.org/r/20211229124029.23348-1-sj@kernel.org Fixes: 4bc05954d007 ("mm/damon: implement a debugfs-based user space interface") Signed-off-by: SeongJae Park Cc: [5.15+] Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit ebb3f994dd92f8fb4d70c7541091216c1e10cb71) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I1d280e32acf9478f48c6946469da1444a2998464 --- mm/damon/dbgfs.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 4fbd729edc9e..ad65436756af 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -353,6 +353,7 @@ static ssize_t dbgfs_target_ids_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { struct damon_ctx *ctx = file->private_data; + struct damon_target *t, *next_t; bool id_is_pid = true; char *kbuf, *nrs; unsigned long *targets; @@ -397,8 +398,12 @@ static ssize_t dbgfs_target_ids_write(struct file *file, goto unlock_out; } - /* remove targets with previously-set primitive */ - damon_set_targets(ctx, NULL, 0); + /* remove previously set targets */ + damon_for_each_target_safe(t, next_t, ctx) { + if (targetid_is_pid(ctx)) + put_pid((struct pid *)t->id); + damon_destroy_target(t); + } /* Configure the context for the address space type */ if (id_is_pid) From 8b02bed7596466b395142df770c4b7b64f835327 Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 14 Jan 2022 14:09:34 -0800 Subject: [PATCH 109/156] UPSTREAM: mm/damon: unified access_check function naming rules Patch series "mm/damon: Do some small changes", v4. This patch (of 4): In damon/paddr.c file, two functions names start with underscore, static void __damon_pa_prepare_access_check(struct damon_ctx *ctx, struct damon_region *r) static void __damon_pa_prepare_access_check(struct damon_ctx *ctx, struct damon_region *r) In damon/vaddr.c file, there are also two functions with the same function, static void damon_va_prepare_access_check(struct damon_ctx *ctx, struct mm_struct *mm, struct damon_region *r) static void damon_va_check_access(struct damon_ctx *ctx, struct mm_struct *mm, struct damon_region *r) It makes sense to keep consistent, and it is not easy to be confused with the function that call them. Link: https://lkml.kernel.org/r/cover.1636989871.git.xhao@linux.alibaba.com Link: https://lkml.kernel.org/r/529054aed932a42b9c09fc9977ad4574b9e7b0bd.1636989871.git.xhao@linux.alibaba.com Signed-off-by: Xin Hao Reviewed-by: SeongJae Park Cc: Muchun Song Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit b627b774911660852ce7f3f3817955ddad2bd130) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I180e7f068ed4745c5fbc0e2a81320b7fda03c52e --- mm/damon/vaddr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 5609d0ad4afd..fa1c6d1da4be 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -409,7 +409,7 @@ static void damon_va_mkold(struct mm_struct *mm, unsigned long addr) * Functions for the access checking of the regions */ -static void damon_va_prepare_access_check(struct damon_ctx *ctx, +static void __damon_va_prepare_access_check(struct damon_ctx *ctx, struct mm_struct *mm, struct damon_region *r) { r->sampling_addr = damon_rand(r->ar.start, r->ar.end); @@ -428,7 +428,7 @@ void damon_va_prepare_access_checks(struct damon_ctx *ctx) if (!mm) continue; damon_for_each_region(r, t) - damon_va_prepare_access_check(ctx, mm, r); + __damon_va_prepare_access_check(ctx, mm, r); mmput(mm); } } @@ -514,7 +514,7 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr, * mm 'mm_struct' for the given virtual address space * r the region to be checked */ -static void damon_va_check_access(struct damon_ctx *ctx, +static void __damon_va_check_access(struct damon_ctx *ctx, struct mm_struct *mm, struct damon_region *r) { static struct mm_struct *last_mm; @@ -550,7 +550,7 @@ unsigned int damon_va_check_accesses(struct damon_ctx *ctx) if (!mm) continue; damon_for_each_region(r, t) { - damon_va_check_access(ctx, mm, r); + __damon_va_check_access(ctx, mm, r); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); } mmput(mm); From 07045a0e5ac5a734dc992201cec0a4cc048ee94f Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 14 Jan 2022 14:09:40 -0800 Subject: [PATCH 110/156] UPSTREAM: mm/damon/core: use abs() instead of diff_of() In kernel, we can use abs(a - b) to get the absolute value, So there is no need to redefine a new one. Link: https://lkml.kernel.org/r/b24e7b82d9efa90daf150d62dea171e19390ad0b.1636989871.git.xhao@linux.alibaba.com Signed-off-by: Xin Hao Reviewed-by: Muchun Song Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit d720bbbd70e968f8a0257393b575c3a29b56f990) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Iab418bdc79cea24a175c2c5e17ab7ee393df6c47 --- mm/damon/core.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index e92497895202..04b8df7fd9e9 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -750,8 +750,6 @@ static void damon_merge_two_regions(struct damon_target *t, damon_destroy_region(r, t); } -#define diff_of(a, b) (a > b ? a - b : b - a) - /* * Merge adjacent regions having similar access frequencies * @@ -765,13 +763,13 @@ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, struct damon_region *r, *prev = NULL, *next; damon_for_each_region_safe(r, next, t) { - if (diff_of(r->nr_accesses, r->last_nr_accesses) > thres) + if (abs(r->nr_accesses - r->last_nr_accesses) > thres) r->age = 0; else r->age++; if (prev && prev->ar.end == r->ar.start && - diff_of(prev->nr_accesses, r->nr_accesses) <= thres && + abs(prev->nr_accesses - r->nr_accesses) <= thres && sz_damon_region(prev) + sz_damon_region(r) <= sz_limit) damon_merge_two_regions(t, prev, r); else From 9a8de9c70223c0670e03a26ebb23a4eae3a42ad2 Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 14 Jan 2022 14:09:44 -0800 Subject: [PATCH 111/156] UPSTREAM: mm/damon: remove some unneeded function definitions in damon.h In damon.h some func definitions about VA & PA can only be used in its own file, so there no need to define in the header file, and the header file will look cleaner. If other files later need these functions, the prototypes can be added to damon.h at that time. [sj@kernel.org: remove unnecessary function prototype position changes] Link: https://lkml.kernel.org/r/20211118114827.20052-1-sj@kernel.org Link: https://lkml.kernel.org/r/45fd5b3ef6cce8e28dbc1c92f9dc845ccfc949d7.1636989871.git.xhao@linux.alibaba.com Signed-off-by: Xin Hao Signed-off-by: SeongJae Park Reviewed-by: SeongJae Park Cc: Muchun Song Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit cdeed009f3bceee41f73f0137db785fd29a05cb8) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I2203b9c8e4625493797e1f3e506431799c4404c9 --- include/linux/damon.h | 21 --------------------- mm/damon/paddr.c | 11 ++++++----- mm/damon/vaddr.c | 18 ++++++++++-------- 3 files changed, 16 insertions(+), 34 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index b4d4be3cc987..1d1be348f506 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -461,34 +461,13 @@ int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); #endif /* CONFIG_DAMON */ #ifdef CONFIG_DAMON_VADDR - -/* Monitoring primitives for virtual memory address spaces */ -void damon_va_init(struct damon_ctx *ctx); -void damon_va_update(struct damon_ctx *ctx); -void damon_va_prepare_access_checks(struct damon_ctx *ctx); -unsigned int damon_va_check_accesses(struct damon_ctx *ctx); bool damon_va_target_valid(void *t); -void damon_va_cleanup(struct damon_ctx *ctx); -int damon_va_apply_scheme(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme); -int damon_va_scheme_score(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme); void damon_va_set_primitives(struct damon_ctx *ctx); - #endif /* CONFIG_DAMON_VADDR */ #ifdef CONFIG_DAMON_PADDR - -/* Monitoring primitives for the physical memory address space */ -void damon_pa_prepare_access_checks(struct damon_ctx *ctx); -unsigned int damon_pa_check_accesses(struct damon_ctx *ctx); bool damon_pa_target_valid(void *t); -int damon_pa_apply_scheme(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme); -int damon_pa_scheme_score(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme); void damon_pa_set_primitives(struct damon_ctx *ctx); - #endif /* CONFIG_DAMON_PADDR */ #endif /* _DAMON_H */ diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index a496d6f203d6..4318134cbc4c 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -73,7 +73,7 @@ static void __damon_pa_prepare_access_check(struct damon_ctx *ctx, damon_pa_mkold(r->sampling_addr); } -void damon_pa_prepare_access_checks(struct damon_ctx *ctx) +static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) { struct damon_target *t; struct damon_region *r; @@ -192,7 +192,7 @@ static void __damon_pa_check_access(struct damon_ctx *ctx, last_addr = r->sampling_addr; } -unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) +static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) { struct damon_target *t; struct damon_region *r; @@ -213,7 +213,7 @@ bool damon_pa_target_valid(void *t) return true; } -int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, +static int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, struct damos *scheme) { unsigned long addr; @@ -246,8 +246,9 @@ int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, return 0; } -int damon_pa_scheme_score(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme) +static int damon_pa_scheme_score(struct damon_ctx *context, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) { switch (scheme->action) { case DAMOS_PAGEOUT: diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index fa1c6d1da4be..c55057264e98 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -271,7 +271,7 @@ static void __damon_va_init_regions(struct damon_ctx *ctx, } /* Initialize '->regions_list' of every target (task) */ -void damon_va_init(struct damon_ctx *ctx) +static void damon_va_init(struct damon_ctx *ctx) { struct damon_target *t; @@ -291,7 +291,8 @@ void damon_va_init(struct damon_ctx *ctx) * * Returns true if it is. */ -static bool damon_intersect(struct damon_region *r, struct damon_addr_range *re) +static bool damon_intersect(struct damon_region *r, + struct damon_addr_range *re) { return !(r->ar.end <= re->start || re->end <= r->ar.start); } @@ -355,7 +356,7 @@ static void damon_va_apply_three_regions(struct damon_target *t, /* * Update regions for current memory mappings */ -void damon_va_update(struct damon_ctx *ctx) +static void damon_va_update(struct damon_ctx *ctx) { struct damon_addr_range three_regions[3]; struct damon_target *t; @@ -417,7 +418,7 @@ static void __damon_va_prepare_access_check(struct damon_ctx *ctx, damon_va_mkold(mm, r->sampling_addr); } -void damon_va_prepare_access_checks(struct damon_ctx *ctx) +static void damon_va_prepare_access_checks(struct damon_ctx *ctx) { struct damon_target *t; struct mm_struct *mm; @@ -538,7 +539,7 @@ static void __damon_va_check_access(struct damon_ctx *ctx, last_addr = r->sampling_addr; } -unsigned int damon_va_check_accesses(struct damon_ctx *ctx) +static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) { struct damon_target *t; struct mm_struct *mm; @@ -602,7 +603,7 @@ out: } #endif /* CONFIG_ADVISE_SYSCALLS */ -int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, +static int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, struct damon_region *r, struct damos *scheme) { int madv_action; @@ -632,8 +633,9 @@ int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, return damos_madvise(t, r, madv_action); } -int damon_va_scheme_score(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme) +static int damon_va_scheme_score(struct damon_ctx *context, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) { switch (scheme->action) { From b198e86d5a02b439dc05f943b72e9a8edcd2ba14 Mon Sep 17 00:00:00 2001 From: Yihao Han Date: Fri, 14 Jan 2022 14:09:47 -0800 Subject: [PATCH 112/156] UPSTREAM: mm/damon/vaddr: remove swap_ranges() and replace it with swap() Remove 'swap_ranges()' and replace it with the macro 'swap()' defined in 'include/linux/minmax.h' to simplify code and improve efficiency Link: https://lkml.kernel.org/r/20211111115355.2808-1-hanyihao@vivo.com Signed-off-by: Yihao Han Reviewed-by: SeongJae Park Reviewed-by: Muchun Song Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 8bd0b9da03c9154e279b1a502636103887b9fbed) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Icf565b52a7642fb830ae004764ca496986aed82a --- mm/damon/vaddr.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index c55057264e98..a65b1a4d236c 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -97,16 +97,6 @@ static unsigned long sz_range(struct damon_addr_range *r) return r->end - r->start; } -static void swap_ranges(struct damon_addr_range *r1, - struct damon_addr_range *r2) -{ - struct damon_addr_range tmp; - - tmp = *r1; - *r1 = *r2; - *r2 = tmp; -} - /* * Find three regions separated by two biggest unmapped regions * @@ -145,9 +135,9 @@ static int __damon_va_three_regions(struct vm_area_struct *vma, gap.start = last_vma->vm_end; gap.end = vma->vm_start; if (sz_range(&gap) > sz_range(&second_gap)) { - swap_ranges(&gap, &second_gap); + swap(gap, second_gap); if (sz_range(&second_gap) > sz_range(&first_gap)) - swap_ranges(&second_gap, &first_gap); + swap(second_gap, first_gap); } next: last_vma = vma; @@ -158,7 +148,7 @@ next: /* Sort the two biggest gaps by address */ if (first_gap.start > second_gap.start) - swap_ranges(&first_gap, &second_gap); + swap(first_gap, second_gap); /* Store the result */ regions[0].start = ALIGN(start, DAMON_MIN_REGION); From b45423116e01bd346f978fe05ac82ffe1f9dd4de Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 14 Jan 2022 14:09:50 -0800 Subject: [PATCH 113/156] UPSTREAM: mm/damon/schemes: add the validity judgment of thresholds In dbgfs "schemes" interface, i do some test like this: # cd /sys/kernel/debug/damon # echo "2 1 2 1 10 1 3 10 1 1 1 1 1 1 1 1 2 3" > schemes # cat schemes # 2 1 2 1 10 1 3 10 1 1 1 1 1 1 1 1 2 3 0 0 There have some unreasonable places, i set the valules of these variables " , , " as "<2, 1>, <2, 1>, <10, 1>, <1, 2, 3>. So there add a validity judgment for these thresholds value. Link: https://lkml.kernel.org/r/d78360e52158d786fcbf20bc62c96785742e76d3.1637239568.git.xhao@linux.alibaba.com Signed-off-by: Xin Hao Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit c89ae63eb0662b6c9f82dbfad3ef010239b8c1b1) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I2b23124b95ebeac935f28f9632dccaeabace4533 --- mm/damon/dbgfs.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index ad65436756af..bf36a2756cfb 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -213,6 +213,13 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, if (!damos_action_valid(action)) goto fail; + if (min_sz > max_sz || min_nr_a > max_nr_a || min_age > max_age) + goto fail; + + if (wmarks.high < wmarks.mid || wmarks.high < wmarks.low || + wmarks.mid < wmarks.low) + goto fail; + pos += parsed; scheme = damon_new_scheme(min_sz, max_sz, min_nr_a, max_nr_a, min_age, max_age, action, "a, &wmarks); From 947d088b1fa105e7a6dad1646c4280f1ef7b6ef6 Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 14 Jan 2022 14:09:53 -0800 Subject: [PATCH 114/156] UPSTREAM: mm/damon: move damon_rand() definition into damon.h damon_rand() is called in three files:damon/core.c, damon/ paddr.c, damon/vaddr.c, i think there is no need to redefine this twice, So move it to damon.h will be a good choice. Link: https://lkml.kernel.org/r/20211202075859.51341-1-xhao@linux.alibaba.com Signed-off-by: Xin Hao Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 9b2a38d6ef25c1748e3964b0ff30a89e4ed26583) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ib7be3062385fac4b422faa86705968aa39095a72 --- include/linux/damon.h | 4 ++++ mm/damon/core.c | 4 ---- mm/damon/prmtv-common.h | 4 ---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 1d1be348f506..3e91a597a1aa 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -11,12 +11,16 @@ #include #include #include +#include /* Minimal region size. Every damon_region is aligned by this. */ #define DAMON_MIN_REGION PAGE_SIZE /* Max priority score for DAMON-based operation schemes */ #define DAMOS_MAX_SCORE (99) +/* Get a random number in [l, r) */ +#define damon_rand(l, r) (l + prandom_u32_max(r - l)) + /** * struct damon_addr_range - Represents an address region of [@start, @end). * @start: Start address of the region (inclusive). diff --git a/mm/damon/core.c b/mm/damon/core.c index 04b8df7fd9e9..61e844d15b13 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -23,9 +22,6 @@ #define DAMON_MIN_REGION 1 #endif -/* Get a random number in [l, r) */ -#define damon_rand(l, r) (l + prandom_u32_max(r - l)) - static DEFINE_MUTEX(damon_lock); static int nr_running_ctxs; diff --git a/mm/damon/prmtv-common.h b/mm/damon/prmtv-common.h index 61f27037603e..e790cb5f8fe0 100644 --- a/mm/damon/prmtv-common.h +++ b/mm/damon/prmtv-common.h @@ -6,10 +6,6 @@ */ #include -#include - -/* Get a random number in [l, r) */ -#define damon_rand(l, r) (l + prandom_u32_max(r - l)) struct page *damon_get_page(unsigned long pfn); From 943c0cd13fecdb3b83f44fe8116b79c228ae5514 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:09:59 -0800 Subject: [PATCH 115/156] UPSTREAM: mm/damon: convert macro functions to static inline functions Patch series "mm/damon: Misc cleanups". This patchset contains miscellaneous cleanups for DAMON's macro functions and documentation. This patch (of 6): This commit converts macro functions in DAMON to static inline functions, for better type checking, code documentation, etc[1]. [1] https://lore.kernel.org/linux-mm/20211202151213.6ec830863342220da4141bc5@linux-foundation.org/ Link: https://lkml.kernel.org/r/20211209131806.19317-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211209131806.19317-2-sj@kernel.org Signed-off-by: SeongJae Park Cc: Jonathan Corbet Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 88f86dcfa454784f7de550966c60fc78a3e95d6d) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Id1e4e08a844b49cf3572cb3ac02158f5c1909ea2 --- include/linux/damon.h | 18 ++++++++++++------ mm/damon/core.c | 5 ++++- mm/damon/vaddr.c | 6 ++++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 3e91a597a1aa..87d04e044a5d 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -396,14 +396,20 @@ struct damon_ctx { struct list_head schemes; }; -#define damon_next_region(r) \ - (container_of(r->list.next, struct damon_region, list)) +static inline struct damon_region *damon_next_region(struct damon_region *r) +{ + return container_of(r->list.next, struct damon_region, list); +} -#define damon_prev_region(r) \ - (container_of(r->list.prev, struct damon_region, list)) +static inline struct damon_region *damon_prev_region(struct damon_region *r) +{ + return container_of(r->list.prev, struct damon_region, list); +} -#define damon_last_region(t) \ - (list_last_entry(&t->regions_list, struct damon_region, list)) +static inline struct damon_region *damon_last_region(struct damon_target *t) +{ + return list_last_entry(&t->regions_list, struct damon_region, list); +} #define damon_for_each_region(r, t) \ list_for_each_entry(r, &t->regions_list, list) diff --git a/mm/damon/core.c b/mm/damon/core.c index 61e844d15b13..4515cf82c433 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -729,7 +729,10 @@ static void kdamond_apply_schemes(struct damon_ctx *c) } } -#define sz_damon_region(r) (r->ar.end - r->ar.start) +static inline unsigned long sz_damon_region(struct damon_region *r) +{ + return r->ar.end - r->ar.start; +} /* * Merge two adjacent regions into one region diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index a65b1a4d236c..981785c82f99 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -25,8 +25,10 @@ * 't->id' should be the pointer to the relevant 'struct pid' having reference * count. Caller must put the returned task, unless it is NULL. */ -#define damon_get_task_struct(t) \ - (get_pid_task((struct pid *)t->id, PIDTYPE_PID)) +static inline struct task_struct *damon_get_task_struct(struct damon_target *t) +{ + return get_pid_task((struct pid *)t->id, PIDTYPE_PID); +} /* * Get the mm_struct of the given target From 7cecfab158b950fb58a7fa12b07323605a0d8251 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:10:17 -0800 Subject: [PATCH 116/156] UPSTREAM: mm/damon/schemes: account scheme actions that successfully applied Patch series "mm/damon/schemes: Extend stats for better online analysis and tuning". To help online access pattern analysis and tuning of DAMON-based Operation Schemes (DAMOS), DAMOS provides simple statistics for each scheme. Introduction of DAMOS time/space quota further made the tuning easier by making the risk management easier. However, that also made understanding of the working schemes a little bit more difficult. For an example, progress of a given scheme can now be throttled by not only the aggressiveness of the target access pattern, but also the time/space quotas. So, when a scheme is showing unexpectedly slow progress, it's difficult to know by what the progress of the scheme is throttled, with currently provided statistics. This patchset extends the statistics to contain some metrics that can be helpful for such online schemes analysis and tuning (patches 1-2), exports those to users (patches 3 and 5), and add documents (patches 4 and 6). This patch (of 6): DAMON-based operation schemes (DAMOS) stats provide only the number and the amount of regions that the action of the scheme has tried to be applied. Because the action could be failed for some reasons, the currently provided information is sometimes not useful or convenient enough for schemes profiling and tuning. To improve this situation, this commit extends the DAMOS stats to provide the number and the amount of regions that the action has successfully applied. Link: https://lkml.kernel.org/r/20211210150016.35349-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211210150016.35349-2-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 0e92c2ee9f459542c5384d9cfab24873c3dd6398) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Iddfe9257cb99091404202576a1addc5cc340cb8b --- include/linux/damon.h | 28 +++++++++++++++++++++------- mm/damon/core.c | 13 ++++++++----- mm/damon/dbgfs.c | 2 +- mm/damon/paddr.c | 13 +++++++------ mm/damon/vaddr.c | 30 ++++++++++++++++-------------- 5 files changed, 53 insertions(+), 33 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 87d04e044a5d..59fcb5f89594 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -189,6 +189,20 @@ struct damos_watermarks { bool activated; }; +/** + * struct damos_stat - Statistics on a given scheme. + * @nr_tried: Total number of regions that the scheme is tried to be applied. + * @sz_tried: Total size of regions that the scheme is tried to be applied. + * @nr_applied: Total number of regions that the scheme is applied. + * @sz_applied: Total size of regions that the scheme is applied. + */ +struct damos_stat { + unsigned long nr_tried; + unsigned long sz_tried; + unsigned long nr_applied; + unsigned long sz_applied; +}; + /** * struct damos - Represents a Data Access Monitoring-based Operation Scheme. * @min_sz_region: Minimum size of target regions. @@ -200,8 +214,7 @@ struct damos_watermarks { * @action: &damo_action to be applied to the target regions. * @quota: Control the aggressiveness of this scheme. * @wmarks: Watermarks for automated (in)activation of this scheme. - * @stat_count: Total number of regions that this scheme is applied. - * @stat_sz: Total size of regions that this scheme is applied. + * @stat: Statistics of this scheme. * @list: List head for siblings. * * For each aggregation interval, DAMON finds regions which fit in the @@ -232,8 +245,7 @@ struct damos { enum damos_action action; struct damos_quota quota; struct damos_watermarks wmarks; - unsigned long stat_count; - unsigned long stat_sz; + struct damos_stat stat; struct list_head list; }; @@ -278,7 +290,8 @@ struct damon_ctx; * as an integer in [0, &DAMOS_MAX_SCORE]. * @apply_scheme is called from @kdamond when a region for user provided * DAMON-based operation scheme is found. It should apply the scheme's action - * to the region. This is not used for &DAMON_ARBITRARY_TARGET case. + * to the region and return bytes of the region that the action is successfully + * applied. * @target_valid should check whether the target is still valid for the * monitoring. * @cleanup is called from @kdamond just before its termination. @@ -292,8 +305,9 @@ struct damon_primitive { int (*get_scheme_score)(struct damon_ctx *context, struct damon_target *t, struct damon_region *r, struct damos *scheme); - int (*apply_scheme)(struct damon_ctx *context, struct damon_target *t, - struct damon_region *r, struct damos *scheme); + unsigned long (*apply_scheme)(struct damon_ctx *context, + struct damon_target *t, struct damon_region *r, + struct damos *scheme); bool (*target_valid)(void *target); void (*cleanup)(struct damon_ctx *context); }; diff --git a/mm/damon/core.c b/mm/damon/core.c index 4515cf82c433..d745bf28509f 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -102,8 +102,7 @@ struct damos *damon_new_scheme( scheme->min_age_region = min_age_region; scheme->max_age_region = max_age_region; scheme->action = action; - scheme->stat_count = 0; - scheme->stat_sz = 0; + scheme->stat = (struct damos_stat){}; INIT_LIST_HEAD(&scheme->list); scheme->quota.ms = quota->ms; @@ -574,6 +573,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, struct damos_quota *quota = &s->quota; unsigned long sz = r->ar.end - r->ar.start; struct timespec64 begin, end; + unsigned long sz_applied = 0; if (!s->wmarks.activated) continue; @@ -627,7 +627,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, damon_split_region_at(c, t, r, sz); } ktime_get_coarse_ts64(&begin); - c->primitive.apply_scheme(c, t, r, s); + sz_applied = c->primitive.apply_scheme(c, t, r, s); ktime_get_coarse_ts64(&end); quota->total_charged_ns += timespec64_to_ns(&end) - timespec64_to_ns(&begin); @@ -641,8 +641,11 @@ static void damon_do_apply_schemes(struct damon_ctx *c, r->age = 0; update_stat: - s->stat_count++; - s->stat_sz += sz; + s->stat.nr_tried++; + s->stat.sz_tried += sz; + if (sz_applied) + s->stat.nr_applied++; + s->stat.sz_applied += sz_applied; } } diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index bf36a2756cfb..9318b52d0b46 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -117,7 +117,7 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) s->quota.weight_age, s->wmarks.metric, s->wmarks.interval, s->wmarks.high, s->wmarks.mid, s->wmarks.low, - s->stat_count, s->stat_sz); + s->stat.nr_tried, s->stat.sz_tried); if (!rc) return -ENOMEM; diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 4318134cbc4c..5e8244f65a1a 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -213,14 +213,15 @@ bool damon_pa_target_valid(void *t) return true; } -static int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, - struct damon_region *r, struct damos *scheme) +static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) { - unsigned long addr; + unsigned long addr, applied; LIST_HEAD(page_list); if (scheme->action != DAMOS_PAGEOUT) - return -EINVAL; + return 0; for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { struct page *page = damon_get_page(PHYS_PFN(addr)); @@ -241,9 +242,9 @@ static int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, put_page(page); } } - reclaim_pages(&page_list); + applied = reclaim_pages(&page_list); cond_resched(); - return 0; + return applied * PAGE_SIZE; } static int damon_pa_scheme_score(struct damon_ctx *context, diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 981785c82f99..11d9a63fd4cb 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -571,32 +571,34 @@ bool damon_va_target_valid(void *target) } #ifndef CONFIG_ADVISE_SYSCALLS -static int damos_madvise(struct damon_target *target, struct damon_region *r, - int behavior) +static unsigned long damos_madvise(struct damon_target *target, + struct damon_region *r, int behavior) { - return -EINVAL; + return 0; } #else -static int damos_madvise(struct damon_target *target, struct damon_region *r, - int behavior) +static unsigned long damos_madvise(struct damon_target *target, + struct damon_region *r, int behavior) { struct mm_struct *mm; - int ret = -ENOMEM; + unsigned long start = PAGE_ALIGN(r->ar.start); + unsigned long len = PAGE_ALIGN(r->ar.end - r->ar.start); + unsigned long applied; mm = damon_get_mm(target); if (!mm) - goto out; + return 0; - ret = do_madvise(mm, PAGE_ALIGN(r->ar.start), - PAGE_ALIGN(r->ar.end - r->ar.start), behavior); + applied = do_madvise(mm, start, len, behavior) ? 0 : len; mmput(mm); -out: - return ret; + + return applied; } #endif /* CONFIG_ADVISE_SYSCALLS */ -static int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, - struct damon_region *r, struct damos *scheme) +static unsigned long damon_va_apply_scheme(struct damon_ctx *ctx, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) { int madv_action; @@ -619,7 +621,7 @@ static int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t, case DAMOS_STAT: return 0; default: - return -EINVAL; + return 0; } return damos_madvise(t, r, madv_action); From f755f1a2bc6160162398694aff1281a2fe92bd10 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:10:20 -0800 Subject: [PATCH 117/156] UPSTREAM: mm/damon/schemes: account how many times quota limit has exceeded If the time/space quotas of a given DAMON-based operation scheme is too small, the scheme could show unexpectedly slow progress. However, there is no good way to notice the case in runtime. This commit extends the DAMOS stat to provide how many times the quota limits exceeded so that the users can easily notice the case and tune the scheme. Link: https://lkml.kernel.org/r/20211210150016.35349-3-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 6268eac34ca30af7f6313504d556ec7fcd295621) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I36184dc51917810c81ac8d576b144e33a4454dc1 --- include/linux/damon.h | 2 ++ mm/damon/core.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 59fcb5f89594..9dbb2210a286 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -195,12 +195,14 @@ struct damos_watermarks { * @sz_tried: Total size of regions that the scheme is tried to be applied. * @nr_applied: Total number of regions that the scheme is applied. * @sz_applied: Total size of regions that the scheme is applied. + * @qt_exceeds: Total number of times the quota of the scheme has exceeded. */ struct damos_stat { unsigned long nr_tried; unsigned long sz_tried; unsigned long nr_applied; unsigned long sz_applied; + unsigned long qt_exceeds; }; /** diff --git a/mm/damon/core.c b/mm/damon/core.c index d745bf28509f..d5120b326e1b 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -693,6 +693,8 @@ static void kdamond_apply_schemes(struct damon_ctx *c) if (time_after_eq(jiffies, quota->charged_from + msecs_to_jiffies( quota->reset_interval))) { + if (quota->esz && quota->charged_sz >= quota->esz) + s->stat.qt_exceeds++; quota->total_charged_sz += quota->charged_sz; quota->charged_from = jiffies; quota->charged_sz = 0; From 1c400b8796b8d025c7455e3ababa684a76243a49 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:10:23 -0800 Subject: [PATCH 118/156] UPSTREAM: mm/damon/reclaim: provide reclamation statistics This implements new DAMON_RECLAIM parameters for statistics reporting. Those can be used for understanding how DAMON_RECLAIM is working, and for tuning the other parameters. Link: https://lkml.kernel.org/r/20211210150016.35349-4-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 60e52e7c46a127bca5ddd48b89002564f3862063) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I721bdacb3b2d8b41154296f5dbf8ef48c5dd0744 --- mm/damon/reclaim.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index dc1485044eaf..bc476cef688e 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -185,6 +185,36 @@ module_param(monitor_region_end, ulong, 0600); static int kdamond_pid __read_mostly = -1; module_param(kdamond_pid, int, 0400); +/* + * Number of memory regions that tried to be reclaimed. + */ +static unsigned long nr_reclaim_tried_regions __read_mostly; +module_param(nr_reclaim_tried_regions, ulong, 0400); + +/* + * Total bytes of memory regions that tried to be reclaimed. + */ +static unsigned long bytes_reclaim_tried_regions __read_mostly; +module_param(bytes_reclaim_tried_regions, ulong, 0400); + +/* + * Number of memory regions that successfully be reclaimed. + */ +static unsigned long nr_reclaimed_regions __read_mostly; +module_param(nr_reclaimed_regions, ulong, 0400); + +/* + * Total bytes of memory regions that successfully be reclaimed. + */ +static unsigned long bytes_reclaimed_regions __read_mostly; +module_param(bytes_reclaimed_regions, ulong, 0400); + +/* + * Number of times that the time/space quota limits have exceeded + */ +static unsigned long nr_quota_exceeds __read_mostly; +module_param(nr_quota_exceeds, ulong, 0400); + static struct damon_ctx *ctx; static struct damon_target *target; @@ -333,6 +363,21 @@ static void damon_reclaim_timer_fn(struct work_struct *work) } static DECLARE_DELAYED_WORK(damon_reclaim_timer, damon_reclaim_timer_fn); +static int damon_reclaim_after_aggregation(struct damon_ctx *c) +{ + struct damos *s; + + /* update the stats parameter */ + damon_for_each_scheme(s, c) { + nr_reclaim_tried_regions = s->stat.nr_tried; + bytes_reclaim_tried_regions = s->stat.sz_tried; + nr_reclaimed_regions = s->stat.nr_applied; + bytes_reclaimed_regions = s->stat.sz_applied; + nr_quota_exceeds = s->stat.qt_exceeds; + } + return 0; +} + static int __init damon_reclaim_init(void) { ctx = damon_new_ctx(); @@ -340,6 +385,7 @@ static int __init damon_reclaim_init(void) return -ENOMEM; damon_pa_set_primitives(ctx); + ctx->callback.after_aggregation = damon_reclaim_after_aggregation; /* 4242 means nothing but fun */ target = damon_new_target(4242); From b0cf3ac6d3b38a33593f652da81a4606ec2c42a5 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:10:29 -0800 Subject: [PATCH 119/156] UPSTREAM: mm/damon/dbgfs: support all DAMOS stats Currently, DAMON debugfs interface is not supporting DAMON-based Operation Schemes (DAMOS) stats for schemes successfully applied regions and time/space quota limit exceeds. This adds the support. Link: https://lkml.kernel.org/r/20211210150016.35349-6-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 3a619fdb8de8a3ecd4200e7d183d2c8ceb32289e) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ide3cfaff1b63e053ece5a8746a463fb87ef6c607 --- mm/damon/dbgfs.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 9318b52d0b46..751c7b835684 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -105,7 +105,7 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) damon_for_each_scheme(s, c) { rc = scnprintf(&buf[written], len - written, - "%lu %lu %u %u %u %u %d %lu %lu %lu %u %u %u %d %lu %lu %lu %lu %lu %lu\n", + "%lu %lu %u %u %u %u %d %lu %lu %lu %u %u %u %d %lu %lu %lu %lu %lu %lu %lu %lu %lu\n", s->min_sz_region, s->max_sz_region, s->min_nr_accesses, s->max_nr_accesses, s->min_age_region, s->max_age_region, @@ -117,7 +117,9 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) s->quota.weight_age, s->wmarks.metric, s->wmarks.interval, s->wmarks.high, s->wmarks.mid, s->wmarks.low, - s->stat.nr_tried, s->stat.sz_tried); + s->stat.nr_tried, s->stat.sz_tried, + s->stat.nr_applied, s->stat.sz_applied, + s->stat.qt_exceeds); if (!rc) return -ENOMEM; From 73faa856e92b64ac268e6b02cbb75ed455057213 Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Fri, 14 Jan 2022 14:10:35 -0800 Subject: [PATCH 120/156] UPSTREAM: mm/damon: add access checking for hugetlb pages The process's VMAs can be mapped by hugetlb page, but now the DAMON did not implement the access checking for hugetlb pte, so we can not get the actual access count like below if a process VMAs were mapped by hugetlb. damon_aggregated: target_id=18446614368406014464 nr_regions=12 4194304-5476352: 0 545 damon_aggregated: target_id=18446614368406014464 nr_regions=12 140662370467840-140662372970496: 0 545 damon_aggregated: target_id=18446614368406014464 nr_regions=12 140662372970496-140662375460864: 0 545 damon_aggregated: target_id=18446614368406014464 nr_regions=12 140662375460864-140662377951232: 0 545 damon_aggregated: target_id=18446614368406014464 nr_regions=12 140662377951232-140662380449792: 0 545 damon_aggregated: target_id=18446614368406014464 nr_regions=12 140662380449792-140662382944256: 0 545 ...... Thus this patch adds hugetlb access checking support, with this patch we can see below VMA mapped by hugetlb access count. damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296486649856-140296489914368: 1 3 damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296489914368-140296492978176: 1 3 damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296492978176-140296495439872: 1 3 damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296495439872-140296498311168: 1 3 damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296498311168-140296501198848: 1 3 damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296501198848-140296504320000: 1 3 damon_aggregated: target_id=18446613056935405824 nr_regions=12 140296504320000-140296507568128: 1 2 ...... [baolin.wang@linux.alibaba.com: fix unused var warning] Link: https://lkml.kernel.org/r/1aaf9c11-0d8e-b92d-5c92-46e50a6e8d4e@linux.alibaba.com [baolin.wang@linux.alibaba.com: v3] Link: https://lkml.kernel.org/r/486927ecaaaecf2e3a7fbe0378ec6e1c58b50747.1640852276.git.baolin.wang@linux.alibaba.com Link: https://lkml.kernel.org/r/6afcbd1fda5f9c7c24f320d26a98188c727ceec3.1639623751.git.baolin.wang@linux.alibaba.com Signed-off-by: Baolin Wang Reviewed-by: SeongJae Park Cc: Mike Kravetz Cc: Randy Dunlap Cc: Stephen Rothwell Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 49f4203aae06ba9d67b500c90339b262b0a52637) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Id7c86f5c0344efef150b3b27f662b696263947ed --- mm/damon/vaddr.c | 96 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 11d9a63fd4cb..57e34d164960 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -387,8 +387,65 @@ out: return 0; } +#ifdef CONFIG_HUGETLB_PAGE +static void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + bool referenced = false; + pte_t entry = huge_ptep_get(pte); + struct page *page = pte_page(entry); + + if (!page) + return; + + get_page(page); + + if (pte_young(entry)) { + referenced = true; + entry = pte_mkold(entry); + huge_ptep_set_access_flags(vma, addr, pte, entry, + vma->vm_flags & VM_WRITE); + } + +#ifdef CONFIG_MMU_NOTIFIER + if (mmu_notifier_clear_young(mm, addr, + addr + huge_page_size(hstate_vma(vma)))) + referenced = true; +#endif /* CONFIG_MMU_NOTIFIER */ + + if (referenced) + set_page_young(page); + + set_page_idle(page); + put_page(page); +} + +static int damon_mkold_hugetlb_entry(pte_t *pte, unsigned long hmask, + unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + struct hstate *h = hstate_vma(walk->vma); + spinlock_t *ptl; + pte_t entry; + + ptl = huge_pte_lock(h, walk->mm, pte); + entry = huge_ptep_get(pte); + if (!pte_present(entry)) + goto out; + + damon_hugetlb_mkold(pte, walk->mm, walk->vma, addr); + +out: + spin_unlock(ptl); + return 0; +} +#else +#define damon_mkold_hugetlb_entry NULL +#endif /* CONFIG_HUGETLB_PAGE */ + static const struct mm_walk_ops damon_mkold_ops = { .pmd_entry = damon_mkold_pmd_entry, + .hugetlb_entry = damon_mkold_hugetlb_entry, }; static void damon_va_mkold(struct mm_struct *mm, unsigned long addr) @@ -483,8 +540,47 @@ out: return 0; } +#ifdef CONFIG_HUGETLB_PAGE +static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask, + unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + struct damon_young_walk_private *priv = walk->private; + struct hstate *h = hstate_vma(walk->vma); + struct page *page; + spinlock_t *ptl; + pte_t entry; + + ptl = huge_pte_lock(h, walk->mm, pte); + entry = huge_ptep_get(pte); + if (!pte_present(entry)) + goto out; + + page = pte_page(entry); + if (!page) + goto out; + + get_page(page); + + if (pte_young(entry) || !page_is_idle(page) || + mmu_notifier_test_young(walk->mm, addr)) { + *priv->page_sz = huge_page_size(h); + priv->young = true; + } + + put_page(page); + +out: + spin_unlock(ptl); + return 0; +} +#else +#define damon_young_hugetlb_entry NULL +#endif /* CONFIG_HUGETLB_PAGE */ + static const struct mm_walk_ops damon_young_ops = { .pmd_entry = damon_young_pmd_entry, + .hugetlb_entry = damon_young_hugetlb_entry, }; static bool damon_va_young(struct mm_struct *mm, unsigned long addr, From 21dc18f9a0e6ce16bb5109cdd034299c7d7c9991 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Fri, 14 Jan 2022 14:10:38 -0800 Subject: [PATCH 121/156] UPSTREAM: mm/damon: move the implementation of damon_insert_region to damon.h Usually, inline function is declared static since it should sit between storage and type. And implement it in a header file if used by multiple files. And this change also fixes compile issue when backport damon to 5.10. mm/damon/vaddr.c: In function `damon_va_evenly_split_region': ./include/linux/damon.h:425:13: error: inlining failed in call to `always_inline' `damon_insert_region': function body not available 425 | inline void damon_insert_region(struct damon_region *r, | ^~~~~~~~~~~~~~~~~~~ mm/damon/vaddr.c:86:3: note: called from here 86 | damon_insert_region(n, r, next, t); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Link: https://lkml.kernel.org/r/20211223085703.6142-1-guoqing.jiang@linux.dev Signed-off-by: Guoqing Jiang Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 2cd4b8e10cc31eadb5b10b1d73b3f28156f3776c) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Iaa05318092b8e98bfbfc72a8c5df2cb6de97d224 --- include/linux/damon.h | 13 +++++++++++-- mm/damon/core.c | 11 ----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index 9dbb2210a286..b4e3dfb62063 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -448,9 +448,18 @@ static inline struct damon_region *damon_last_region(struct damon_target *t) #ifdef CONFIG_DAMON struct damon_region *damon_new_region(unsigned long start, unsigned long end); -inline void damon_insert_region(struct damon_region *r, + +/* + * Add a region between two other regions + */ +static inline void damon_insert_region(struct damon_region *r, struct damon_region *prev, struct damon_region *next, - struct damon_target *t); + struct damon_target *t) +{ + __list_add(&r->list, &prev->list, &next->list); + t->nr_regions++; +} + void damon_add_region(struct damon_region *r, struct damon_target *t); void damon_destroy_region(struct damon_region *r, struct damon_target *t); diff --git a/mm/damon/core.c b/mm/damon/core.c index d5120b326e1b..6482d510dcbe 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -49,17 +49,6 @@ struct damon_region *damon_new_region(unsigned long start, unsigned long end) return region; } -/* - * Add a region between two other regions - */ -inline void damon_insert_region(struct damon_region *r, - struct damon_region *prev, struct damon_region *next, - struct damon_target *t) -{ - __list_add(&r->list, &prev->list, &next->list); - t->nr_regions++; -} - void damon_add_region(struct damon_region *r, struct damon_target *t) { list_add_tail(&r->list, &t->regions_list); From 98dcd2427c7752d360e0804ea76a9a62da2c27c5 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:10:41 -0800 Subject: [PATCH 122/156] UPSTREAM: mm/damon/dbgfs: remove an unnecessary variable Patch series "mm/damon: Hide unnecessary information disclosures". DAMON is exposing some unnecessary information including kernel pointer in kernel log and tracepoint. This patchset hides such information. The first patch is only for a trivial cleanup, though. This patch (of 4): This commit removes a unnecessarily used variable in dbgfs_target_ids_write(). Link: https://lkml.kernel.org/r/20211229131016.23641-1-sj@kernel.org Link: https://lkml.kernel.org/r/20211229131016.23641-2-sj@kernel.org Fixes: 4bc05954d007 ("mm/damon: implement a debugfs-based user space interface") Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 70b8480812d0a3930049a44820a1fa149b090c10) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I774c900de92780ae9ebf09f01c0b0536eb43f822 --- mm/damon/dbgfs.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 751c7b835684..5b899601e56c 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -364,7 +364,7 @@ static ssize_t dbgfs_target_ids_write(struct file *file, struct damon_ctx *ctx = file->private_data; struct damon_target *t, *next_t; bool id_is_pid = true; - char *kbuf, *nrs; + char *kbuf; unsigned long *targets; ssize_t nr_targets; ssize_t ret; @@ -374,14 +374,13 @@ static ssize_t dbgfs_target_ids_write(struct file *file, if (IS_ERR(kbuf)) return PTR_ERR(kbuf); - nrs = kbuf; if (!strncmp(kbuf, "paddr\n", count)) { id_is_pid = false; /* target id is meaningless here, but we set it just for fun */ scnprintf(kbuf, count, "42 "); } - targets = str_to_target_ids(nrs, count, &nr_targets); + targets = str_to_target_ids(kbuf, count, &nr_targets); if (!targets) { ret = -ENOMEM; goto out; From c44028a162dcba93db820c837bb25995e1df3df1 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:10:44 -0800 Subject: [PATCH 123/156] UPSTREAM: mm/damon/vaddr: use pr_debug() for damon_va_three_regions() failure logging Failure of 'damon_va_three_regions()' is logged using 'pr_err()'. But, the function can fail in legal situations. To avoid making users be surprised and to keep the kernel clean, this makes the log to be printed using 'pr_debug()'. Link: https://lkml.kernel.org/r/20211229131016.23641-3-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 251403f19aab6a122f4dcfb14149814e85564202) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I0a2c08d856d0067af6af7a008e89f63fa5fac381 --- mm/damon/vaddr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 57e34d164960..420ef396b136 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -237,7 +237,7 @@ static void __damon_va_init_regions(struct damon_ctx *ctx, int i; if (damon_va_three_regions(t, regions)) { - pr_err("Failed to get three regions of target %lu\n", t->id); + pr_debug("Failed to get three regions of target %lu\n", t->id); return; } From 4c350065aad27ea5447c90e43a671d349f4d0184 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:10:47 -0800 Subject: [PATCH 124/156] UPSTREAM: mm/damon/vaddr: hide kernel pointer from damon_va_three_regions() failure log The failure log message for 'damon_va_three_regions()' prints the target id, which is a 'struct pid' pointer in the case. To avoid exposing the kernel pointer via the log, this makes the log to use the index of the target in the context's targets list instead. Link: https://lkml.kernel.org/r/20211229131016.23641-4-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 962fe7a6b1b2f9deb1b31b3344afa3b11afdf7ab) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I6b9b09a9b7024630f05ec41db030006d63f47ce1 --- mm/damon/vaddr.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 420ef396b136..5a8945571c59 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -231,13 +231,19 @@ static int damon_va_three_regions(struct damon_target *t, static void __damon_va_init_regions(struct damon_ctx *ctx, struct damon_target *t) { + struct damon_target *ti; struct damon_region *r; struct damon_addr_range regions[3]; unsigned long sz = 0, nr_pieces; - int i; + int i, tidx = 0; if (damon_va_three_regions(t, regions)) { - pr_debug("Failed to get three regions of target %lu\n", t->id); + damon_for_each_target(ti, ctx) { + if (ti == t) + break; + tidx++; + } + pr_debug("Failed to get three regions of %dth target\n", tidx); return; } From 94006b8a3a728cc958fb4107f7db5647882720e3 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 14 Jan 2022 14:10:50 -0800 Subject: [PATCH 125/156] UPSTREAM: mm/damon: hide kernel pointer from tracepoint event DAMON's virtual address spaces monitoring primitive uses 'struct pid *' of the target process as its monitoring target id. The kernel address is exposed as-is to the user space via the DAMON tracepoint, 'damon_aggregated'. Though primarily only privileged users are allowed to access that, it would be better to avoid unnecessarily exposing kernel pointers so. Because the trace result is only required to be able to distinguish each target, we aren't need to use the pointer as-is. This makes the tracepoint to use the index of the target in the context's targets list as its id in the tracepoint, to hide the kernel space address. Link: https://lkml.kernel.org/r/20211229131016.23641-5-sj@kernel.org Signed-off-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 76fd0285b447991267e838842c0be7395eb454bb) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Iee4a6f56cf9bf5f61fb95f438dfc3316c10198e5 --- include/trace/events/damon.h | 8 ++++---- mm/damon/core.c | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h index 2f422f4f1fb9..e0a344fda98e 100644 --- a/include/trace/events/damon.h +++ b/include/trace/events/damon.h @@ -11,10 +11,10 @@ TRACE_EVENT(damon_aggregated, - TP_PROTO(struct damon_target *t, struct damon_region *r, - unsigned int nr_regions), + TP_PROTO(struct damon_target *t, unsigned int target_id, + struct damon_region *r, unsigned int nr_regions), - TP_ARGS(t, r, nr_regions), + TP_ARGS(t, target_id, r, nr_regions), TP_STRUCT__entry( __field(unsigned long, target_id) @@ -25,7 +25,7 @@ TRACE_EVENT(damon_aggregated, ), TP_fast_assign( - __entry->target_id = t->id; + __entry->target_id = target_id; __entry->nr_regions = nr_regions; __entry->start = r->ar.start; __entry->end = r->ar.end; diff --git a/mm/damon/core.c b/mm/damon/core.c index 6482d510dcbe..1dd153c31c9e 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -514,15 +514,17 @@ static bool kdamond_aggregate_interval_passed(struct damon_ctx *ctx) static void kdamond_reset_aggregated(struct damon_ctx *c) { struct damon_target *t; + unsigned int ti = 0; /* target's index */ damon_for_each_target(t, c) { struct damon_region *r; damon_for_each_region(r, t) { - trace_damon_aggregated(t, r, damon_nr_regions(t)); + trace_damon_aggregated(t, ti, r, damon_nr_regions(t)); r->last_nr_accesses = r->nr_accesses; r->nr_accesses = 0; } + ti++; } } From 3ecd29b57b4f72b814d51d6649cb47310fa36fe5 Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 14 Jan 2022 14:09:37 -0800 Subject: [PATCH 126/156] UPSTREAM: mm/damon: add 'age' of region tracepoint support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In Damon, we can get age information by analyzing the nr_access change, But short time sampling is not effective, we have to obtain enough data for analysis through long time trace, this also means that we need to consume more cpu resources and storage space. Now the region add a new 'age' variable, we only need to get the change of age value through a little time trace, for example, age has been increasing to 141, but nr_access shows a value of 0 at the same time, Through this，we can conclude that the region has a very low nr_access value for a long time. Link: https://lkml.kernel.org/r/b9def1262af95e0dc1d0caea447886434db01161.1636989871.git.xhao@linux.alibaba.com Signed-off-by: Xin Hao Reviewed-by: SeongJae Park Cc: Muchun Song Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit c46b0bb6a735db0b6140e12e750b5acb1b032982) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I39aeea71756f5af6f270efe9d5653815c905af46 --- include/trace/events/damon.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h index e0a344fda98e..c79f1d4c39af 100644 --- a/include/trace/events/damon.h +++ b/include/trace/events/damon.h @@ -22,6 +22,7 @@ TRACE_EVENT(damon_aggregated, __field(unsigned long, start) __field(unsigned long, end) __field(unsigned int, nr_accesses) + __field(unsigned int, age) ), TP_fast_assign( @@ -30,11 +31,13 @@ TRACE_EVENT(damon_aggregated, __entry->start = r->ar.start; __entry->end = r->ar.end; __entry->nr_accesses = r->nr_accesses; + __entry->age = r->age; ), - TP_printk("target_id=%lu nr_regions=%u %lu-%lu: %u", + TP_printk("target_id=%lu nr_regions=%u %lu-%lu: %u %u", __entry->target_id, __entry->nr_regions, - __entry->start, __entry->end, __entry->nr_accesses) + __entry->start, __entry->end, + __entry->nr_accesses, __entry->age) ); #endif /* _TRACE_DAMON_H */ From 70ff0aeea7ee373995134abfccdd05cc7521b7e4 Mon Sep 17 00:00:00 2001 From: Xin Hao Date: Fri, 14 Jan 2022 14:09:56 -0800 Subject: [PATCH 127/156] UPSTREAM: mm/damon: modify damon_rand() macro to static inline function damon_rand() cannot be implemented as a macro. Example: damon_rand(a++, b); The value of 'a' will be incremented twice, This is obviously unreasonable, So there fix it. Link: https://lkml.kernel.org/r/110ffcd4e420c86c42b41ce2bc9f0fe6a4f32cd3.1638795127.git.xhao@linux.alibaba.com Fixes: b9a6ac4e4ede ("mm/damon: adaptively adjust regions") Signed-off-by: Xin Hao Reported-by: Andrew Morton Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit 234d68732b6c135087bdebfa0630a43ae8c27758) Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Idcc316e6f582959254111fb160d04875d235c306 --- include/linux/damon.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/linux/damon.h b/include/linux/damon.h index b4e3dfb62063..5e1e3a128b77 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -19,7 +19,10 @@ #define DAMOS_MAX_SCORE (99) /* Get a random number in [l, r) */ -#define damon_rand(l, r) (l + prandom_u32_max(r - l)) +static inline unsigned long damon_rand(unsigned long l, unsigned long r) +{ + return l + prandom_u32_max(r - l); +} /** * struct damon_addr_range - Represents an address region of [@start, @end). From 2522f6c4da7e3fe18d71627c23eb028d4db2468c Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Thu, 2 Dec 2021 12:34:00 -0800 Subject: [PATCH 128/156] BACKPORT: treewide: Add missing includes masked by cgroup -> bpf dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cgroup.h (therefore swap.h, therefore half of the universe) includes bpf.h which in turn includes module.h and slab.h. Since we're about to get rid of that dependency we need to clean things up. v2: drop the cpu.h include from cacheinfo.h, it's not necessary and it makes riscv sensitive to ordering of include files. Signed-off-by: Jakub Kicinski Signed-off-by: Alexei Starovoitov Reviewed-by: Christoph Hellwig Acked-by: Krzysztof Wilczyński Acked-by: Peter Chen Acked-by: SeongJae Park Acked-by: Jani Nikula Acked-by: Greg Kroah-Hartman Link: https://lore.kernel.org/all/20211120035253.72074-1-kuba@kernel.org/ # v1 Link: https://lore.kernel.org/all/20211120165528.197359-1-kuba@kernel.org/ # cacheinfo discussion Link: https://lore.kernel.org/bpf/20211202203400.1208663-1-kuba@kernel.org (cherry picked from commit 8581fd402a0cf80b5298e3b225e7a7bd8f110e69) Dropped all the changes except for the ones made in mm/damon/vaddr.c Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ib64ecbe4e06f192ed576af77e03e6b49d538bac9 --- mm/damon/vaddr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 5a8945571c59..89b6468da2b9 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "prmtv-common.h" From 0453acd7fbf57bb97da2c30b569d86955c963246 Mon Sep 17 00:00:00 2001 From: Hailong Tu Date: Wed, 20 Apr 2022 16:37:15 +0800 Subject: [PATCH 129/156] FROMLIST: mm/damon/reclaim: Fix the timer always stays active The timer stays active even if the reclaim mechanism is never enabled. It is unnecessary overhead can be completely avoided by using module_param_cb() for enabled flag. Signed-off-by: Hailong Tu Link: https://lore.kernel.org/all/20220421125910.1052459-1-tuhailong@gmail.com/ Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: I77591e41ce424ce16a4a5c70e7a86cdae996a354 --- mm/damon/reclaim.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index bc476cef688e..183d4f3b4fde 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -28,7 +28,6 @@ * this. */ static bool enabled __read_mostly; -module_param(enabled, bool, 0600); /* * Time threshold for cold memory regions identification in microseconds. @@ -358,11 +357,35 @@ static void damon_reclaim_timer_fn(struct work_struct *work) enabled = last_enabled; } - schedule_delayed_work(&damon_reclaim_timer, + if (enabled) + schedule_delayed_work(&damon_reclaim_timer, msecs_to_jiffies(ENABLE_CHECK_INTERVAL_MS)); } static DECLARE_DELAYED_WORK(damon_reclaim_timer, damon_reclaim_timer_fn); +static int enabled_store(const char *val, + const struct kernel_param *kp) +{ + int rc = param_set_bool(val, kp); + + if (rc < 0) + return rc; + + if (enabled) + schedule_delayed_work(&damon_reclaim_timer, 0); + + return 0; +} + +static const struct kernel_param_ops enabled_param_ops = { + .set = enabled_store, + .get = param_get_bool, +}; + +module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); +MODULE_PARM_DESC(enabled, + "Enable or disable DAMON_RECLAIM (default: disabled)"); + static int damon_reclaim_after_aggregation(struct damon_ctx *c) { struct damos *s; From bb18f818eed314ee3b0a71293ce68462a0cf638d Mon Sep 17 00:00:00 2001 From: Hailong Tu Date: Sat, 23 Apr 2022 08:47:29 +0800 Subject: [PATCH 130/156] ANDROID: GKI: build damon reclaim To enable it, echo 1 > /sys/module/damon_reclaim/parameters/enable Bug: 228223814 Signed-off-by: Hailong Tu Change-Id: Ibb0701b6dd83070c6fbc8c96fdac3bc839821377 --- arch/arm64/configs/gki_defconfig | 3 +++ arch/x86/configs/gki_defconfig | 3 +++ 2 files changed, 6 insertions(+) diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 9703932d9262..7d7c4086de6c 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -118,6 +118,9 @@ CONFIG_CMA_DEBUGFS=y CONFIG_CMA_SYSFS=y CONFIG_CMA_AREAS=16 CONFIG_READ_ONLY_THP_FOR_FS=y +CONFIG_DAMON=y +CONFIG_DAMON_PADDR=y +CONFIG_DAMON_RECLAIM=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index 8525a0e365c5..14ea69c0d417 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -94,6 +94,9 @@ CONFIG_CMA_DEBUGFS=y CONFIG_CMA_SYSFS=y CONFIG_CMA_AREAS=16 CONFIG_READ_ONLY_THP_FOR_FS=y +CONFIG_DAMON=y +CONFIG_DAMON_PADDR=y +CONFIG_DAMON_RECLAIM=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y From 5558db2674b358df4e196272fb72629a07e1605a Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Wed, 27 Apr 2022 12:55:25 -0700 Subject: [PATCH 131/156] ANDROID: Add flag to indicate compiling against ACK Add a flag: __ANDROID_COMMON_KERNEL__ which out-of-tree vendor drivers can use to check if they are compiling against an Android Common Kernel. These out-of-tree vendor drivers can use this flag + LINUX_KERNEL_VERSION to determine if a feature has been backported. Bug: 229953929 Change-Id: I832344d63f3639479784753edfb7ac405068312f Signed-off-by: Elliot Berman --- build.config.common | 1 + 1 file changed, 1 insertion(+) diff --git a/build.config.common b/build.config.common index b321ae20ecea..e7d0b3e0003f 100644 --- a/build.config.common +++ b/build.config.common @@ -7,6 +7,7 @@ DTC=dtc CLANG_PREBUILT_BIN=prebuilts-master/clang/host/linux-x86/clang-r416183b/bin BUILDTOOLS_PREBUILT_BIN=build/build-tools/path/linux-x86 +KCFLAGS=-D__ANDROID_COMMON_KERNEL__ EXTRA_CMDS='' STOP_SHIP_TRACEPRINTK=1 IN_KERNEL_MODULES=1 From eb80a7e84fa80f14c9103358c1de5fce650d8b81 Mon Sep 17 00:00:00 2001 From: Peifeng LI Date: Fri, 29 Apr 2022 16:32:33 +0800 Subject: [PATCH 132/156] ANDROID: vendor_hooks: Add hooks for rwsem we want to record tasks who will own sem, so we need to add some hook in rwsem.c Bug: 230829284 Change-Id: Ide66540aa38d6058d8aad5f94f4403be991078a4 Signed-off-by: Peifeng Li --- drivers/android/vendor_hooks.c | 5 +++++ include/trace/hooks/rwsem.h | 16 +++++++++++++++- kernel/locking/rwsem.c | 5 +++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 7b5148376de2..142a7b14ba80 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -113,6 +113,11 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_read_wait_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_read_wait_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_write_wait_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_write_wait_finish); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_set_owner); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_set_reader_owned); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_mark_wake_readers); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_up_read_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_up_write_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sched_show_task); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shmem_alloc_page); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_enter); diff --git a/include/trace/hooks/rwsem.h b/include/trace/hooks/rwsem.h index d644a6e21259..c8e445216910 100644 --- a/include/trace/hooks/rwsem.h +++ b/include/trace/hooks/rwsem.h @@ -29,7 +29,21 @@ DECLARE_HOOK(android_vh_alter_rwsem_list_add, DECLARE_HOOK(android_vh_rwsem_wake_finish, TP_PROTO(struct rw_semaphore *sem), TP_ARGS(sem)); - +DECLARE_HOOK(android_vh_rwsem_set_owner, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_set_reader_owned, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_up_write_end, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_up_read_end, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_mark_wake_readers, + TP_PROTO(struct rw_semaphore *sem, struct rwsem_waiter *waiter), + TP_ARGS(sem, waiter)); /* macro versions of hooks are no longer required */ #endif /* _TRACE_HOOK_RWSEM_H */ diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index 30a5f91f99cf..a33e6a36dae9 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -176,6 +176,7 @@ static inline void rwsem_set_owner(struct rw_semaphore *sem) { atomic_long_set(&sem->owner, (long)current); + trace_android_vh_rwsem_set_owner(sem); } static inline void rwsem_clear_owner(struct rw_semaphore *sem) @@ -213,6 +214,7 @@ static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) { __rwsem_set_reader_owned(sem, current); + trace_android_vh_rwsem_set_reader_owned(sem); } /* @@ -496,6 +498,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, woken++; list_move_tail(&waiter->list, &wlist); + trace_android_vh_rwsem_mark_wake_readers(sem, waiter); /* * Limit # of readers that can be woken up per wakeup call. */ @@ -1460,6 +1463,7 @@ static inline void __up_read(struct rw_semaphore *sem) clear_wr_nonspinnable(sem); rwsem_wake(sem, tmp); } + trace_android_vh_rwsem_up_read_end(sem); } /* @@ -1481,6 +1485,7 @@ static inline void __up_write(struct rw_semaphore *sem) tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); if (unlikely(tmp & RWSEM_FLAG_WAITERS)) rwsem_wake(sem, tmp); + trace_android_vh_rwsem_up_write_end(sem); } /* From e5b4949bfc066d34876ce94725fe1355ce62d4c1 Mon Sep 17 00:00:00 2001 From: xiaofeng Date: Tue, 26 Apr 2022 18:06:09 +0800 Subject: [PATCH 133/156] ANDROID: vendor_hooks: tune reclaim scan type for specified mem_cgroup Add memcg support for hooks in the reclaim path Bug: 230450931 Change-Id: Ia3e6949985d915c8640139fbb93800d91e1e46f8 Signed-off-by: xiaofeng --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/vmscan.h | 3 +++ mm/vmscan.c | 1 + 3 files changed, 5 insertions(+) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 142a7b14ba80..de3a032822f5 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -403,3 +403,4 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_read_done); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_handle_tlb_conf); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_node_memcgs); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ra_tuning_max_page); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tune_memcg_scan_type); diff --git a/include/trace/hooks/vmscan.h b/include/trace/hooks/vmscan.h index 48a00c6cede0..7b4d222301af 100644 --- a/include/trace/hooks/vmscan.h +++ b/include/trace/hooks/vmscan.h @@ -31,6 +31,9 @@ DECLARE_HOOK(android_vh_page_referenced_check_bypass, DECLARE_HOOK(android_vh_shrink_node_memcgs, TP_PROTO(struct mem_cgroup *memcg, bool *skip), TP_ARGS(memcg, skip)); +DECLARE_HOOK(android_vh_tune_memcg_scan_type, + TP_PROTO(struct mem_cgroup *memcg, char *scan_type), + TP_ARGS(memcg, scan_type)); #endif /* _TRACE_HOOK_VMSCAN_H */ /* This part must be outside protection */ #include diff --git a/mm/vmscan.c b/mm/vmscan.c index 89beec62bfa2..0c47c99b4cdb 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2396,6 +2396,7 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, denominator = ap + fp; out: trace_android_vh_tune_scan_type((char *)(&scan_balance)); + trace_android_vh_tune_memcg_scan_type(memcg, (char *)(&scan_balance)); for_each_evictable_lru(lru) { int file = is_file_lru(lru); unsigned long lruvec_size; From 7c9d15f68b8ba5feea6287bf39c8566f57e36808 Mon Sep 17 00:00:00 2001 From: Lecopzer Chen Date: Fri, 29 Apr 2022 16:37:36 +0800 Subject: [PATCH 134/156] ANDROID: fix KCFLAGS override by __ANDROID_COMMON_KERNEL__ Our test build is broken by KCFLAGS overrided in build.config.comm. Since Linux Makefile supports 'export KCFLAGS=XXX' to customize the KCFLAGS, and we should keep this functionality. Bug: 230818006 Fixes: 5558db2674b3 ("ANDROID: Add flag to indicate compiling against ACK") Signed-off-by: Lecopzer Chen Change-Id: I9425d79697bc1fe816ce82d523f91631dee6b8f4 --- build.config.common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.config.common b/build.config.common index e7d0b3e0003f..9ec1c637a79f 100644 --- a/build.config.common +++ b/build.config.common @@ -7,7 +7,7 @@ DTC=dtc CLANG_PREBUILT_BIN=prebuilts-master/clang/host/linux-x86/clang-r416183b/bin BUILDTOOLS_PREBUILT_BIN=build/build-tools/path/linux-x86 -KCFLAGS=-D__ANDROID_COMMON_KERNEL__ +KCFLAGS="${KCFLAGS} -D__ANDROID_COMMON_KERNEL__" EXTRA_CMDS='' STOP_SHIP_TRACEPRINTK=1 IN_KERNEL_MODULES=1 From 75059d208e7d86e5b4568947f44ca55f1a7fcd67 Mon Sep 17 00:00:00 2001 From: Faiyaz Mohammed Date: Thu, 28 Apr 2022 13:58:42 +0530 Subject: [PATCH 135/156] ANDROID: Update QCOM symbol list for trace_map/unmap synchronize QCOM symbol list in android/abi_gki_aarch64_qcom for trace_map/unmap. Leaf changes summary: 4 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 2 Added functions Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 2 Added variables 2 Added functions: [A] 'function int __traceiter_map(void*, unsigned long int, phys_addr_t, size_t)' [A] 'function int __traceiter_unmap(void*, unsigned long int, size_t, size_t)' 2 Added variables: [A] 'tracepoint __tracepoint_map' [A] 'tracepoint __tracepoint_unmap' BUG: 230306469 Change-Id: I4f2a6f561f6c34912904a6ba354b5b9bd82cbdab Signed-off-by: Faiyaz Mohammed --- android/abi_gki_aarch64.xml | 20 ++++++++++++++++++++ android/abi_gki_aarch64_qcom | 4 ++++ 2 files changed, 24 insertions(+) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 7f3641720ed4..e5b5a17b490b 100644 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -614,6 +614,7 @@ + @@ -639,6 +640,7 @@ + @@ -6388,6 +6390,7 @@ + @@ -6413,6 +6416,7 @@ + @@ -117982,6 +117986,13 @@ + + + + + + + @@ -118130,6 +118141,13 @@ + + + + + + + @@ -118517,6 +118535,7 @@ + @@ -118542,6 +118561,7 @@ + diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index bcfc4c9ef782..fe1688823b24 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -2603,12 +2603,14 @@ __traceiter_ipi_entry __traceiter_ipi_raise __traceiter_irq_handler_entry + __traceiter_map __traceiter_rwmmio_post_read __traceiter_rwmmio_read __traceiter_rwmmio_write __traceiter_sched_overutilized_tp __traceiter_sched_switch __traceiter_suspend_resume + __traceiter_unmap __tracepoint_android_rvh_account_irq __tracepoint_android_rvh_after_dequeue_task __tracepoint_android_rvh_after_enqueue_task @@ -2727,6 +2729,7 @@ __tracepoint_ipi_entry __tracepoint_ipi_raise __tracepoint_irq_handler_entry + __tracepoint_map tracepoint_probe_register tracepoint_probe_register_prio tracepoint_probe_unregister @@ -2736,6 +2739,7 @@ __tracepoint_sched_overutilized_tp __tracepoint_sched_switch __tracepoint_suspend_resume + __tracepoint_unmap trace_print_array_seq trace_print_flags_seq trace_print_hex_seq From fe513e1c2685f5040f09e68e2f50f2feb3c73c67 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Wed, 9 Mar 2022 12:54:02 -0800 Subject: [PATCH 136/156] UPSTREAM: usb: dwc3: gadget: Wait for ep0 xfers to complete during dequeue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a Setup packet is received but yet to DMA out, the controller will not process the End Transfer command of any endpoint. Polling of its DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a command timeout. This may occur if the driver doesn’t service the completion interrupt of the control status stage yet due to system latency, then it won’t prepare TRB and start the transfer for the next Setup Stage. To the host side, the control transfer had completed, and the host can send a new Setup packet at this point. In the meanwhile, if the driver receives an async call to dequeue a request (triggering End Transfer) to any endpoint, then the driver will service that End transfer first, blocking the control status stage completion handler. Since no TRB is available for the Setup stage, the Setup packet can’t be DMA’ed out and the End Transfer gets hung. The driver must not block setting up of the Setup stage. So track and only issue the End Transfer command only when there’s Setup TRB prepared so that the controller can DMA out the Setup packet. Delay the End transfer command if there's no Setup TRB available. This is applicable to all DWC_usb3x IPs. Co-developed-by: Wesley Cheng Signed-off-by: Thinh Nguyen Signed-off-by: Wesley Cheng Link: https://lore.kernel.org/r/20220309205402.4467-1-quic_wcheng@quicinc.com Signed-off-by: Greg Kroah-Hartman Bug: 230843866 (cherry picked from commit e4cf6580ac740f766dae26203bd6311d353dcd42) Change-Id: I7f643ae340a20f12d5f86965674aa58df1be3979 Signed-off-by: Wesley Cheng --- drivers/usb/dwc3/core.h | 1 + drivers/usb/dwc3/ep0.c | 14 ++++++++++++++ drivers/usb/dwc3/gadget.c | 15 ++++++++++++++- drivers/usb/dwc3/gadget.h | 1 + 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 44f1364d1682..f234dac3e832 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -729,6 +729,7 @@ struct dwc3_ep { #define DWC3_EP_FIRST_STREAM_PRIMED BIT(10) #define DWC3_EP_PENDING_CLEAR_STALL BIT(11) #define DWC3_EP_TXFIFO_RESIZED BIT(12) +#define DWC3_EP_DELAY_STOP BIT(13) /* This last one is specific to EP0 */ #define DWC3_EP0_DIR_IN BIT(31) diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index 658739410992..1064be5518f6 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -271,6 +271,7 @@ void dwc3_ep0_out_start(struct dwc3 *dwc) { struct dwc3_ep *dep; int ret; + int i; complete(&dwc->ep0_in_setup); @@ -279,6 +280,19 @@ void dwc3_ep0_out_start(struct dwc3 *dwc) DWC3_TRBCTL_CONTROL_SETUP, false); ret = dwc3_ep0_start_trans(dep); WARN_ON(ret < 0); + for (i = 2; i < DWC3_ENDPOINTS_NUM; i++) { + struct dwc3_ep *dwc3_ep; + + dwc3_ep = dwc->eps[i]; + if (!dwc3_ep) + continue; + + if (!(dwc3_ep->flags & DWC3_EP_DELAY_STOP)) + continue; + + dwc3_ep->flags &= ~DWC3_EP_DELAY_STOP; + dwc3_stop_active_transfer(dwc3_ep, true, true); + } } static struct dwc3_ep *dwc3_wIndex_to_dep(struct dwc3 *dwc, __le16 wIndex_le) diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 93759e839267..ebb6deac2bd0 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -1874,6 +1874,7 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) */ if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || (dep->flags & DWC3_EP_WEDGE) || + (dep->flags & DWC3_EP_DELAY_STOP) || (dep->flags & DWC3_EP_STALL)) { dep->flags |= DWC3_EP_DELAY_START; return 0; @@ -2008,6 +2009,16 @@ static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, if (r == req) { struct dwc3_request *t; + /* + * If a Setup packet is received but yet to DMA out, the controller will + * not process the End Transfer command of any endpoint. Polling of its + * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a + * timeout. Delay issuing the End Transfer command until the Setup TRB is + * prepared. + */ + if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) + dep->flags |= DWC3_EP_DELAY_STOP; + /* wait until it is processed */ dwc3_stop_active_transfer(dep, true, true); @@ -2089,7 +2100,8 @@ int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) if (!list_empty(&dep->started_list)) dep->flags |= DWC3_EP_DELAY_START; - if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) { + if (dep->flags & DWC3_EP_END_TRANSFER_PENDING || + (dep->flags & DWC3_EP_DELAY_STOP)) { dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; return 0; } @@ -3585,6 +3597,7 @@ void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, int ret; if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || + (dep->flags & DWC3_EP_DELAY_STOP) || (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) return; diff --git a/drivers/usb/dwc3/gadget.h b/drivers/usb/dwc3/gadget.h index 77df4b6d6c13..f763380e672e 100644 --- a/drivers/usb/dwc3/gadget.h +++ b/drivers/usb/dwc3/gadget.h @@ -116,6 +116,7 @@ int dwc3_gadget_ep0_queue(struct usb_ep *ep, struct usb_request *request, gfp_t gfp_flags); int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol); void dwc3_ep0_send_delayed_status(struct dwc3 *dwc); +void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt); /** * dwc3_gadget_ep_get_transfer_index - Gets transfer index from HW From 1a73ed4b2e1e4c9bbf18bbb864c9175cf296883b Mon Sep 17 00:00:00 2001 From: Wesley Cheng Date: Tue, 15 Mar 2022 18:13:58 -0700 Subject: [PATCH 137/156] UPSTREAM: usb: dwc3: Issue core soft reset before enabling run/stop It is recommended by the Synopsis databook to issue a DCTL.CSftReset when reconnecting from a device-initiated disconnect routine. This resolves issues with enumeration during fast composition switching cases, which result in an unknown device on the host. Reviewed-by: Thinh Nguyen Signed-off-by: Wesley Cheng Link: https://lore.kernel.org/r/20220316011358.3057-1-quic_wcheng@quicinc.com Signed-off-by: Greg Kroah-Hartman Bug: 230843866 (cherry picked from commit 0066472de157439d58454f4a55786f1045ea5681) Change-Id: Idcfbb43e844c22cdfcb1362177e9c1a8e9a7e18e Signed-off-by: Wesley Cheng --- drivers/usb/dwc3/core.c | 4 +--- drivers/usb/dwc3/core.h | 2 ++ drivers/usb/dwc3/gadget.c | 11 +++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index cb514a3bd449..ca0966badf42 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -114,8 +114,6 @@ void dwc3_set_prtcap(struct dwc3 *dwc, u32 mode) dwc->current_dr_role = mode; } -static int dwc3_core_soft_reset(struct dwc3 *dwc); - static void __dwc3_set_mode(struct work_struct *work) { struct dwc3 *dwc = work_to_dwc(work); @@ -260,7 +258,7 @@ u32 dwc3_core_fifo_space(struct dwc3_ep *dep, u8 type) * dwc3_core_soft_reset - Issues core soft reset and PHY reset * @dwc: pointer to our context structure */ -static int dwc3_core_soft_reset(struct dwc3 *dwc) +int dwc3_core_soft_reset(struct dwc3 *dwc) { u32 reg; int retries = 1000; diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index f234dac3e832..28df4e8b9359 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1540,6 +1540,8 @@ bool dwc3_has_imod(struct dwc3 *dwc); int dwc3_event_buffers_setup(struct dwc3 *dwc); void dwc3_event_buffers_cleanup(struct dwc3 *dwc); +int dwc3_core_soft_reset(struct dwc3 *dwc); + #if IS_ENABLED(CONFIG_USB_DWC3_HOST) || IS_ENABLED(CONFIG_USB_DWC3_DUAL_ROLE) int dwc3_host_init(struct dwc3 *dwc); void dwc3_host_exit(struct dwc3 *dwc); diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index ebb6deac2bd0..148434e8f267 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -2532,6 +2532,17 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) dwc->ev_buf->length; } } else { + /* + * In the Synopsys DWC_usb31 1.90a programming guide section + * 4.1.9, it specifies that for a reconnect after a + * device-initiated disconnect requires a core soft reset + * (DCTL.CSftRst) before enabling the run/stop bit. + */ + spin_unlock_irqrestore(&dwc->lock, flags); + dwc3_core_soft_reset(dwc); + spin_lock_irqsave(&dwc->lock, flags); + + dwc3_event_buffers_setup(dwc); __dwc3_gadget_start(dwc); } From 0780756946db50a7a17d98876958ed96e5fdee68 Mon Sep 17 00:00:00 2001 From: Wesley Cheng Date: Thu, 14 Apr 2022 11:35:21 -0700 Subject: [PATCH 138/156] UPSTREAM: usb: dwc3: gadget: Replace list_for_each_entry_safe() if using giveback The list_for_each_entry_safe() macro saves the current item (n) and the item after (n+1), so that n can be safely removed without corrupting the list. However, when traversing the list and removing items using gadget giveback, the DWC3 lock is briefly released, allowing other routines to execute. There is a situation where, while items are being removed from the cancelled_list using dwc3_gadget_ep_cleanup_cancelled_requests(), the pullup disable routine is running in parallel (due to UDC unbind). As the cleanup routine removes n, and the pullup disable removes n+1, once the cleanup retakes the DWC3 lock, it references a request who was already removed/handled. With list debug enabled, this leads to a panic. Ensure all instances of the macro are replaced where gadget giveback is used. Example call stack: Thread#1: __dwc3_gadget_ep_set_halt() - CLEAR HALT -> dwc3_gadget_ep_cleanup_cancelled_requests() ->list_for_each_entry_safe() ->dwc3_gadget_giveback(n) ->dwc3_gadget_del_and_unmap_request()- n deleted[cancelled_list] ->spin_unlock ->Thread#2 executes ... ->dwc3_gadget_giveback(n+1) ->Already removed! Thread#2: dwc3_gadget_pullup() ->waiting for dwc3 spin_lock ... ->Thread#1 released lock ->dwc3_stop_active_transfers() ->dwc3_remove_requests() ->fetches n+1 item from cancelled_list (n removed by Thread#1) ->dwc3_gadget_giveback() ->dwc3_gadget_del_and_unmap_request()- n+1 deleted[cancelled_list] ->spin_unlock Fixes: d4f1afe5e896 ("usb: dwc3: gadget: move requests to cancelled_list") Signed-off-by: Wesley Cheng Link: https://lore.kernel.org/r/20220414183521.23451-1-quic_wcheng@quicinc.com Signed-off-by: Greg Kroah-Hartman Bug: 230843866 (cherry picked from commit bf594d1d0c1d7b895954018043536ffd327844f9) Change-Id: I357028fb51441a7effe90d115f6faa3f2034d326 Signed-off-by: Wesley Cheng --- drivers/usb/dwc3/gadget.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 148434e8f267..fe10c72ad1e3 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -1954,10 +1954,10 @@ static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *r static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep) { struct dwc3_request *req; - struct dwc3_request *tmp; struct dwc3 *dwc = dep->dwc; - list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) { + while (!list_empty(&dep->cancelled_list)) { + req = next_request(&dep->cancelled_list); dwc3_gadget_ep_skip_trbs(dep, req); switch (req->status) { case DWC3_REQUEST_STATUS_DISCONNECTED: @@ -1974,6 +1974,12 @@ static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep) dwc3_gadget_giveback(dep, req, -ECONNRESET); break; } + /* + * The endpoint is disabled, let the dwc3_remove_requests() + * handle the cleanup. + */ + if (!dep->endpoint.desc) + break; } } @@ -3264,15 +3270,21 @@ static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep, const struct dwc3_event_depevt *event, int status) { struct dwc3_request *req; - struct dwc3_request *tmp; - list_for_each_entry_safe(req, tmp, &dep->started_list, list) { + while (!list_empty(&dep->started_list)) { int ret; + req = next_request(&dep->started_list); ret = dwc3_gadget_ep_cleanup_completed_request(dep, event, req, status); if (ret) break; + /* + * The endpoint is disabled, let the dwc3_remove_requests() + * handle the cleanup. + */ + if (!dep->endpoint.desc) + break; } } From 2f55d5cbe232170c7807465caf6617eca8980bd6 Mon Sep 17 00:00:00 2001 From: Liujie Xie Date: Fri, 6 May 2022 14:04:56 +0800 Subject: [PATCH 139/156] ANDROID: vendor_hooks: Add hooks for mutex Current might be preempt after spin_unlock(&lock->wait_lock), we want to add a hook after wake_up_q(&wake_q) in which to disable owner's privilege in scheduler. Bug: 231647361 Change-Id: I3016da2fd31b8bdc8435df4e800f91381a64af4f Signed-off-by: Liujie Xie --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/dtask.h | 3 +++ kernel/locking/mutex.c | 1 + 3 files changed, 5 insertions(+) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index de3a032822f5..eb50db428e8c 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -241,6 +241,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sync_txn_recvd); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_build_sched_domains); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alter_mutex_list_add); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_unlock_slowpath); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_unlock_slowpath_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_wake_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_undefinstr); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_ptrauth_fault); diff --git a/include/trace/hooks/dtask.h b/include/trace/hooks/dtask.h index 3c49af0e6560..675634238fbd 100644 --- a/include/trace/hooks/dtask.h +++ b/include/trace/hooks/dtask.h @@ -54,6 +54,9 @@ DECLARE_HOOK(android_vh_alter_mutex_list_add, DECLARE_HOOK(android_vh_mutex_unlock_slowpath, TP_PROTO(struct mutex *lock), TP_ARGS(lock)); +DECLARE_HOOK(android_vh_mutex_unlock_slowpath_end, + TP_PROTO(struct mutex *lock, struct task_struct *next), + TP_ARGS(lock, next)); /* macro versions of hooks are no longer required */ diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 016f40c76a83..fac88b1e134e 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -1293,6 +1293,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne spin_unlock(&lock->wait_lock); wake_up_q(&wake_q); + trace_android_vh_mutex_unlock_slowpath_end(lock, next); } #ifndef CONFIG_DEBUG_LOCK_ALLOC From a37eec78a6b7a08454f2a3a3029351ac98e6436d Mon Sep 17 00:00:00 2001 From: Minchan Kim Date: Thu, 14 Apr 2022 19:13:46 -0700 Subject: [PATCH 140/156] UPSTREAM: mm: fix unexpected zeroed page mapping with zram swap commit e914d8f00391520ecc4495dd0ca0124538ab7119 upstream. Two processes under CLONE_VM cloning, user process can be corrupted by seeing zeroed page unexpectedly. CPU A CPU B do_swap_page do_swap_page SWP_SYNCHRONOUS_IO path SWP_SYNCHRONOUS_IO path swap_readpage valid data swap_slot_free_notify delete zram entry swap_readpage zeroed(invalid) data pte_lock map the *zero data* to userspace pte_unlock pte_lock if (!pte_same) goto out_nomap; pte_unlock return and next refault will read zeroed data The swap_slot_free_notify is bogus for CLONE_VM case since it doesn't increase the refcount of swap slot at copy_mm so it couldn't catch up whether it's safe or not to discard data from backing device. In the case, only the lock it could rely on to synchronize swap slot freeing is page table lock. Thus, this patch gets rid of the swap_slot_free_notify function. With this patch, CPU A will see correct data. CPU A CPU B do_swap_page do_swap_page SWP_SYNCHRONOUS_IO path SWP_SYNCHRONOUS_IO path swap_readpage original data pte_lock map the original data swap_free swap_range_free bd_disk->fops->swap_slot_free_notify swap_readpage read zeroed data pte_unlock pte_lock if (!pte_same) goto out_nomap; pte_unlock return on next refault will see mapped data by CPU B The concern of the patch would increase memory consumption since it could keep wasted memory with compressed form in zram as well as uncompressed form in address space. However, most of cases of zram uses no readahead and do_swap_page is followed by swap_free so it will free the compressed form from in zram quickly. Link: https://lkml.kernel.org/r/YjTVVxIAsnKAXjTd@google.com Fixes: 0bcac06f27d7 ("mm, swap: skip swapcache for swapin of synchronous device") Reported-by: Ivan Babrou Tested-by: Ivan Babrou Signed-off-by: Minchan Kim Cc: Nitin Gupta Cc: Sergey Senozhatsky Cc: Jens Axboe Cc: David Hildenbrand Cc: [4.14+] Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds Signed-off-by: Greg Kroah-Hartman Signed-off-by: Minchan Kim Bug: 214353194 (cherry picked from commit 20ed94f8181a25212e7404e44958e234f407624b) git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git linux-5.10.y) Change-Id: I06552d47d3d71f97da49385fccdad80ae65dba26 --- mm/page_io.c | 54 ---------------------------------------------------- 1 file changed, 54 deletions(-) diff --git a/mm/page_io.c b/mm/page_io.c index f2900cf6c5b4..d5efe9558b8a 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -69,54 +69,6 @@ void end_swap_bio_write(struct bio *bio) bio_put(bio); } -static void swap_slot_free_notify(struct page *page) -{ - struct swap_info_struct *sis; - struct gendisk *disk; - swp_entry_t entry; - - /* - * There is no guarantee that the page is in swap cache - the software - * suspend code (at least) uses end_swap_bio_read() against a non- - * swapcache page. So we must check PG_swapcache before proceeding with - * this optimization. - */ - if (unlikely(!PageSwapCache(page))) - return; - - sis = page_swap_info(page); - if (data_race(!(sis->flags & SWP_BLKDEV))) - return; - - /* - * The swap subsystem performs lazy swap slot freeing, - * expecting that the page will be swapped out again. - * So we can avoid an unnecessary write if the page - * isn't redirtied. - * This is good for real swap storage because we can - * reduce unnecessary I/O and enhance wear-leveling - * if an SSD is used as the as swap device. - * But if in-memory swap device (eg zram) is used, - * this causes a duplicated copy between uncompressed - * data in VM-owned memory and compressed data in - * zram-owned memory. So let's free zram-owned memory - * and make the VM-owned decompressed page *dirty*, - * so the page should be swapped out somewhere again if - * we again wish to reclaim it. - */ - disk = sis->bdev->bd_disk; - entry.val = page_private(page); - if (disk->fops->swap_slot_free_notify && __swap_count(entry) == 1) { - unsigned long offset; - - offset = swp_offset(entry); - - SetPageDirty(page); - disk->fops->swap_slot_free_notify(sis->bdev, - offset); - } -} - static void end_swap_bio_read(struct bio *bio) { struct page *page = bio_first_page_all(bio); @@ -132,7 +84,6 @@ static void end_swap_bio_read(struct bio *bio) } SetPageUptodate(page); - swap_slot_free_notify(page); out: unlock_page(page); WRITE_ONCE(bio->bi_private, NULL); @@ -409,11 +360,6 @@ int swap_readpage(struct page *page, bool synchronous) if (sis->flags & SWP_SYNCHRONOUS_IO) { ret = bdev_read_page(sis->bdev, swap_page_sector(page), page); if (!ret) { - if (trylock_page(page)) { - swap_slot_free_notify(page); - unlock_page(page); - } - count_vm_event(PSWPIN); goto out; } From 1e037dd7909134ff72d3757bff4c9b38710494b4 Mon Sep 17 00:00:00 2001 From: Lina Wang Date: Thu, 5 May 2022 13:48:49 +0800 Subject: [PATCH 141/156] FROMGIT: net: fix wrong network header length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When clatd starts with ebpf offloaing, and NETIF_F_GRO_FRAGLIST is enable, several skbs are gathered in skb_shinfo(skb)->frag_list. The first skb's ipv6 header will be changed to ipv4 after bpf_skb_proto_6_to_4, network_header\transport_header\mac_header have been updated as ipv4 acts, but other skbs in frag_list didnot update anything, just ipv6 packets. udp_queue_rcv_skb will call skb_segment_list to traverse other skbs in frag_list and make sure right udp payload is delivered to user space. Unfortunately, other skbs in frag_list who are still ipv6 packets are updated like the first skb and will have wrong transport header length. e.g.before bpf_skb_proto_6_to_4,the first skb and other skbs in frag_list has the same network_header(24)& transport_header(64), after bpf_skb_proto_6_to_4, ipv6 protocol has been changed to ipv4, the first skb's network_header is 44,transport_header is 64, other skbs in frag_list didnot change.After skb_segment_list, the other skbs in frag_list has different network_header(24) and transport_header(44), so there will be 20 bytes different from original,that is difference between ipv6 header and ipv4 header. Just change transport_header to be the same with original. Actually, there are two solutions to fix it, one is traversing all skbs and changing every skb header in bpf_skb_proto_6_to_4, the other is modifying frag_list skb's header in skb_segment_list. Considering efficiency, adopt the second one--- when the first skb and other skbs in frag_list has different network_header length, restore them to make sure right udp payload is delivered to user space. Signed-off-by: Lina Wang Signed-off-by: David S. Miller (cherry picked from commit cf3ab8d4a797960b4be20565abb3bcd227b18a68 https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git master) Bug: 218157620 Test: TreeHugger Signed-off-by: Maciej Żenczykowski Change-Id: I36f2f329ec1a56bb0742141a7fa482cafa183ad3 --- net/core/skbuff.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 5bf27d8c5ea9..39dd750a74b1 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -3669,7 +3669,7 @@ struct sk_buff *skb_segment_list(struct sk_buff *skb, unsigned int delta_len = 0; struct sk_buff *tail = NULL; struct sk_buff *nskb, *tmp; - int err; + int len_diff, err; skb_push(skb, -skb_network_offset(skb) + offset); @@ -3709,9 +3709,11 @@ struct sk_buff *skb_segment_list(struct sk_buff *skb, skb_push(nskb, -skb_network_offset(nskb) + offset); skb_release_head_state(nskb); + len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb); __copy_skb_header(nskb, skb); skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb)); + nskb->transport_header += len_diff; skb_copy_from_linear_data_offset(skb, -tnl_hlen, nskb->data - tnl_hlen, offset + tnl_hlen); From 0d752f78b20dbe4eeb9bc76f118889f1898948ca Mon Sep 17 00:00:00 2001 From: Lee Jones Date: Mon, 9 May 2022 15:17:10 +0100 Subject: [PATCH 142/156] BACKPORT: staging: ion: Prevent incorrect reference counting behavour Supply additional check in order to prevent unexpected results. Bug: 205573273 Fixes: b892bf75b2034 ("ion: Switch ion to use dma-buf") Suggested-by: Dan Carpenter Signed-off-by: Lee Jones Signed-off-by: Greg Kroah-Hartman [Lee: Patch now applies to ion_buffer.c instead of ion.c] Change-Id: Ia6afdd9ca502caa9cad6619d438fc6c8e8457679 (cherry picked from commit fea24b07edfc348c67a019b6e17b39c0698e631f) --- drivers/staging/android/ion/ion_buffer.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/staging/android/ion/ion_buffer.c b/drivers/staging/android/ion/ion_buffer.c index 9baca1a472b6..9c7a353a4220 100644 --- a/drivers/staging/android/ion/ion_buffer.c +++ b/drivers/staging/android/ion/ion_buffer.c @@ -249,6 +249,9 @@ void *ion_buffer_kmap_get(struct ion_buffer *buffer) void *vaddr; if (buffer->kmap_cnt) { + if (buffer->kmap_cnt == INT_MAX) + return ERR_PTR(-EOVERFLOW); + buffer->kmap_cnt++; return buffer->vaddr; } From a07b60e85c4aa4af9f53552e1a923877eabfac90 Mon Sep 17 00:00:00 2001 From: Lee Jones Date: Fri, 11 Mar 2022 14:19:59 +0000 Subject: [PATCH 143/156] ANDROID: dm-bow: Protect Ranges fetched and erased from the RB tree Bug: 195565510 Signed-off-by: Lee Jones Change-Id: Ic8134eb902aa7d929e3121b2f69b1d258f570652 (cherry picked from commit 98c15b2bad1a277da43c65c642f8c3c3ee07bacc) --- drivers/md/dm-bow.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/md/dm-bow.c b/drivers/md/dm-bow.c index cfd1fa63ff97..2e95d6149692 100644 --- a/drivers/md/dm-bow.c +++ b/drivers/md/dm-bow.c @@ -599,6 +599,7 @@ static void dm_bow_dtr(struct dm_target *ti) struct bow_context *bc = (struct bow_context *) ti->private; struct kobject *kobj; + mutex_lock(&bc->ranges_lock); while (rb_first(&bc->ranges)) { struct bow_range *br = container_of(rb_first(&bc->ranges), struct bow_range, node); @@ -606,6 +607,8 @@ static void dm_bow_dtr(struct dm_target *ti) rb_erase(&br->node, &bc->ranges); kfree(br); } + mutex_unlock(&bc->ranges_lock); + if (bc->workqueue) destroy_workqueue(bc->workqueue); if (bc->bufio) @@ -1181,6 +1184,7 @@ static void dm_bow_tablestatus(struct dm_target *ti, char *result, return; } + mutex_lock(&bc->ranges_lock); for (i = rb_first(&bc->ranges); i; i = rb_next(i)) { struct bow_range *br = container_of(i, struct bow_range, node); @@ -1188,11 +1192,11 @@ static void dm_bow_tablestatus(struct dm_target *ti, char *result, readable_type[br->type], (unsigned long long)br->sector); if (result >= end) - return; + goto unlock; result += scnprintf(result, end - result, "\n"); if (result >= end) - return; + goto unlock; if (br->type == TRIMMED) ++trimmed_range_count; @@ -1214,19 +1218,22 @@ static void dm_bow_tablestatus(struct dm_target *ti, char *result, if (!rb_next(i)) { scnprintf(result, end - result, "\nERROR: Last range not of type TOP"); - return; + goto unlock; } if (br->sector > range_top(br)) { scnprintf(result, end - result, "\nERROR: sectors out of order"); - return; + goto unlock; } } if (trimmed_range_count != trimmed_list_length) scnprintf(result, end - result, "\nERROR: not all trimmed ranges in trimmed list"); + +unlock: + mutex_unlock(&bc->ranges_lock); } static void dm_bow_status(struct dm_target *ti, status_type_t type, From 3172c9558296d114ef2100dbe85fda70aac263eb Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Wed, 12 Jan 2022 12:14:58 -0800 Subject: [PATCH 144/156] FROMLIST: kbuild: Add environment variables for userprogs flags Allow additional arguments be passed to userprogs compilation. Reproducible clang builds need to provide a sysroot and gcc path to ensure the same toolchain is used across hosts. KCFLAGS is not currently used for any user programs compilation, so add new USERCFLAGS and USERLDFLAGS which serves similar purpose as HOSTCFLAGS/HOSTLDFLAGS. Clang 13+ might detect GCC installation on hosts which have it installed to a default location in /. With addition of these environment variables, you can specify flags such as: $ make USERCFLAGS=--sysroot=/path/to/sysroot This can also be used to specify different sysroots such as musl or bionic which may be installed on the host in paths that the compiler may not search by default. Signed-off-by: Elliot Berman Reviewed-by: Nick Desaulniers Bug: 214127735 Link: https://lore.kernel.org/linux-kbuild/20220201213542.2808035-1-quic_eberman@quicinc.com/ Change-Id: I035309a569130774a1212f294973eee2830a6588 (cherry picked from commit df704104322c6bd1b3d40cb7ad109cba7453f3be) --- Documentation/kbuild/kbuild.rst | 11 +++++++++++ Documentation/kbuild/makefiles.rst | 2 ++ Makefile | 10 ++++++---- init/Kconfig | 8 ++++---- usr/include/Makefile | 3 +++ 5 files changed, 26 insertions(+), 8 deletions(-) diff --git a/Documentation/kbuild/kbuild.rst b/Documentation/kbuild/kbuild.rst index 2d1fc03d346e..ef19b9c13523 100644 --- a/Documentation/kbuild/kbuild.rst +++ b/Documentation/kbuild/kbuild.rst @@ -77,6 +77,17 @@ HOSTLDLIBS ---------- Additional libraries to link against when building host programs. +.. _userkbuildflags: + +USERCFLAGS +---------- +Additional options used for $(CC) when compiling userprogs. + +USERLDFLAGS +----------- +Additional options used for $(LD) when linking userprogs. userprogs are linked +with CC, so $(USERLDFLAGS) should include "-Wl," prefix as applicable. + KBUILD_KCONFIG -------------- Set the top-level Kconfig file to the value of this environment diff --git a/Documentation/kbuild/makefiles.rst b/Documentation/kbuild/makefiles.rst index 0d5dd5413af0..1db205dc1d0f 100644 --- a/Documentation/kbuild/makefiles.rst +++ b/Documentation/kbuild/makefiles.rst @@ -852,6 +852,8 @@ The syntax is quite similar. The difference is to use "userprogs" instead of When linking bpfilter_umh, it will be passed the extra option -static. + From command line, :ref:`USERCFLAGS and USERLDFLAGS ` will also be used. + 5.4 When userspace programs are actually built ---------------------------------------------- diff --git a/Makefile b/Makefile index cfebae21df4b..3f3743799a06 100644 --- a/Makefile +++ b/Makefile @@ -429,11 +429,12 @@ HOSTCC = gcc HOSTCXX = g++ endif -export KBUILD_USERCFLAGS := -Wall -Wmissing-prototypes -Wstrict-prototypes \ - -O2 -fomit-frame-pointer -std=gnu89 -export KBUILD_USERLDFLAGS := +KBUILD_USERHOSTCFLAGS := -Wall -Wmissing-prototypes -Wstrict-prototypes \ + -O2 -fomit-frame-pointer -std=gnu89 +KBUILD_USERCFLAGS := $(KBUILD_USERHOSTCFLAGS) $(USERCFLAGS) +KBUILD_USERLDFLAGS := $(USERLDFLAGS) -KBUILD_HOSTCFLAGS := $(KBUILD_USERCFLAGS) $(HOST_LFS_CFLAGS) $(HOSTCFLAGS) +KBUILD_HOSTCFLAGS := $(KBUILD_USERHOSTCFLAGS) $(HOST_LFS_CFLAGS) $(HOSTCFLAGS) KBUILD_HOSTCXXFLAGS := -Wall -O2 $(HOST_LFS_CFLAGS) $(HOSTCXXFLAGS) KBUILD_HOSTLDFLAGS := $(HOST_LFS_LDFLAGS) $(HOSTLDFLAGS) KBUILD_HOSTLDLIBS := $(HOST_LFS_LIBS) $(HOSTLDLIBS) @@ -526,6 +527,7 @@ export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AW export PERL PYTHON PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE +export KBUILD_USERCFLAGS KBUILD_USERLDFLAGS export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE diff --git a/init/Kconfig b/init/Kconfig index c08acd288f9f..ee9b2396972d 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -53,13 +53,13 @@ config LLD_VERSION config CC_CAN_LINK bool - default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(m64-flag)) if 64BIT - default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(m32-flag)) + default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(m64-flag)) if 64BIT + default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(m32-flag)) config CC_CAN_LINK_STATIC bool - default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(m64-flag) -static) if 64BIT - default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(m32-flag) -static) + default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(m64-flag) -static) if 64BIT + default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(m32-flag) -static) config CC_HAS_ASM_GOTO def_bool $(success,$(srctree)/scripts/gcc-goto.sh $(CC)) diff --git a/usr/include/Makefile b/usr/include/Makefile index 703a255cddc6..a8ed689918ce 100644 --- a/usr/include/Makefile +++ b/usr/include/Makefile @@ -12,6 +12,9 @@ UAPI_CFLAGS := -std=c90 -Wall -Werror=implicit-function-declaration # It is here just because CONFIG_CC_CAN_LINK is tested with -m32 or -m64. UAPI_CFLAGS += $(filter -m32 -m64, $(KBUILD_CFLAGS)) +# USERCFLAGS might contain sysroot location for CC. +UAPI_CFLAGS += $(USERCFLAGS) + override c_flags = $(UAPI_CFLAGS) -Wp,-MMD,$(depfile) -I$(objtree)/usr/include # The following are excluded for now because they fail to build. From 15a663ebbe05fd511480a1719414af71073cb3a0 Mon Sep 17 00:00:00 2001 From: Qihang Hu Date: Wed, 10 Nov 2021 18:11:29 +0800 Subject: [PATCH 145/156] UPSTREAM: usb: gadget: composite: Show warning if function driver's descriptors are incomplete. In the config_ep_by_speed_and_alt function, select the corresponding descriptor through g->speed. But some legacy or not well designed function drivers may not support the corresponding speed. So, we can directly display warnings instead of causing kernel panic. At the same time, it indicates the reasons in warning message. Reviewed-by: Peter Chen Signed-off-by: Qihang Hu Link: https://lore.kernel.org/r/20211110101129.462357-1-huqihang@oppo.com Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit 16d42759207fc3d1bff7cfd330a08a225e470ba0) Signed-off-by: Dan Vacura Change-Id: Iaef2ecfc080dc4a97fe9911af4da78316af8e063 --- drivers/usb/gadget/composite.c | 39 ++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c index 553382ce3837..9315313108c9 100644 --- a/drivers/usb/gadget/composite.c +++ b/drivers/usb/gadget/composite.c @@ -159,6 +159,8 @@ int config_ep_by_speed_and_alt(struct usb_gadget *g, int want_comp_desc = 0; struct usb_descriptor_header **d_spd; /* cursor for speed desc */ + struct usb_composite_dev *cdev; + bool incomplete_desc = false; if (!g || !f || !_ep) return -EIO; @@ -167,28 +169,43 @@ int config_ep_by_speed_and_alt(struct usb_gadget *g, switch (g->speed) { case USB_SPEED_SUPER_PLUS: if (gadget_is_superspeed_plus(g)) { - speed_desc = f->ssp_descriptors; - want_comp_desc = 1; - break; + if (f->ssp_descriptors) { + speed_desc = f->ssp_descriptors; + want_comp_desc = 1; + break; + } + incomplete_desc = true; } fallthrough; case USB_SPEED_SUPER: if (gadget_is_superspeed(g)) { - speed_desc = f->ss_descriptors; - want_comp_desc = 1; - break; + if (f->ss_descriptors) { + speed_desc = f->ss_descriptors; + want_comp_desc = 1; + break; + } + incomplete_desc = true; } fallthrough; case USB_SPEED_HIGH: if (gadget_is_dualspeed(g)) { - speed_desc = f->hs_descriptors; - break; + if (f->hs_descriptors) { + speed_desc = f->hs_descriptors; + break; + } + incomplete_desc = true; } fallthrough; default: speed_desc = f->fs_descriptors; } + cdev = get_gadget_data(g); + if (incomplete_desc) + WARNING(cdev, + "%s doesn't hold the descriptors for current speed\n", + f->name); + /* find correct alternate setting descriptor */ for_each_desc(speed_desc, d_spd, USB_DT_INTERFACE) { int_desc = (struct usb_interface_descriptor *)*d_spd; @@ -244,12 +261,8 @@ ep_found: _ep->maxburst = comp_desc->bMaxBurst + 1; break; default: - if (comp_desc->bMaxBurst != 0) { - struct usb_composite_dev *cdev; - - cdev = get_gadget_data(g); + if (comp_desc->bMaxBurst != 0) ERROR(cdev, "ep0 bMaxBurst must be 0\n"); - } _ep->maxburst = 1; break; } From 5d084e9710a02d7cf4f6c667f6fe84acdbb286ef Mon Sep 17 00:00:00 2001 From: Michael Grzeschik Date: Mon, 28 Jun 2021 17:53:08 +0200 Subject: [PATCH 146/156] UPSTREAM: usb: gadget: uvc: make uvc_num_requests depend on gadget speed While sending bigger images is possible with USB_SPEED_SUPER it is better to use more isochronous requests in flight. This patch makes the number uvc_num_requests dynamic by changing it depending on the gadget speed. Reviewed-by: Paul Elder Reviewed-by: Laurent Pinchart Signed-off-by: Michael Grzeschik Link: https://lore.kernel.org/r/20210628155311.16762-3-m.grzeschik@pengutronix.de Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit 9973772dbb2b9c12a0707eca692f0dabf6295978) Signed-off-by: Dan Vacura Change-Id: I1385bf8f973f3e5ee6c76193bffe0046eca22439 --- drivers/usb/gadget/function/uvc.h | 11 +++-- drivers/usb/gadget/function/uvc_queue.c | 6 +++ drivers/usb/gadget/function/uvc_video.c | 55 +++++++++++++++---------- 3 files changed, 47 insertions(+), 25 deletions(-) diff --git a/drivers/usb/gadget/function/uvc.h b/drivers/usb/gadget/function/uvc.h index 893aaa70f81a..e60049ec9824 100644 --- a/drivers/usb/gadget/function/uvc.h +++ b/drivers/usb/gadget/function/uvc.h @@ -65,13 +65,17 @@ extern unsigned int uvc_gadget_trace_param; * Driver specific constants */ -#define UVC_NUM_REQUESTS 4 #define UVC_MAX_REQUEST_SIZE 64 #define UVC_MAX_EVENTS 4 /* ------------------------------------------------------------------------ * Structures */ +struct uvc_request { + struct usb_request *req; + u8 *req_buffer; + struct uvc_video *video; +}; struct uvc_video { struct uvc_device *uvc; @@ -87,10 +91,11 @@ struct uvc_video { unsigned int imagesize; struct mutex mutex; /* protects frame parameters */ + unsigned int uvc_num_requests; + /* Requests */ unsigned int req_size; - struct usb_request *req[UVC_NUM_REQUESTS]; - __u8 *req_buffer[UVC_NUM_REQUESTS]; + struct uvc_request *ureq; struct list_head req_free; spinlock_t req_lock; diff --git a/drivers/usb/gadget/function/uvc_queue.c b/drivers/usb/gadget/function/uvc_queue.c index 61e2c94cc0b0..ff0cc08531d2 100644 --- a/drivers/usb/gadget/function/uvc_queue.c +++ b/drivers/usb/gadget/function/uvc_queue.c @@ -43,6 +43,7 @@ static int uvc_queue_setup(struct vb2_queue *vq, { struct uvc_video_queue *queue = vb2_get_drv_priv(vq); struct uvc_video *video = container_of(queue, struct uvc_video, queue); + struct usb_composite_dev *cdev = video->uvc->func.config->cdev; if (*nbuffers > UVC_MAX_VIDEO_BUFFERS) *nbuffers = UVC_MAX_VIDEO_BUFFERS; @@ -51,6 +52,11 @@ static int uvc_queue_setup(struct vb2_queue *vq, sizes[0] = video->imagesize; + if (cdev->gadget->speed < USB_SPEED_SUPER) + video->uvc_num_requests = 4; + else + video->uvc_num_requests = 64; + return 0; } diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index 633e23d58d86..303cb427f687 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -145,7 +145,8 @@ static int uvcg_video_ep_queue(struct uvc_video *video, struct usb_request *req) static void uvc_video_complete(struct usb_ep *ep, struct usb_request *req) { - struct uvc_video *video = req->context; + struct uvc_request *ureq = req->context; + struct uvc_video *video = ureq->video; struct uvc_video_queue *queue = &video->queue; unsigned long flags; @@ -177,16 +178,21 @@ uvc_video_free_requests(struct uvc_video *video) { unsigned int i; - for (i = 0; i < UVC_NUM_REQUESTS; ++i) { - if (video->req[i]) { - usb_ep_free_request(video->ep, video->req[i]); - video->req[i] = NULL; + if (video->ureq) { + for (i = 0; i < video->uvc_num_requests; ++i) { + if (video->ureq[i].req) { + usb_ep_free_request(video->ep, video->ureq[i].req); + video->ureq[i].req = NULL; + } + + if (video->ureq[i].req_buffer) { + kfree(video->ureq[i].req_buffer); + video->ureq[i].req_buffer = NULL; + } } - if (video->req_buffer[i]) { - kfree(video->req_buffer[i]); - video->req_buffer[i] = NULL; - } + kfree(video->ureq); + video->ureq = NULL; } INIT_LIST_HEAD(&video->req_free); @@ -207,21 +213,26 @@ uvc_video_alloc_requests(struct uvc_video *video) * max_t(unsigned int, video->ep->maxburst, 1) * (video->ep->mult); - for (i = 0; i < UVC_NUM_REQUESTS; ++i) { - video->req_buffer[i] = kmalloc(req_size, GFP_KERNEL); - if (video->req_buffer[i] == NULL) + video->ureq = kcalloc(video->uvc_num_requests, sizeof(struct uvc_request), GFP_KERNEL); + if (video->ureq == NULL) + return -ENOMEM; + + for (i = 0; i < video->uvc_num_requests; ++i) { + video->ureq[i].req_buffer = kmalloc(req_size, GFP_KERNEL); + if (video->ureq[i].req_buffer == NULL) goto error; - video->req[i] = usb_ep_alloc_request(video->ep, GFP_KERNEL); - if (video->req[i] == NULL) + video->ureq[i].req = usb_ep_alloc_request(video->ep, GFP_KERNEL); + if (video->ureq[i].req == NULL) goto error; - video->req[i]->buf = video->req_buffer[i]; - video->req[i]->length = 0; - video->req[i]->complete = uvc_video_complete; - video->req[i]->context = video; + video->ureq[i].req->buf = video->ureq[i].req_buffer; + video->ureq[i].req->length = 0; + video->ureq[i].req->complete = uvc_video_complete; + video->ureq[i].req->context = &video->ureq[i]; + video->ureq[i].video = video; - list_add_tail(&video->req[i]->list, &video->req_free); + list_add_tail(&video->ureq[i].req->list, &video->req_free); } video->req_size = req_size; @@ -312,9 +323,9 @@ int uvcg_video_enable(struct uvc_video *video, int enable) cancel_work_sync(&video->pump); uvcg_queue_cancel(&video->queue, 0); - for (i = 0; i < UVC_NUM_REQUESTS; ++i) - if (video->req[i]) - usb_ep_dequeue(video->ep, video->req[i]); + for (i = 0; i < video->uvc_num_requests; ++i) + if (video->ureq && video->ureq[i].req) + usb_ep_dequeue(video->ep, video->ureq[i].req); uvc_video_free_requests(video); uvcg_queue_enable(&video->queue, 0); From 1c14550b60b2ddd15612490fcf800fe375a2c407 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Mon, 7 Mar 2022 18:59:56 -0800 Subject: [PATCH 147/156] UPSTREAM: usb: dwc3: gadget: Give some time to schedule isoc Currently the driver will schedule isoc transfers immediately on the next interval, which is quite aggressive when the interval is 125us. There's report that some platforms may need more time to process the transfer, otherwise the controller may miss the first interval. Let's keep it simple and give the controller at least 500us to schedule the isoc transfer. Link: https://lore.kernel.org/linux-usb/20220302143539.GI11577@pengutronix.de/ Tested-by: Michael Grzeschik Signed-off-by: Thinh Nguyen Link: https://lore.kernel.org/r/deb8146b8e1f7f8495ef2d5647017270934cb2d8.1646708142.git.Thinh.Nguyen@synopsys.com Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit aa6812be1feb75139b9bc263915b3f1ceeb4e154) Signed-off-by: Dan Vacura Change-Id: I3bbf5abfdb249fb9a10c664f11d4f3b682ceafd1 --- drivers/usb/dwc3/gadget.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index fe10c72ad1e3..03a59253d088 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -1805,7 +1805,13 @@ static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep) } for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) { - dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1); + int future_interval = i + 1; + + /* Give the controller at least 500us to schedule transfers */ + if (desc->bInterval < 3) + future_interval += 3 - desc->bInterval; + + dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval); ret = __dwc3_gadget_kick_transfer(dep); if (ret != -EAGAIN) From 6692312df6dc9254e107e12c352bf793382b2361 Mon Sep 17 00:00:00 2001 From: Michael Grzeschik Date: Sun, 17 Oct 2021 23:50:15 +0200 Subject: [PATCH 148/156] UPSTREAM: usb: gadget: uvc: only schedule stream in streaming state This patch ensures that the video pump thread will only be scheduled if the uvc is really in streaming state. This way the worker will not have to run on an empty queue. Reviewed-by: Laurent Pinchart Reviewed-by: Paul Elder Signed-off-by: Michael Grzeschik Link: https://lore.kernel.org/r/20211017215017.18392-5-m.grzeschik@pengutronix.de Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit 5fc49d8bee73648a706f5892663f5bd728ab07ea) Signed-off-by: Dan Vacura Change-Id: I5724e6c41a4b298f42d4009bdfe2f0232ae16517 --- drivers/usb/gadget/function/uvc_v4l2.c | 3 ++- drivers/usb/gadget/function/uvc_video.c | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/usb/gadget/function/uvc_v4l2.c b/drivers/usb/gadget/function/uvc_v4l2.c index 197c26f7aec6..a2c78690c5c2 100644 --- a/drivers/usb/gadget/function/uvc_v4l2.c +++ b/drivers/usb/gadget/function/uvc_v4l2.c @@ -169,7 +169,8 @@ uvc_v4l2_qbuf(struct file *file, void *fh, struct v4l2_buffer *b) if (ret < 0) return ret; - schedule_work(&video->pump); + if (uvc->state == UVC_STATE_STREAMING) + schedule_work(&video->pump); return ret; } diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index 303cb427f687..2e2b23915084 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -148,6 +148,7 @@ uvc_video_complete(struct usb_ep *ep, struct usb_request *req) struct uvc_request *ureq = req->context; struct uvc_video *video = ureq->video; struct uvc_video_queue *queue = &video->queue; + struct uvc_device *uvc = video->uvc; unsigned long flags; switch (req->status) { @@ -170,7 +171,8 @@ uvc_video_complete(struct usb_ep *ep, struct usb_request *req) list_add_tail(&req->list, &video->req_free); spin_unlock_irqrestore(&video->req_lock, flags); - schedule_work(&video->pump); + if (uvc->state == UVC_STATE_STREAMING) + schedule_work(&video->pump); } static int From 23cca7ad30b4896923c54645b03f49813d336640 Mon Sep 17 00:00:00 2001 From: Michael Grzeschik Date: Sun, 17 Oct 2021 23:50:16 +0200 Subject: [PATCH 149/156] UPSTREAM: usb: gadget: uvc: only pump video data if necessary If the streaming endpoint is not enabled, the worker has nothing to do. In the case buffers are still queued, this patch ensures that it will bail out without handling any data. Reviewed-by: Laurent Pinchart Reviewed-by: Paul Elder Signed-off-by: Michael Grzeschik Link: https://lore.kernel.org/r/20211017215017.18392-6-m.grzeschik@pengutronix.de Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit f9897ec0f6d34e8b2bc2f4c8ab8789351090f3d2) Signed-off-by: Dan Vacura Change-Id: Ice201abf1f4e4f38ea2b80a990896597e365e74b --- drivers/usb/gadget/function/uvc_video.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index 2e2b23915084..51756c4a07b6 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -260,12 +260,12 @@ static void uvcg_video_pump(struct work_struct *work) { struct uvc_video *video = container_of(work, struct uvc_video, pump); struct uvc_video_queue *queue = &video->queue; - struct usb_request *req; + struct usb_request *req = NULL; struct uvc_buffer *buf; unsigned long flags; int ret; - while (1) { + while (video->ep->enabled) { /* Retrieve the first available USB request, protected by the * request lock. */ @@ -301,6 +301,9 @@ static void uvcg_video_pump(struct work_struct *work) } } + if (!req) + return; + spin_lock_irqsave(&video->req_lock, flags); list_add_tail(&req->list, &video->req_free); spin_unlock_irqrestore(&video->req_lock, flags); From 53129aceec8c40e7db9d984f14c99c1272597863 Mon Sep 17 00:00:00 2001 From: Michael Grzeschik Date: Sun, 17 Oct 2021 23:50:14 +0200 Subject: [PATCH 150/156] UPSTREAM: usb: gadget: uvc: test if ep->desc is valid on ep_queue The reason that the ep_queue has failed could be a disabled endpoint. In that case it is not guaranteed that the ep->desc is still valid. This patch adds a check for NULL. Reviewed-by: Laurent Pinchart Reviewed-by: Paul Elder Signed-off-by: Michael Grzeschik Link: https://lore.kernel.org/r/20211017215017.18392-4-m.grzeschik@pengutronix.de Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit 38db3716a5f8f022ae38f7431913e6b479015b74) Signed-off-by: Dan Vacura Change-Id: I8a17f64088832947e3a4481140d884881bf603ff --- drivers/usb/gadget/function/uvc_video.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index 51756c4a07b6..e5e2c2bae3b9 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -134,9 +134,12 @@ static int uvcg_video_ep_queue(struct uvc_video *video, struct usb_request *req) uvcg_err(&video->uvc->func, "Failed to queue request (%d).\n", ret); - /* Isochronous endpoints can't be halted. */ - if (usb_endpoint_xfer_bulk(video->ep->desc)) - usb_ep_set_halt(video->ep); + /* If the endpoint is disabled the descriptor may be NULL. */ + if (video->ep->desc) { + /* Isochronous endpoints can't be halted. */ + if (usb_endpoint_xfer_bulk(video->ep->desc)) + usb_ep_set_halt(video->ep); + } } return ret; From 74b55107d082630f443d0d1485c9ea619bde54f8 Mon Sep 17 00:00:00 2001 From: Dan Vacura Date: Thu, 31 Mar 2022 13:40:23 -0500 Subject: [PATCH 151/156] UPSTREAM: usb: gadget: uvc: Fix crash when encoding data for usb request During the uvcg_video_pump() process, if an error occurs and uvcg_queue_cancel() is called, the buffer queue will be cleared out, but the current marker (queue->buf_used) of the active buffer (no longer active) is not reset. On the next iteration of uvcg_video_pump() the stale buf_used count will be used and the logic of min((unsigned int)len, buf->bytesused - queue->buf_used) may incorrectly calculate a nbytes size, causing an invalid memory access. [80802.185460][ T315] configfs-gadget gadget: uvc: VS request completed with status -18. [80802.185519][ T315] configfs-gadget gadget: uvc: VS request completed with status -18. ... uvcg_queue_cancel() is called and the queue is cleared out, but the marker queue->buf_used is not reset. ... [80802.262328][ T8682] Unable to handle kernel paging request at virtual address ffffffc03af9f000 ... ... [80802.263138][ T8682] Call trace: [80802.263146][ T8682] __memcpy+0x12c/0x180 [80802.263155][ T8682] uvcg_video_pump+0xcc/0x1e0 [80802.263165][ T8682] process_one_work+0x2cc/0x568 [80802.263173][ T8682] worker_thread+0x28c/0x518 [80802.263181][ T8682] kthread+0x160/0x170 [80802.263188][ T8682] ret_from_fork+0x10/0x18 [80802.263198][ T8682] Code: a8c12829 a88130cb a8c130 Fixes: d692522577c0 ("usb: gadget/uvc: Port UVC webcam gadget to use videobuf2 framework") Cc: Signed-off-by: Dan Vacura Link: https://lore.kernel.org/r/20220331184024.23918-1-w36195@motorola.com Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit 71d471e3faf90c9674cadc7605ac719e82cb7fac) Change-Id: I0e801793e56537177b0ff08dd8a5460d98c9614a Signed-off-by: Dan Vacura --- drivers/usb/gadget/function/uvc_queue.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/usb/gadget/function/uvc_queue.c b/drivers/usb/gadget/function/uvc_queue.c index ff0cc08531d2..49d0e8b430a2 100644 --- a/drivers/usb/gadget/function/uvc_queue.c +++ b/drivers/usb/gadget/function/uvc_queue.c @@ -248,6 +248,8 @@ void uvcg_queue_cancel(struct uvc_video_queue *queue, int disconnect) buf->state = UVC_BUF_STATE_ERROR; vb2_buffer_done(&buf->buf.vb2_buf, VB2_BUF_STATE_ERROR); } + queue->buf_used = 0; + /* This must be protected by the irqlock spinlock to avoid race * conditions between uvc_queue_buffer and the disconnection event that * could result in an interruptible wait in uvc_dequeue_buffer. Do not From 36aa07174b8344d8ada3005d425f05b92c8e285d Mon Sep 17 00:00:00 2001 From: Dan Vacura Date: Fri, 1 Apr 2022 11:04:45 -0500 Subject: [PATCH 152/156] FROMGIT: usb: gadget: uvc: allow changing interface name via configfs Add a configfs entry, "function_name", to change the iInterface field for VideoControl. This name is used on host devices for user selection, useful when multiple cameras are present. The default will remain "UVC Camera". Signed-off-by: Dan Vacura Link: https://lore.kernel.org/r/20220401160447.5919-1-w36195@motorola.com Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit 324e4f85070f89b58c5f9926370ad19dea907dc7 https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb-next) Change-Id: I6943d40dbc0dec125464b66684ae31252e807d9b Signed-off-by: Dan Vacura --- .../ABI/testing/configfs-usb-gadget-uvc | 1 + Documentation/usb/gadget-testing.rst | 1 + drivers/usb/gadget/function/f_uvc.c | 4 +- drivers/usb/gadget/function/u_uvc.h | 1 + drivers/usb/gadget/function/uvc_configfs.c | 41 +++++++++++++++++++ 5 files changed, 47 insertions(+), 1 deletion(-) diff --git a/Documentation/ABI/testing/configfs-usb-gadget-uvc b/Documentation/ABI/testing/configfs-usb-gadget-uvc index ac5e11af79a8..a87724ecbf65 100644 --- a/Documentation/ABI/testing/configfs-usb-gadget-uvc +++ b/Documentation/ABI/testing/configfs-usb-gadget-uvc @@ -7,6 +7,7 @@ Description: UVC function directory streaming_maxburst 0..15 (ss only) streaming_maxpacket 1..1023 (fs), 1..3072 (hs/ss) streaming_interval 1..16 + function_name string [32] =================== ============================= What: /config/usb-gadget/gadget/functions/uvc.name/control diff --git a/Documentation/usb/gadget-testing.rst b/Documentation/usb/gadget-testing.rst index 9d6276f82774..be1b085a0901 100644 --- a/Documentation/usb/gadget-testing.rst +++ b/Documentation/usb/gadget-testing.rst @@ -774,6 +774,7 @@ The uvc function provides these attributes in its function directory: streaming_maxpacket maximum packet size this endpoint is capable of sending or receiving when this configuration is selected + function_name name of the interface =================== ================================================ There are also "control" and "streaming" subdirectories, each of which contain diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c index f48a00e49794..2f691abf9def 100644 --- a/drivers/usb/gadget/function/f_uvc.c +++ b/drivers/usb/gadget/function/f_uvc.c @@ -44,7 +44,7 @@ MODULE_PARM_DESC(trace, "Trace level bitmask"); #define UVC_STRING_STREAMING_IDX 1 static struct usb_string uvc_en_us_strings[] = { - [UVC_STRING_CONTROL_IDX].s = "UVC Camera", + /* [UVC_STRING_CONTROL_IDX].s = DYNAMIC, */ [UVC_STRING_STREAMING_IDX].s = "Video Streaming", { } }; @@ -674,6 +674,7 @@ uvc_function_bind(struct usb_configuration *c, struct usb_function *f) uvc_hs_streaming_ep.bEndpointAddress = uvc->video.ep->address; uvc_ss_streaming_ep.bEndpointAddress = uvc->video.ep->address; + uvc_en_us_strings[UVC_STRING_CONTROL_IDX].s = opts->function_name; us = usb_gstrings_attach(cdev, uvc_function_strings, ARRAY_SIZE(uvc_en_us_strings)); if (IS_ERR(us)) { @@ -864,6 +865,7 @@ static struct usb_function_instance *uvc_alloc_inst(void) opts->streaming_interval = 1; opts->streaming_maxpacket = 1024; + snprintf(opts->function_name, sizeof(opts->function_name), "UVC Camera"); ret = uvcg_attach_configfs(opts); if (ret < 0) { diff --git a/drivers/usb/gadget/function/u_uvc.h b/drivers/usb/gadget/function/u_uvc.h index 9a01a7d4f17f..24b8681b0d6f 100644 --- a/drivers/usb/gadget/function/u_uvc.h +++ b/drivers/usb/gadget/function/u_uvc.h @@ -27,6 +27,7 @@ struct f_uvc_opts { unsigned int control_interface; unsigned int streaming_interface; + char function_name[32]; /* * Control descriptors array pointers for full-/high-speed and diff --git a/drivers/usb/gadget/function/uvc_configfs.c b/drivers/usb/gadget/function/uvc_configfs.c index 00fb58e50a15..9fafa8001d33 100644 --- a/drivers/usb/gadget/function/uvc_configfs.c +++ b/drivers/usb/gadget/function/uvc_configfs.c @@ -2430,10 +2430,51 @@ UVCG_OPTS_ATTR(streaming_maxburst, streaming_maxburst, 15); #undef UVCG_OPTS_ATTR +#define UVCG_OPTS_STRING_ATTR(cname, aname) \ +static ssize_t f_uvc_opts_string_##cname##_show(struct config_item *item,\ + char *page) \ +{ \ + struct f_uvc_opts *opts = to_f_uvc_opts(item); \ + int result; \ + \ + mutex_lock(&opts->lock); \ + result = snprintf(page, sizeof(opts->aname), "%s", opts->aname);\ + mutex_unlock(&opts->lock); \ + \ + return result; \ +} \ + \ +static ssize_t f_uvc_opts_string_##cname##_store(struct config_item *item,\ + const char *page, size_t len) \ +{ \ + struct f_uvc_opts *opts = to_f_uvc_opts(item); \ + int ret = 0; \ + \ + mutex_lock(&opts->lock); \ + if (opts->refcnt) { \ + ret = -EBUSY; \ + goto end; \ + } \ + \ + ret = snprintf(opts->aname, min(sizeof(opts->aname), len), \ + "%s", page); \ + \ +end: \ + mutex_unlock(&opts->lock); \ + return ret; \ +} \ + \ +UVC_ATTR(f_uvc_opts_string_, cname, aname) + +UVCG_OPTS_STRING_ATTR(function_name, function_name); + +#undef UVCG_OPTS_STRING_ATTR + static struct configfs_attribute *uvc_attrs[] = { &f_uvc_opts_attr_streaming_interval, &f_uvc_opts_attr_streaming_maxpacket, &f_uvc_opts_attr_streaming_maxburst, + &f_uvc_opts_string_attr_function_name, NULL, }; From a35bc53848a4e8497f5791085e8c820e2f75d2d0 Mon Sep 17 00:00:00 2001 From: Dan Vacura Date: Wed, 13 Apr 2022 14:24:02 -0500 Subject: [PATCH 153/156] FROMGIT: usb: gadget: uvc: remove pause flag use The pause flag doesn't appear to serve a purpose as it is possible for the pump worker thread to not be running when new buffers are queued. Before the following change was introduced: "43cd0023872e usb: gadget: uvc_video: add worker to handle the frame pumping", the pause flag status was discarded and never returned. This masked a current issues of the userspace receiving non-zero values (considered unsuccessful) and prevents trace_v4l2_qbuf() from being called. Signed-off-by: Dan Vacura Link: https://lore.kernel.org/r/20220413192402.26063-1-w36195@motorola.com Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit be30e3502771ac0b5080ec61223d915354cde1cc https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb-next) Change-Id: Id841e7f0dc6c79f74ca41bb93f2504effc2fefd8 Signed-off-by: Dan Vacura --- drivers/usb/gadget/function/uvc_queue.c | 15 +-------------- drivers/usb/gadget/function/uvc_queue.h | 1 - 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/drivers/usb/gadget/function/uvc_queue.c b/drivers/usb/gadget/function/uvc_queue.c index 49d0e8b430a2..51c83eea8318 100644 --- a/drivers/usb/gadget/function/uvc_queue.c +++ b/drivers/usb/gadget/function/uvc_queue.c @@ -169,18 +169,7 @@ int uvcg_query_buffer(struct uvc_video_queue *queue, struct v4l2_buffer *buf) int uvcg_queue_buffer(struct uvc_video_queue *queue, struct v4l2_buffer *buf) { - unsigned long flags; - int ret; - - ret = vb2_qbuf(&queue->queue, NULL, buf); - if (ret < 0) - return ret; - - spin_lock_irqsave(&queue->irqlock, flags); - ret = (queue->flags & UVC_QUEUE_PAUSED) != 0; - queue->flags &= ~UVC_QUEUE_PAUSED; - spin_unlock_irqrestore(&queue->irqlock, flags); - return ret; + return vb2_qbuf(&queue->queue, NULL, buf); } /* @@ -348,8 +337,6 @@ struct uvc_buffer *uvcg_queue_head(struct uvc_video_queue *queue) if (!list_empty(&queue->irqqueue)) buf = list_first_entry(&queue->irqqueue, struct uvc_buffer, queue); - else - queue->flags |= UVC_QUEUE_PAUSED; return buf; } diff --git a/drivers/usb/gadget/function/uvc_queue.h b/drivers/usb/gadget/function/uvc_queue.h index 2f0fff769843..4338ff871ee6 100644 --- a/drivers/usb/gadget/function/uvc_queue.h +++ b/drivers/usb/gadget/function/uvc_queue.h @@ -40,7 +40,6 @@ struct uvc_buffer { #define UVC_QUEUE_DISCONNECTED (1 << 0) #define UVC_QUEUE_DROP_INCOMPLETE (1 << 1) -#define UVC_QUEUE_PAUSED (1 << 2) struct uvc_video_queue { struct vb2_queue queue; From 300cec35029fbd1a45376c6773af825fc36ec877 Mon Sep 17 00:00:00 2001 From: Vijayavardhan Vennapusa Date: Wed, 13 Apr 2022 16:10:38 -0500 Subject: [PATCH 154/156] UPSTREAM: usb: gadget: configfs: clear deactivation flag in configfs_composite_unbind() If any function like UVC is deactivating gadget as part of composition switch which results in not calling pullup enablement, it is not getting enabled after switch to new composition due to this deactivation flag not cleared. This results in USB enumeration not happening after switch to new USB composition. Hence clear deactivation flag inside gadget structure in configfs_composite_unbind() before switch to new USB composition. Signed-off-by: Vijayavardhan Vennapusa Signed-off-by: Dan Vacura Cc: stable Link: https://lore.kernel.org/r/20220413211038.72797-1-w36195@motorola.com Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit bf95c4d4630c7a2c16e7b424fdea5177d9ce0864) Change-Id: I0013ab8443e47788291951516baec3e7c8e445aa Signed-off-by: Dan Vacura --- drivers/usb/gadget/configfs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index 2d7b7cd6d3a6..21dd268175c6 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -1530,6 +1530,8 @@ static void configfs_composite_unbind(struct usb_gadget *gadget) usb_ep_autoconfig_reset(cdev->gadget); spin_lock_irqsave(&gi->spinlock, flags); cdev->gadget = NULL; + cdev->deactivations = 0; + gadget->deactivated = false; set_gadget_data(gadget, NULL); spin_unlock_irqrestore(&gi->spinlock, flags); } From 3736ae703bb967d6f3bf9186e13e644685e5ff9e Mon Sep 17 00:00:00 2001 From: Dan Vacura Date: Wed, 20 Apr 2022 13:33:38 -0500 Subject: [PATCH 155/156] FROMGIT: usb: dwc3: gadget: increase tx fifo size for ss isoc endpoints Improve performance of isoc transfers in superspeed by increasing the number of possible endpoints available, matching that of what was configured for bulk transfers. Signed-off-by: Dan Vacura Link: https://lore.kernel.org/r/20220420183338.445622-1-w36195@motorola.com Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit 6a7c7df993bd3c90014453ec59a7b455613cf0e4 https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb-next) Change-Id: Iaa105cd98b0f4de3450bb5f6b10df67ed20a8336 Signed-off-by: Dan Vacura --- drivers/usb/dwc3/gadget.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 03a59253d088..74d6c30c3b54 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -769,7 +769,8 @@ static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep) num_fifos = 3; if (dep->endpoint.maxburst > 6 && - usb_endpoint_xfer_bulk(dep->endpoint.desc) && DWC3_IP_IS(DWC31)) + (usb_endpoint_xfer_bulk(dep->endpoint.desc) || + usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31)) num_fifos = dwc->tx_fifo_resize_max_num; /* FIFO size for a single buffer */ From 503435dc8cc3a13110a30a32a2ce1a2b2292f017 Mon Sep 17 00:00:00 2001 From: Dan Vacura Date: Tue, 3 May 2022 15:10:38 -0500 Subject: [PATCH 156/156] FROMGIT: usb: gadget: uvc: allow for application to cleanly shutdown Several types of kernel panics can occur due to timing during the uvc gadget removal. This appears to be a problem with gadget resources being managed by both the client application's v4l2 open/close and the UDC gadget bind/unbind. Since the concept of USB_GADGET_DELAYED_STATUS doesn't exist for unbind, add a wait to allow for the application to close out. Some examples of the panics that can occur are: <1>[ 1147.652313] Unable to handle kernel NULL pointer dereference at virtual address 0000000000000028 <4>[ 1147.652510] Call trace: <4>[ 1147.652514] usb_gadget_disconnect+0x74/0x1f0 <4>[ 1147.652516] usb_gadget_deactivate+0x38/0x168 <4>[ 1147.652520] usb_function_deactivate+0x54/0x90 <4>[ 1147.652524] uvc_function_disconnect+0x14/0x38 <4>[ 1147.652527] uvc_v4l2_release+0x34/0xa0 <4>[ 1147.652537] __fput+0xdc/0x2c0 <4>[ 1147.652540] ____fput+0x10/0x1c <4>[ 1147.652545] task_work_run+0xe4/0x12c <4>[ 1147.652549] do_notify_resume+0x108/0x168 <1>[ 282.950561][ T1472] Unable to handle kernel NULL pointer dereference at virtual address 00000000000005b8 <6>[ 282.953111][ T1472] Call trace: <6>[ 282.953121][ T1472] usb_function_deactivate+0x54/0xd4 <6>[ 282.953134][ T1472] uvc_v4l2_release+0xac/0x1e4 <6>[ 282.953145][ T1472] v4l2_release+0x134/0x1f0 <6>[ 282.953167][ T1472] __fput+0xf4/0x428 <6>[ 282.953178][ T1472] ____fput+0x14/0x24 <6>[ 282.953193][ T1472] task_work_run+0xac/0x130 <3>[ 213.410077][ T29] configfs-gadget gadget: uvc: Failed to queue request (-108). <1>[ 213.410116][ T29] Unable to handle kernel NULL pointer dereference at virtual address 0000000000000003 <6>[ 213.413460][ T29] Call trace: <6>[ 213.413474][ T29] uvcg_video_pump+0x1f0/0x384 <6>[ 213.413489][ T29] process_one_work+0x2a4/0x544 <6>[ 213.413502][ T29] worker_thread+0x350/0x784 <6>[ 213.413515][ T29] kthread+0x2ac/0x320 <6>[ 213.413528][ T29] ret_from_fork+0x10/0x30 Signed-off-by: Dan Vacura Cc: stable Link: https://lore.kernel.org/r/20220503201039.71720-1-w36195@motorola.com Signed-off-by: Greg Kroah-Hartman Bug: 229140375 (cherry picked from commit b81ac4395bbeaf36e078dea1a48c02dd97b76235 https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb-linus) Signed-off-by: Dan Vacura Change-Id: I451813637a02429041aa34ce9cb4cbd63c557415 --- drivers/usb/gadget/function/f_uvc.c | 25 +++++++++++++++++++++++++ drivers/usb/gadget/function/uvc.h | 2 ++ drivers/usb/gadget/function/uvc_v4l2.c | 3 ++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c index 2f691abf9def..875df9ef8887 100644 --- a/drivers/usb/gadget/function/f_uvc.c +++ b/drivers/usb/gadget/function/f_uvc.c @@ -889,13 +889,37 @@ static void uvc_unbind(struct usb_configuration *c, struct usb_function *f) { struct usb_composite_dev *cdev = c->cdev; struct uvc_device *uvc = to_uvc(f); + long wait_ret = 1; uvcg_info(f, "%s\n", __func__); + /* If we know we're connected via v4l2, then there should be a cleanup + * of the device from userspace either via UVC_EVENT_DISCONNECT or + * though the video device removal uevent. Allow some time for the + * application to close out before things get deleted. + */ + if (uvc->func_connected) { + uvcg_dbg(f, "waiting for clean disconnect\n"); + wait_ret = wait_event_interruptible_timeout(uvc->func_connected_queue, + uvc->func_connected == false, msecs_to_jiffies(500)); + uvcg_dbg(f, "done waiting with ret: %ld\n", wait_ret); + } + device_remove_file(&uvc->vdev.dev, &dev_attr_function_name); video_unregister_device(&uvc->vdev); v4l2_device_unregister(&uvc->v4l2_dev); + if (uvc->func_connected) { + /* Wait for the release to occur to ensure there are no longer any + * pending operations that may cause panics when resources are cleaned + * up. + */ + uvcg_warn(f, "%s no clean disconnect, wait for release\n", __func__); + wait_ret = wait_event_interruptible_timeout(uvc->func_connected_queue, + uvc->func_connected == false, msecs_to_jiffies(1000)); + uvcg_dbg(f, "done waiting for release with ret: %ld\n", wait_ret); + } + usb_ep_free_request(cdev->gadget->ep0, uvc->control_req); kfree(uvc->control_buf); @@ -914,6 +938,7 @@ static struct usb_function *uvc_alloc(struct usb_function_instance *fi) mutex_init(&uvc->video.mutex); uvc->state = UVC_STATE_DISCONNECTED; + init_waitqueue_head(&uvc->func_connected_queue); opts = fi_to_f_uvc_opts(fi); mutex_lock(&opts->lock); diff --git a/drivers/usb/gadget/function/uvc.h b/drivers/usb/gadget/function/uvc.h index e60049ec9824..59c5a67b5ff3 100644 --- a/drivers/usb/gadget/function/uvc.h +++ b/drivers/usb/gadget/function/uvc.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -123,6 +124,7 @@ struct uvc_device { struct usb_function func; struct uvc_video video; bool func_connected; + wait_queue_head_t func_connected_queue; /* Descriptors */ struct { diff --git a/drivers/usb/gadget/function/uvc_v4l2.c b/drivers/usb/gadget/function/uvc_v4l2.c index a2c78690c5c2..fd8f73bb726d 100644 --- a/drivers/usb/gadget/function/uvc_v4l2.c +++ b/drivers/usb/gadget/function/uvc_v4l2.c @@ -253,10 +253,11 @@ uvc_v4l2_subscribe_event(struct v4l2_fh *fh, static void uvc_v4l2_disable(struct uvc_device *uvc) { - uvc->func_connected = false; uvc_function_disconnect(uvc); uvcg_video_enable(&uvc->video, 0); uvcg_free_buffers(&uvc->video.queue); + uvc->func_connected = false; + wake_up_interruptible(&uvc->func_connected_queue); } static int