From c3ac7418e64d08862c5a8530b3cd8d9b2c512396 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Thu, 17 Feb 2022 20:17:56 -0700 Subject: [PATCH 01/80] ANDROID: qcom: Add sysfs related symbol Add sysfs_group_change_owner which is needed by rmnet modules. Leaf changes summary: 1 artifact changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 1 Added function Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 0 Added variable 1 Added function: [A] 'function int sysfs_group_change_owner(kobject*, const attribute_group*, kuid_t, kgid_t)' Bug: 220227362 Change-Id: I64e9ddd87f10a1b943fc1b3aa49661b3d3b70726 Signed-off-by: Subash Abhinov Kasiviswanathan --- android/abi_gki_aarch64.xml | 1392 +++++++++++++++++++--------------- android/abi_gki_aarch64_qcom | 1 + 2 files changed, 790 insertions(+), 603 deletions(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 88197fd0f8dd..1896708d2e2a 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -4982,6 +4982,7 @@ + @@ -12418,6 +12419,7 @@ + @@ -14631,6 +14633,7 @@ + @@ -21486,6 +21489,11 @@ + + + + + @@ -23445,6 +23453,17 @@ + + + + + + + + + + + @@ -24870,6 +24889,7 @@ + @@ -32753,7 +32773,32 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -35147,6 +35192,9 @@ + + + @@ -43689,7 +43737,11 @@ - + + + + + @@ -45133,6 +45185,9 @@ + + + @@ -47381,6 +47436,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -48873,6 +48954,7 @@ + @@ -52338,6 +52420,11 @@ + + + + + @@ -58666,15 +58753,15 @@ - + - + - + - + @@ -62569,15 +62656,15 @@ - + - + - + - + @@ -64125,6 +64212,14 @@ + + + + + + + + @@ -66469,6 +66564,11 @@ + + + + + @@ -70482,6 +70582,7 @@ + @@ -71145,7 +71246,11 @@ - + + + + + @@ -74134,18 +74239,18 @@ - + - + - + - + - + @@ -75833,6 +75938,11 @@ + + + + + @@ -79703,6 +79813,7 @@ + @@ -80269,6 +80380,7 @@ + @@ -82109,6 +82221,12 @@ + + + + + + @@ -83222,291 +83340,291 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -83915,6 +84033,7 @@ + @@ -86443,6 +86562,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -87927,6 +88093,7 @@ + @@ -89130,6 +89297,7 @@ + @@ -93022,6 +93190,17 @@ + + + + + + + + + + + @@ -94123,27 +94302,27 @@ - + - + - + - + - + - + - + - + @@ -108541,60 +108720,60 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114751,8 +114930,8 @@ - - + + @@ -115593,35 +115772,35 @@ - - - - + + + + - - - + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -115630,12 +115809,12 @@ - - - - - - + + + + + + @@ -115664,10 +115843,10 @@ - - - - + + + + @@ -115690,17 +115869,17 @@ - - - - - + + + + + - - - - + + + + @@ -117213,22 +117392,22 @@ - - - - - + + + + + - + - + - - + + @@ -117688,45 +117867,45 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -118067,12 +118246,12 @@ - - - - - - + + + + + + @@ -118377,8 +118556,8 @@ - - + + @@ -118386,9 +118565,9 @@ - - - + + + @@ -118996,22 +119175,22 @@ - - - + + + - - - + + + - - + + - - + + @@ -120218,8 +120397,8 @@ - - + + @@ -121394,14 +121573,14 @@ - - + + - - - - + + + + @@ -121762,17 +121941,17 @@ - - - - + + + + - - - - + + + + @@ -121791,9 +121970,9 @@ - - - + + + @@ -121811,11 +121990,11 @@ - - - - - + + + + + @@ -121920,8 +122099,8 @@ - - + + @@ -121930,9 +122109,9 @@ - - - + + + @@ -121943,54 +122122,54 @@ - - - - - - + + + + + + - - - + + + - - - + + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -122030,8 +122209,8 @@ - - + + @@ -122049,19 +122228,19 @@ - - - + + + - - - + + + - - - + + + @@ -122107,42 +122286,42 @@ - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -122155,34 +122334,34 @@ - - - - + + + + - - - - + + + + - - - - - + + + + + - - - - - + + + + + - - + + @@ -122304,19 +122483,19 @@ - - - + + + - - - + + + - - - + + + @@ -122754,15 +122933,15 @@ - - - + + + - - - - + + + + @@ -126223,10 +126402,10 @@ - - - - + + + + @@ -127349,8 +127528,8 @@ - - + + @@ -127446,9 +127625,9 @@ - - - + + + @@ -134866,8 +135045,8 @@ - - + + @@ -134875,89 +135054,89 @@ - - - - + + + + - - - + + + - - + + - - - + + + - - - - - + + + + + - - + + - - - - + + + + - + - - - - + + + + - - + + - - - + + + - - + + - - - - + + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -135198,8 +135377,8 @@ - - + + @@ -136138,10 +136317,10 @@ - - - - + + + + @@ -136343,59 +136522,59 @@ - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - - + + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -138046,11 +138225,11 @@ - - - - - + + + + + @@ -138069,8 +138248,8 @@ - - + + @@ -138080,9 +138259,9 @@ - - - + + + @@ -138130,9 +138309,9 @@ - - - + + + @@ -138146,10 +138325,10 @@ - - - - + + + + @@ -138157,9 +138336,9 @@ - - - + + + @@ -138210,11 +138389,11 @@ - - - - - + + + + + @@ -138222,9 +138401,9 @@ - - - + + + @@ -140032,7 +140211,7 @@ - + @@ -140141,6 +140320,13 @@ + + + + + + + @@ -141384,48 +141570,48 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - - - + + + + + - - - - - - + + + + + + @@ -141435,13 +141621,13 @@ - - - + + + - - + + @@ -141450,31 +141636,31 @@ - - + + - - + + - - - + + + - - + + - - + + - - - - + + + + @@ -141506,91 +141692,91 @@ - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + @@ -141906,16 +142092,16 @@ - - + + - - + + - - + + diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index e26f08dfe1c4..cd8a90f237a6 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -2430,6 +2430,7 @@ sysfs_create_groups sysfs_create_link sysfs_emit + sysfs_group_change_owner __sysfs_match_string sysfs_notify sysfs_remove_bin_file From acefa91e517b9321295429d3a0e534908e2939c1 Mon Sep 17 00:00:00 2001 From: Liujie Xie Date: Sun, 20 Feb 2022 16:53:05 +0800 Subject: [PATCH 02/80] ANDROID: vendor_hooks: Add hooks for binder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit trace_android_vh_binder_proc_transaction_entry: We need change binder thread so that this work can be added in proc->todo, if we found the binder thread, skip native logic. trace_android_vh_binder_select_worklist_ilocked: we need this because we can't change list point in ”trace_android_vh_binder_thread_read“, otherwise, If a work has beed added in our own defined list before, current may goto retry and loop again and again. Bug: 219898723 Change-Id: Ifdb3429c9ddac521bc75c1d21740ee7cc4b8f143 Signed-off-by: Liujie Xie --- drivers/android/binder.c | 12 ++++++++++-- drivers/android/vendor_hooks.c | 2 ++ include/trace/hooks/binder.h | 9 +++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index c51478a185cc..cd688e896239 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2490,6 +2490,7 @@ static int binder_proc_transaction(struct binder_transaction *t, struct binder_priority node_prio; bool oneway = !!(t->flags & TF_ONE_WAY); bool pending_async = false; + bool skip = false; BUG_ON(!node); binder_node_lock(node); @@ -2517,7 +2518,10 @@ static int binder_proc_transaction(struct binder_transaction *t, return proc->is_frozen ? BR_FROZEN_REPLY : BR_DEAD_REPLY; } - if (!thread && !pending_async) + trace_android_vh_binder_proc_transaction_entry(proc, t, + &thread, node->debug_id, pending_async, !oneway, &skip); + + if (!thread && !pending_async && !skip) thread = binder_select_thread_ilocked(proc); trace_android_vh_binder_proc_transaction(current, proc->tsk, @@ -4032,6 +4036,10 @@ retry: size_t trsize = sizeof(*trd); binder_inner_proc_lock(proc); + trace_android_vh_binder_select_worklist_ilocked(&list, thread, + proc, wait_for_proc_work); + if (list) + goto skip; if (!binder_worklist_empty_ilocked(&thread->todo)) list = &thread->todo; else if (!binder_worklist_empty_ilocked(&proc->todo) && @@ -4045,7 +4053,7 @@ retry: goto retry; break; } - +skip: if (end - ptr < sizeof(tr) + 4) { binder_inner_proc_unlock(proc); break; diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index ef6b8e851608..253f2ce82192 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -277,8 +277,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_reply); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_trans); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_binder_transaction); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_preset); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_proc_transaction_entry); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_proc_transaction); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_proc_transaction_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_select_worklist_ilocked); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_new_ref); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_del_ref); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_post_init_entity_util_avg); diff --git a/include/trace/hooks/binder.h b/include/trace/hooks/binder.h index 42d974f76573..a5c090f38388 100644 --- a/include/trace/hooks/binder.h +++ b/include/trace/hooks/binder.h @@ -59,6 +59,11 @@ DECLARE_RESTRICTED_HOOK(android_rvh_binder_transaction, DECLARE_HOOK(android_vh_binder_preset, TP_PROTO(struct hlist_head *hhead, struct mutex *lock), TP_ARGS(hhead, lock)); +DECLARE_HOOK(android_vh_binder_proc_transaction_entry, + TP_PROTO(struct binder_proc *proc, struct binder_transaction *t, + struct binder_thread **thread, int node_debug_id, bool pending_async, + bool sync, bool *skip), + TP_ARGS(proc, t, thread, node_debug_id, pending_async, sync, skip)); DECLARE_HOOK(android_vh_binder_proc_transaction, TP_PROTO(struct task_struct *caller_task, struct task_struct *binder_proc_task, struct task_struct *binder_th_task, int node_debug_id, @@ -69,6 +74,10 @@ DECLARE_HOOK(android_vh_binder_proc_transaction_end, struct task_struct *binder_th_task, unsigned int code, bool pending_async, bool sync), TP_ARGS(caller_task, binder_proc_task, binder_th_task, code, pending_async, sync)); +DECLARE_HOOK(android_vh_binder_select_worklist_ilocked, + TP_PROTO(struct list_head **list, struct binder_thread *thread, struct binder_proc *proc, + int wait_for_proc_work), + TP_ARGS(list, thread, proc, wait_for_proc_work)); DECLARE_HOOK(android_vh_binder_new_ref, TP_PROTO(struct task_struct *proc, uint32_t ref_desc, int node_debug_id), TP_ARGS(proc, ref_desc, node_debug_id)); From 3b9fe10e46e06c8c442c07fd4d36181e1e28668f Mon Sep 17 00:00:00 2001 From: Steve Muckle Date: Mon, 28 Feb 2022 20:49:55 +0000 Subject: [PATCH 03/80] ANDROID: incremental-fs: remove spurious kfree() The reverts in commits 07630c80731a (Revert "ANDROID: incremental-fs: fix mount_fs issue") and 5db3e72c57d3 (Revert "ANDROID: incremental-fs: remove index and incomplete dir on umount") were applied out of order, resulting in a spurious call to kfree() being left over. Remove it. Bug: 218732047 Signed-off-by: Steve Muckle Change-Id: I6ae8d8a9775981a88d28e462b64b259bca905ffb --- fs/incfs/data_mgmt.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/incfs/data_mgmt.c b/fs/incfs/data_mgmt.c index a383c5b5ad7f..fbab68a280d5 100644 --- a/fs/incfs/data_mgmt.c +++ b/fs/incfs/data_mgmt.c @@ -175,7 +175,6 @@ void incfs_free_mount_info(struct mount_info *mi) kfree(mi->pseudo_file_xattr[i].data); kfree(mi->mi_per_uid_read_timeouts); incfs_free_sysfs_node(mi->mi_sysfs_node); - kfree(mi->mi_options.sysfs_name); kfree(mi); } From 62412e5b8c33ee711aa20500cb1a9bb948c1f7d7 Mon Sep 17 00:00:00 2001 From: Suren Baghdasaryan Date: Thu, 24 Feb 2022 01:01:37 -0500 Subject: [PATCH 04/80] FROMGIT: mm: count time in drain_all_pages during direct reclaim as memory pressure When page allocation in direct reclaim path fails, the system will make one attempt to shrink per-cpu page lists and free pages from high alloc reserves. Draining per-cpu pages into buddy allocator can be a very slow operation because it's done using workqueues and the task in direct reclaim waits for all of them to finish before proceeding. Currently this time is not accounted as psi memory stall. While testing mobile devices under extreme memory pressure, when allocations are failing during direct reclaim, we notices that psi events which would be expected in such conditions were not triggered. After profiling these cases it was determined that the reason for missing psi events was that a big chunk of time spent in direct reclaim is not accounted as memory stall, therefore psi would not reach the levels at which an event is generated. Further investigation revealed that the bulk of that unaccounted time was spent inside drain_all_pages call. A typical captured case when drain_all_pages path gets activated: __alloc_pages_slowpath took 44.644.613ns __perform_reclaim took 751.668ns (1.7%) drain_all_pages took 43.887.167ns (98.3%) PSI in this case records the time spent in __perform_reclaim but ignores drain_all_pages, IOW it misses 98.3% of the time spent in __alloc_pages_slowpath. Annotate __alloc_pages_direct_reclaim in its entirety so that delays from handling page allocation failure in the direct reclaim path are accounted as memory stall. Link: https://lkml.kernel.org/r/20220223194812.1299646-1-surenb@google.com Signed-off-by: Suren Baghdasaryan Reported-by: Tim Murray Acked-by: Johannes Weiner Cc: Michal Hocko Cc: Petr Mladek Cc: Peter Zijlstra Cc: Roman Gushchin Cc: Shakeel Butt Cc: Minchan Kim Signed-off-by: Andrew Morton (cherry picked from commit d4f448732857375eb3dc422225a61e64f8257cb1 https://github.com/hnaz/linux-mm.git master) Bug: 205182133 Signed-off-by: Suren Baghdasaryan Change-Id: Ia3a4138f8d5e8ce612bd5c371cfcc0f21e1ebc42 --- mm/page_alloc.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 5b9b465582d6..845a88712f0b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4476,13 +4476,12 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, const struct alloc_context *ac) { unsigned int noreclaim_flag; - unsigned long pflags, progress; + unsigned long progress; cond_resched(); /* We now go into synchronous reclaim */ cpuset_memory_pressure_bump(); - psi_memstall_enter(&pflags); fs_reclaim_acquire(gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); @@ -4491,7 +4490,6 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, memalloc_noreclaim_restore(noreclaim_flag); fs_reclaim_release(gfp_mask); - psi_memstall_leave(&pflags); cond_resched(); @@ -4505,11 +4503,13 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, unsigned long *did_some_progress) { struct page *page = NULL; + unsigned long pflags; bool drained = false; + psi_memstall_enter(&pflags); *did_some_progress = __perform_reclaim(gfp_mask, order, ac); if (unlikely(!(*did_some_progress))) - return NULL; + goto out; retry: page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac); @@ -4525,6 +4525,8 @@ retry: drained = true; goto retry; } +out: + psi_memstall_leave(&pflags); return page; } From f2d0c305766eb3fe9aeef48ec1f6794e5ddb218a Mon Sep 17 00:00:00 2001 From: Liujie Xie Date: Wed, 23 Feb 2022 10:32:01 +0800 Subject: [PATCH 05/80] ANDROID: vendor_hooks: Add hooks for shrink_active_list Provide a vendor hook to allow page_referenced to be skipped during shrink_active_list to avoid heavy cpuloading caused by it. Bug: 220878851 Signed-off-by: Liujie Xie Change-Id: Ie0e369f8f8739fea59a95470af20ab0e976869d1 --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/vmscan.h | 4 ++++ mm/vmscan.c | 7 ++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 253f2ce82192..f8d55c68584c 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -291,6 +291,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_logbuf_pr_cont); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tune_scan_type); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tune_swappiness); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_slab_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_referenced_check_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_psi_event); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_psi_group); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpuset_fork); diff --git a/include/trace/hooks/vmscan.h b/include/trace/hooks/vmscan.h index ecbff00093ee..95fb17af29e8 100644 --- a/include/trace/hooks/vmscan.h +++ b/include/trace/hooks/vmscan.h @@ -25,6 +25,10 @@ DECLARE_HOOK(android_vh_tune_inactive_ratio, DECLARE_RESTRICTED_HOOK(android_rvh_set_balance_anon_file_reclaim, TP_PROTO(bool *balance_anon_file_reclaim), TP_ARGS(balance_anon_file_reclaim), 1); +DECLARE_HOOK(android_vh_page_referenced_check_bypass, + TP_PROTO(struct page *page, unsigned long nr_to_scan, int lru, bool *bypass), + TP_ARGS(page, nr_to_scan, lru, bypass)); + #endif /* _TRACE_HOOK_VMSCAN_H */ /* This part must be outside protection */ #include diff --git a/mm/vmscan.c b/mm/vmscan.c index 3f5a834dd764..46d8e65071d8 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2083,6 +2083,7 @@ static void shrink_active_list(unsigned long nr_to_scan, unsigned nr_rotated = 0; int file = is_file_lru(lru); struct pglist_data *pgdat = lruvec_pgdat(lruvec); + bool bypass = false; lru_add_drain(); @@ -2117,6 +2118,10 @@ static void shrink_active_list(unsigned long nr_to_scan, } } + trace_android_vh_page_referenced_check_bypass(page, nr_to_scan, lru, &bypass); + if (bypass) + goto skip_page_referenced; + if (page_referenced(page, 0, sc->target_mem_cgroup, &vm_flags)) { /* @@ -2134,7 +2139,7 @@ static void shrink_active_list(unsigned long nr_to_scan, continue; } } - +skip_page_referenced: ClearPageActive(page); /* we are de-activating */ SetPageWorkingset(page); list_add(&page->lru, &l_inactive); From 43754d8b7f2ad89c689fa661a3f1fefa0a088289 Mon Sep 17 00:00:00 2001 From: Liujie Xie Date: Tue, 1 Mar 2022 12:21:53 +0800 Subject: [PATCH 06/80] ANDROID: GKI: Update symbols to symbol list Update symbols to symbol list externed by oem modules. Leaf changes summary: 2 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 1 Added function Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 1 Added variable 1 Added function: [A] 'function int __traceiter_android_vh_page_referenced_check_bypass(void*, page*, unsigned long int, int, bool*)' 1 Added variable: [A] 'tracepoint __tracepoint_android_vh_page_referenced_check_bypass' Bug: 193384408 Signed-off-by: Liujie Xie Change-Id: Ib64786fe7a29eb73d229be65d873aafffdc897e9 --- android/abi_gki_aarch64.xml | 767 +++++++++++++++++----------------- android/abi_gki_aarch64_oplus | 2 + 2 files changed, 391 insertions(+), 378 deletions(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 1896708d2e2a..9a82dd40d12b 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -484,6 +484,7 @@ + @@ -6076,6 +6077,7 @@ + @@ -113520,11 +113522,11 @@ - - - - - + + + + + @@ -114128,9 +114130,9 @@ - - - + + + @@ -114138,9 +114140,9 @@ - - - + + + @@ -115772,35 +115774,35 @@ - - - - + + + + - - - + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -115809,12 +115811,12 @@ - - - - - - + + + + + + @@ -115823,30 +115825,30 @@ - - - - - - - - - - - + + - - - - - + + + + + + + + + + + + + + @@ -115869,17 +115871,17 @@ - - - - - + + + + + - - - - + + + + @@ -116435,6 +116437,14 @@ + + + + + + + + @@ -117392,22 +117402,22 @@ - - - - - + + + + + - + - - - + + + - - + + @@ -117499,6 +117509,7 @@ + @@ -117849,22 +117860,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -118058,10 +118069,10 @@ - - - - + + + + @@ -118122,9 +118133,9 @@ - - - + + + @@ -118241,9 +118252,9 @@ - - - + + + @@ -118382,8 +118393,8 @@ - - + + @@ -119521,9 +119532,9 @@ - - - + + + @@ -119808,10 +119819,10 @@ - - - - + + + + @@ -121298,10 +121309,10 @@ - - - - + + + + @@ -121489,11 +121500,11 @@ - - - - - + + + + + @@ -122964,20 +122975,20 @@ - - - - - - - - + + + + + + + + - - - - + + + + @@ -126299,10 +126310,10 @@ - - - - + + + + @@ -127018,14 +127029,14 @@ - - - + + + - - - + + + @@ -127340,8 +127351,8 @@ - - + + @@ -127392,9 +127403,9 @@ - - - + + + @@ -127705,8 +127716,8 @@ - - + + @@ -128961,13 +128972,13 @@ - - - - - - - + + + + + + + @@ -129284,64 +129295,64 @@ - + - - + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -129384,42 +129395,42 @@ - - + + - - + + - - - - + + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -129427,100 +129438,100 @@ - + - - - - + + + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + - - + + - - - - + + + + - - - + + + - - - - + + + + - - - - + + + + - - + + @@ -129532,53 +129543,53 @@ - - - + + + - - + + - - - - - - + + + + + + - - - - - + + + + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + - - + + @@ -130123,7 +130134,7 @@ - + @@ -130449,7 +130460,7 @@ - + @@ -131389,11 +131400,11 @@ - - - - - + + + + + @@ -133570,8 +133581,8 @@ - - + + @@ -135147,10 +135158,10 @@ - + - + @@ -135908,13 +135919,13 @@ - - - - - - - + + + + + + + @@ -135947,9 +135958,9 @@ - - - + + + @@ -135974,13 +135985,13 @@ - - + + - - - + + + @@ -136338,11 +136349,11 @@ - - - - - + + + + + @@ -136577,7 +136588,7 @@ - + @@ -137025,43 +137036,43 @@ - - + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + - - - - + + + + - - + + @@ -137098,8 +137109,8 @@ - - + + @@ -137853,9 +137864,9 @@ - - - + + + @@ -137885,14 +137896,14 @@ - - - + + + - - - + + + @@ -138090,11 +138101,11 @@ - + - - + + @@ -140413,7 +140424,7 @@ - + @@ -140424,8 +140435,8 @@ - - + + @@ -140879,11 +140890,11 @@ - - - - - + + + + + @@ -141521,9 +141532,9 @@ - - - + + + @@ -144319,12 +144330,12 @@ - - + + - - + + diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index 2c5078428d3e..de8b71a756f7 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -2692,6 +2692,7 @@ __traceiter_android_vh_tune_inactive_ratio __traceiter_android_vh_tune_scan_type __traceiter_android_vh_tune_swappiness + __traceiter_android_vh_page_referenced_check_bypass __traceiter_android_vh_ufs_compl_command __traceiter_android_vh_ufs_send_command __traceiter_android_vh_ufs_send_tm_command @@ -2894,6 +2895,7 @@ __tracepoint_android_vh_tune_inactive_ratio __tracepoint_android_vh_tune_scan_type __tracepoint_android_vh_tune_swappiness + __tracepoint_android_vh_page_referenced_check_bypass __tracepoint_android_vh_ufs_compl_command __tracepoint_android_vh_ufs_send_command __tracepoint_android_vh_ufs_send_tm_command From cf221db753e676bddf1cc1c33149ce2ab930981a Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Wed, 1 Dec 2021 10:06:14 -0800 Subject: [PATCH 07/80] UPSTREAM: fget: check that the fd still exists after getting a ref to it commit 054aa8d439b9185d4f5eb9a90282d1ce74772969 upstream. Jann Horn points out that there is another possible race wrt Unix domain socket garbage collection, somewhat reminiscent of the one fixed in commit cbcf01128d0a ("af_unix: fix garbage collect vs MSG_PEEK"). See the extended comment about the garbage collection requirements added to unix_peek_fds() by that commit for details. The race comes from how we can locklessly look up a file descriptor just as it is in the process of being closed, and with the right artificial timing (Jann added a few strategic 'mdelay(500)' calls to do that), the Unix domain socket garbage collector could see the reference count decrement of the close() happen before fget() took its reference to the file and the file was attached onto a new file descriptor. This is all (intentionally) correct on the 'struct file *' side, with RCU lookups and lockless reference counting very much part of the design. Getting that reference count out of order isn't a problem per se. But the garbage collector can get confused by seeing this situation of having seen a file not having any remaining external references and then seeing it being attached to an fd. In commit cbcf01128d0a ("af_unix: fix garbage collect vs MSG_PEEK") the fix was to serialize the file descriptor install with the garbage collector by taking and releasing the unix_gc_lock. That's not really an option here, but since this all happens when we are in the process of looking up a file descriptor, we can instead simply just re-check that the file hasn't been closed in the meantime, and just re-do the lookup if we raced with a concurrent close() of the same file descriptor. Reported-and-tested-by: Jann Horn Acked-by: Miklos Szeredi Signed-off-by: Linus Torvalds Signed-off-by: Greg Kroah-Hartman Change-Id: I876f1ab7997e9546b9e13f5f5b681755acd229d6 --- fs/file.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/file.c b/fs/file.c index 21c0893f2f1d..9d02352fa18c 100644 --- a/fs/file.c +++ b/fs/file.c @@ -834,6 +834,10 @@ loop: file = NULL; else if (!get_file_rcu_many(file, refs)) goto loop; + else if (__fcheck_files(files, fd) != file) { + fput_many(file, refs); + goto loop; + } } rcu_read_unlock(); From 8f66dc1a78a743ea3c3f039500d2aa0cddd776d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20L=C3=B6hle?= Date: Fri, 4 Feb 2022 15:11:37 +0000 Subject: [PATCH 08/80] UPSTREAM: mmc: block: fix read single on recovery logic commit 54309fde1a352ad2674ebba004a79f7d20b9f037 upstream. On reads with MMC_READ_MULTIPLE_BLOCK that fail, the recovery handler will use MMC_READ_SINGLE_BLOCK for each of the blocks, up to MMC_READ_SINGLE_RETRIES times each. The logic for this is fixed to never report unsuccessful reads as success to the block layer. On command error with retries remaining, blk_update_request was called with whatever value error was set last to. In case it was last set to BLK_STS_OK (default), the read will be reported as success, even though there was no data read from the device. This could happen on a CRC mismatch for the response, a card rejecting the command (e.g. again due to a CRC mismatch). In case it was last set to BLK_STS_IOERR, the error is reported correctly, but no retries will be attempted. Fixes: 81196976ed946c ("mmc: block: Add blk-mq support") Cc: stable@vger.kernel.org Signed-off-by: Christian Loehle Reviewed-by: Adrian Hunter Link: https://lore.kernel.org/r/bc706a6ab08c4fe2834ba0c05a804672@hyperstone.com Signed-off-by: Ulf Hansson Signed-off-by: Greg Kroah-Hartman Signed-off-by: Lee Jones Change-Id: Ie5a28313bbbf794fc62132db366f566e70ce80c8 --- drivers/mmc/core/block.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 74e34cd14160..14dddf05f0ac 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -1652,32 +1652,32 @@ static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req) struct mmc_card *card = mq->card; struct mmc_host *host = card->host; blk_status_t error = BLK_STS_OK; - int retries = 0; do { u32 status; int err; + int retries = 0; - mmc_blk_rw_rq_prep(mqrq, card, 1, mq); + while (retries++ <= MMC_READ_SINGLE_RETRIES) { + mmc_blk_rw_rq_prep(mqrq, card, 1, mq); - mmc_wait_for_req(host, mrq); + mmc_wait_for_req(host, mrq); - err = mmc_send_status(card, &status); - if (err) - goto error_exit; - - if (!mmc_host_is_spi(host) && - !mmc_ready_for_data(status)) { - err = mmc_blk_fix_state(card, req); + err = mmc_send_status(card, &status); if (err) goto error_exit; + + if (!mmc_host_is_spi(host) && + !mmc_ready_for_data(status)) { + err = mmc_blk_fix_state(card, req); + if (err) + goto error_exit; + } + + if (!mrq->cmd->error) + break; } - if (mrq->cmd->error && retries++ < MMC_READ_SINGLE_RETRIES) - continue; - - retries = 0; - if (mrq->cmd->error || mrq->data->error || (!mmc_host_is_spi(host) && From bf96382fb9220359fdffc3fe485a5b535a094a2d Mon Sep 17 00:00:00 2001 From: Chris Goldsworthy Date: Thu, 24 Feb 2022 16:13:30 -0800 Subject: [PATCH 09/80] ANDROID: dma-direct: Make DMA32 disablement work for CONFIG_NUMA zone_dma32_is_empty() currently lacks the proper validation to ensure that the NUMA node ID it receives as an argument is valid. This has no effect on kernels with CONFIG_NUMA=n as NODE_DATA() will return the same pglist_data on these devices, but on kernels with CONFIG_NUMA=y, this is not the case, and the node passed to NODE_DATA must be validated. Rather than trying to find the node containing ZONE_DMA32, replace calls of zone_dma32_is_empty() with zone_dma32_are_empty() (which iterates over all nodes and returns false if one of the nodes holds DMA32 and it is non-empty). Bug: 199917449 Fixes: c3c2bb34ac8f ("ANDROID: arm64/mm: Add command line option to make ZONE_DMA32 empty") Signed-off-by: Chris Goldsworthy Change-Id: I850fb9213b71a1ef29106728bfda0cc6de46fdbb --- include/linux/dma-direct.h | 5 +++++ kernel/dma/direct.c | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index ed69618cf1fc..80f0dbac77c4 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -37,11 +37,16 @@ static inline bool zone_dma32_is_empty(int node) static inline bool zone_dma32_are_empty(void) { +#ifdef CONFIG_NUMA int node; for_each_node(node) if (!zone_dma32_is_empty(node)) return false; +#else + if (!zone_dma32_is_empty(numa_node_id())) + return false; +#endif return true; } diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 6f58cde1558b..8ca84610d4d4 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -62,7 +62,7 @@ static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, if (*phys_limit <= DMA_BIT_MASK(zone_dma_bits)) return GFP_DMA; if (*phys_limit <= DMA_BIT_MASK(32) && - !zone_dma32_is_empty(dev_to_node(dev))) + !zone_dma32_are_empty()) return GFP_DMA32; return 0; } @@ -103,7 +103,7 @@ again: if (IS_ENABLED(CONFIG_ZONE_DMA32) && phys_limit < DMA_BIT_MASK(64) && !(gfp & (GFP_DMA32 | GFP_DMA)) && - !zone_dma32_is_empty(node)) { + !zone_dma32_are_empty()) { gfp |= GFP_DMA32; goto again; } From 135406cecb00983c7e98c2e3638f6f59080081ea Mon Sep 17 00:00:00 2001 From: Chris Goldsworthy Date: Fri, 25 Feb 2022 18:59:08 -0800 Subject: [PATCH 10/80] ANDROID: dma-direct: Document disable_dma32 Document the functionality of disable_dma32 as introduced in commit c3c2bb34ac8f ("ANDROID: arm64/mm: Add command line option to make ZONE_DMA32 empty"). Bug: 199917449 Signed-off-by: Chris Goldsworthy Change-Id: I32ab2969f59fcc49e9ac49e7e6b545f816d120f9 --- Documentation/admin-guide/kernel-parameters.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index fadb6b3b93cb..38614bd62e19 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -939,6 +939,10 @@ can be useful when debugging issues that require an SLB miss to occur. + disable_dma32= [KNL] + Dynamically disable ZONE_DMA32 on kernels compiled with + CONFIG_ZONE_DMA32=y. + stress_slb [PPC] Limits the number of kernel SLB entries, and flushes them frequently to increase the rate of SLB faults From ebbf267fc0f7122f76c24adcb1b5bce932fb597c Mon Sep 17 00:00:00 2001 From: Liujie Xie Date: Tue, 22 Feb 2022 19:29:53 +0800 Subject: [PATCH 11/80] ANDROID: vendor_hooks: Add hooks for __alloc_pages_direct_reclaim Provide a vendor hook to allow drain_all_pages to be skipped during direct reclaim in some cases to avoid delays caused by it in cases when the benefits of draining pcp lists are known to be small. Bug: 220811627 Signed-off-by: Liujie Xie Change-Id: I0805241f81e0a94afcf62c98e97cff125d4061e2 --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/mm.h | 5 +++++ mm/page_alloc.c | 6 +++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index f8d55c68584c..6492e896e61f 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -292,6 +292,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tune_scan_type); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tune_swappiness); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_slab_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_page_referenced_check_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_drain_all_pages_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_psi_event); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_psi_group); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpuset_fork); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 1e7c0a0811de..cc00dcecfd71 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -117,6 +117,11 @@ DECLARE_HOOK(android_vh_mmap_region, DECLARE_HOOK(android_vh_try_to_unmap_one, TP_PROTO(struct vm_area_struct *vma, struct page *page, unsigned long addr, bool ret), TP_ARGS(vma, page, addr, ret)); +DECLARE_HOOK(android_vh_drain_all_pages_bypass, + TP_PROTO(gfp_t gfp_mask, unsigned int order, unsigned long alloc_flags, + int migratetype, unsigned long did_some_progress, + bool *bypass), + TP_ARGS(gfp_mask, order, alloc_flags, migratetype, did_some_progress, bypass)); struct device; DECLARE_HOOK(android_vh_subpage_dma_contig_alloc, TP_PROTO(bool *allow_subpage_alloc, struct device *dev, size_t *size), diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 845a88712f0b..7c5399d0e25f 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4505,6 +4505,7 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; bool drained = false; + bool skip_pcp_drain = false; psi_memstall_enter(&pflags); *did_some_progress = __perform_reclaim(gfp_mask, order, ac); @@ -4521,7 +4522,10 @@ retry: */ if (!page && !drained) { unreserve_highatomic_pageblock(ac, false); - drain_all_pages(NULL); + trace_android_vh_drain_all_pages_bypass(gfp_mask, order, + alloc_flags, ac->migratetype, *did_some_progress, &skip_pcp_drain); + if (!skip_pcp_drain) + drain_all_pages(NULL); drained = true; goto retry; } From 767f384155e943975d51df054da14504a3bd0861 Mon Sep 17 00:00:00 2001 From: Mathias Nyman Date: Thu, 3 Mar 2022 13:08:55 +0200 Subject: [PATCH 12/80] FROMGIT: xhci: make xhci_handshake timeout for xhci_reset() adjustable xhci_reset() timeout was increased from 250ms to 10 seconds in order to give Renesas 720201 xHC enough time to get ready in probe. xhci_reset() is called with interrupts disabled in other places, and waiting for 10 seconds there is not acceptable. Add a timeout parameter to xhci_reset(), and adjust it back to 250ms when called from xhci_stop() or xhci_shutdown() where interrupts are disabled, and successful reset isn't that critical. This solves issues when deactivating host mode on platforms like SM8450. For now don't change the timeout if xHC is reset in xhci_resume(). No issues are reported for it, and we need the reset to succeed. Locking around that reset needs to be revisited later. Additionally change the signed integer timeout parameter in xhci_handshake() to a u64 to match the timeout value we pass to readl_poll_timeout_atomic() Fixes: 22ceac191211 ("xhci: Increase reset timeout for Renesas 720201 host.") Cc: stable@vger.kernel.org Reported-by: Sergey Shtylyov Reported-by: Pavan Kondeti Tested-by: Pavan Kondeti Signed-off-by: Mathias Nyman Link: https://lore.kernel.org/r/20220303110903.1662404-2-mathias.nyman@linux.intel.com Signed-off-by: Greg Kroah-Hartman (cherry picked from commit 14073ce951b5919da450022c050772902f24f054 https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git/ usb-testing) BUG: 218973924 Change-Id: I2299b2116701d1563d7b9567433162b35488295d Signed-off-by: Udipto Goswami --- drivers/usb/host/xhci-hub.c | 2 +- drivers/usb/host/xhci-mem.c | 2 +- drivers/usb/host/xhci.c | 20 +++++++++----------- drivers/usb/host/xhci.h | 7 +++++-- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c index c5f17dbc50c9..0994e39e086d 100644 --- a/drivers/usb/host/xhci-hub.c +++ b/drivers/usb/host/xhci-hub.c @@ -680,7 +680,7 @@ static int xhci_exit_test_mode(struct xhci_hcd *xhci) } pm_runtime_allow(xhci_to_hcd(xhci)->self.controller); xhci->test_mode = 0; - return xhci_reset(xhci); + return xhci_reset(xhci, XHCI_RESET_SHORT_USEC); } void xhci_set_link_state(struct xhci_hcd *xhci, struct xhci_port *port, diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c index 4324fd31b2c4..644ddcb26e76 100644 --- a/drivers/usb/host/xhci-mem.c +++ b/drivers/usb/host/xhci-mem.c @@ -2695,7 +2695,7 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) fail: xhci_halt(xhci); - xhci_reset(xhci); + xhci_reset(xhci, XHCI_RESET_SHORT_USEC); xhci_mem_cleanup(xhci); return -ENOMEM; } diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index a3d787df3fe9..472c65ca4a9d 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -65,7 +65,7 @@ static bool td_on_ring(struct xhci_td *td, struct xhci_ring *ring) * handshake done). There are two failure modes: "usec" have passed (major * hardware flakeout), or the register reads as all-ones (hardware removed). */ -int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, int usec) +int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, u64 timeout_us) { u32 result; int ret; @@ -73,7 +73,7 @@ int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, int usec) ret = readl_poll_timeout_atomic(ptr, result, (result & mask) == done || result == U32_MAX, - 1, usec); + 1, timeout_us); if (result == U32_MAX) /* card removed */ return -ENODEV; @@ -162,7 +162,7 @@ int xhci_start(struct xhci_hcd *xhci) * Transactions will be terminated immediately, and operational registers * will be set to their defaults. */ -int xhci_reset(struct xhci_hcd *xhci) +int xhci_reset(struct xhci_hcd *xhci, u64 timeout_us) { u32 command; u32 state; @@ -195,8 +195,7 @@ int xhci_reset(struct xhci_hcd *xhci) if (xhci->quirks & XHCI_INTEL_HOST) udelay(1000); - ret = xhci_handshake(&xhci->op_regs->command, - CMD_RESET, 0, 10 * 1000 * 1000); + ret = xhci_handshake(&xhci->op_regs->command, CMD_RESET, 0, timeout_us); if (ret) return ret; @@ -209,8 +208,7 @@ int xhci_reset(struct xhci_hcd *xhci) * xHCI cannot write to any doorbells or operational registers other * than status until the "Controller Not Ready" flag is cleared. */ - ret = xhci_handshake(&xhci->op_regs->status, - STS_CNR, 0, 10 * 1000 * 1000); + ret = xhci_handshake(&xhci->op_regs->status, STS_CNR, 0, timeout_us); xhci->usb2_rhub.bus_state.port_c_suspend = 0; xhci->usb2_rhub.bus_state.suspended_ports = 0; @@ -731,7 +729,7 @@ static void xhci_stop(struct usb_hcd *hcd) xhci->xhc_state |= XHCI_STATE_HALTED; xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; xhci_halt(xhci); - xhci_reset(xhci); + xhci_reset(xhci, XHCI_RESET_SHORT_USEC); spin_unlock_irq(&xhci->lock); xhci_cleanup_msix(xhci); @@ -784,7 +782,7 @@ void xhci_shutdown(struct usb_hcd *hcd) xhci_halt(xhci); /* Workaround for spurious wakeups at shutdown with HSW */ if (xhci->quirks & XHCI_SPURIOUS_WAKEUP) - xhci_reset(xhci); + xhci_reset(xhci, XHCI_RESET_SHORT_USEC); spin_unlock_irq(&xhci->lock); xhci_cleanup_msix(xhci); @@ -1163,7 +1161,7 @@ int xhci_resume(struct xhci_hcd *xhci, bool hibernated) xhci_dbg(xhci, "Stop HCD\n"); xhci_halt(xhci); xhci_zero_64b_regs(xhci); - retval = xhci_reset(xhci); + retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC); spin_unlock_irq(&xhci->lock); if (retval) return retval; @@ -5268,7 +5266,7 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks) xhci_dbg(xhci, "Resetting HCD\n"); /* Reset the internal HC memory state and registers. */ - retval = xhci_reset(xhci); + retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC); if (retval) return retval; xhci_dbg(xhci, "Reset complete\n"); diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 2bba700a7805..2a71477fab71 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -230,6 +230,9 @@ struct xhci_op_regs { #define CMD_ETE (1 << 14) /* bits 15:31 are reserved (and should be preserved on writes). */ +#define XHCI_RESET_LONG_USEC (10 * 1000 * 1000) +#define XHCI_RESET_SHORT_USEC (250 * 1000) + /* IMAN - Interrupt Management Register */ #define IMAN_IE (1 << 1) #define IMAN_IP (1 << 0) @@ -2097,11 +2100,11 @@ void xhci_free_container_ctx(struct xhci_hcd *xhci, /* xHCI host controller glue */ typedef void (*xhci_get_quirks_t)(struct device *, struct xhci_hcd *); -int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, int usec); +int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, u64 timeout_us); void xhci_quiesce(struct xhci_hcd *xhci); int xhci_halt(struct xhci_hcd *xhci); int xhci_start(struct xhci_hcd *xhci); -int xhci_reset(struct xhci_hcd *xhci); +int xhci_reset(struct xhci_hcd *xhci, u64 timeout_us); int xhci_run(struct usb_hcd *hcd); int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks); void xhci_shutdown(struct usb_hcd *hcd); From e53b1b9ad42deea73298f82813dfb4db076a89b4 Mon Sep 17 00:00:00 2001 From: Puma Hsu Date: Tue, 15 Feb 2022 14:33:19 +0200 Subject: [PATCH 13/80] UPSTREAM: xhci: re-initialize the HC during resume if HCE was set (Upstream commit 8b328f8002bcf29ef517ee4bf234e09aabec4d2e) When HCE(Host Controller Error) is set, it means an internal error condition has been detected. Software needs to re-initialize the HC, so add this check in xhci resume. Cc: stable@vger.kernel.org Signed-off-by: Puma Hsu Signed-off-by: Mathias Nyman Link: https://lore.kernel.org/r/20220215123320.1253947-2-mathias.nyman@linux.intel.com Signed-off-by: Greg Kroah-Hartman Bug: 210947520 Signed-off-by: Puma Hsu Change-Id: If40d9f696972f5da94d0d3c6130e7060bfcae5a3 --- drivers/usb/host/xhci.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 472c65ca4a9d..cd1d186c09ef 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -1089,6 +1089,7 @@ int xhci_resume(struct xhci_hcd *xhci, bool hibernated) int retval = 0; bool comp_timer_running = false; bool pending_portevent = false; + bool reinit_xhc = false; if (!hcd->state) return 0; @@ -1105,10 +1106,11 @@ int xhci_resume(struct xhci_hcd *xhci, bool hibernated) set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags); spin_lock_irq(&xhci->lock); - if ((xhci->quirks & XHCI_RESET_ON_RESUME) || xhci->broken_suspend) - hibernated = true; - if (!hibernated) { + if (hibernated || xhci->quirks & XHCI_RESET_ON_RESUME || xhci->broken_suspend) + reinit_xhc = true; + + if (!reinit_xhc) { /* * Some controllers might lose power during suspend, so wait * for controller not ready bit to clear, just as in xHC init. @@ -1141,12 +1143,17 @@ int xhci_resume(struct xhci_hcd *xhci, bool hibernated) spin_unlock_irq(&xhci->lock); return -ETIMEDOUT; } - temp = readl(&xhci->op_regs->status); } - /* If restore operation fails, re-initialize the HC during resume */ - if ((temp & STS_SRE) || hibernated) { + temp = readl(&xhci->op_regs->status); + /* re-initialize the HC on Restore Error, or Host Controller Error */ + if (temp & (STS_SRE | STS_HCE)) { + reinit_xhc = true; + xhci_warn(xhci, "xHC error in resume, USBSTS 0x%x, Reinit\n", temp); + } + + if (reinit_xhc) { if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !(xhci_all_ports_seen_u0(xhci))) { del_timer_sync(&xhci->comp_mode_recovery_timer); From a817d6ed8746d9de8960acde229c3a9dfa4d465a Mon Sep 17 00:00:00 2001 From: Vijayanand Jitta Date: Wed, 2 Mar 2022 22:25:21 +0530 Subject: [PATCH 14/80] ANDROID: mm: Fix page table lookup in speculative fault path In speculative fault path, while doing page table lookup, offset is obtained at each level and value at that offset is read and checks are perfomed on it, later to get next level offset we read from previous level offset again. A concurrent page table reclaimation operation could result in change in value at this offset, and we go ahead and access it, this would result in reading an invalid entry. Fix this by reading from previous level offset again and comparing before performing next level access. Bug: 221005439 Change-Id: I66b3d24ae79c7ee5ccce4ba7a94f028f4cf3fda0 Signed-off-by: Vijayanand Jitta --- mm/memory.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mm/memory.c b/mm/memory.c index dfe88223c483..c05b1f22f20e 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5033,11 +5033,15 @@ static vm_fault_t ___handle_speculative_fault(struct mm_struct *mm, goto out_walk; p4d = p4d_offset(pgd, address); + if (pgd_val(READ_ONCE(*pgd)) != pgd_val(pgdval)) + goto out_walk; p4dval = READ_ONCE(*p4d); if (p4d_none(p4dval) || unlikely(p4d_bad(p4dval))) goto out_walk; vmf.pud = pud_offset(p4d, address); + if (p4d_val(READ_ONCE(*p4d)) != p4d_val(p4dval)) + goto out_walk; pudval = READ_ONCE(*vmf.pud); if (pud_none(pudval) || unlikely(pud_bad(pudval))) goto out_walk; @@ -5047,6 +5051,8 @@ static vm_fault_t ___handle_speculative_fault(struct mm_struct *mm, goto out_walk; vmf.pmd = pmd_offset(vmf.pud, address); + if (pud_val(READ_ONCE(*vmf.pud)) != pud_val(pudval)) + goto out_walk; vmf.orig_pmd = READ_ONCE(*vmf.pmd); /* * pmd_none could mean that a hugepage collapse is in progress @@ -5074,6 +5080,11 @@ static vm_fault_t ___handle_speculative_fault(struct mm_struct *mm, */ vmf.pte = pte_offset_map(vmf.pmd, address); + if (pmd_val(READ_ONCE(*vmf.pmd)) != pmd_val(vmf.orig_pmd)) { + pte_unmap(vmf.pte); + vmf.pte = NULL; + goto out_walk; + } vmf.orig_pte = READ_ONCE(*vmf.pte); barrier(); /* See comment in handle_pte_fault() */ if (pte_none(vmf.orig_pte)) { From 91be4236fbe0d70e092f58d1503d874a76dee1e7 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 7 Mar 2022 20:45:25 +0100 Subject: [PATCH 15/80] ANDROID: GKI: remove vfs-only namespace from 2 symbols Commit d483eed85ffb ("ANDROID: GKI: set vfs-only exports into their own namespace") moved a bunch of symbols into a vfs-only namespace to make it possible for some external filesystem modules to be able to use them. Unfortunately the following two symbols were already being marked used by external modules, and moving them into a different namespace broke existing users of these symbols: kern_path __sync_dirty_buffer The ABI checking tools do not take the namespace of the symbol into consideration when checking, as that is a Linux kernel "add-on" and not part of the kernel symbol table information directly, which is why this was not caught earlier. Bug: 157965270 Bug: 210074446 Bug: 216253405 Bug: 219830266 Fixes: d483eed85ffb ("ANDROID: GKI: set vfs-only exports into their own namespace") Signed-off-by: Greg Kroah-Hartman Change-Id: I4a791edb33312da232cb088613bd4eb8f5548239 --- fs/buffer.c | 2 +- fs/namei.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index d3916df01380..dd992943b270 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -3189,7 +3189,7 @@ int __sync_dirty_buffer(struct buffer_head *bh, int op_flags) } return ret; } -EXPORT_SYMBOL_NS(__sync_dirty_buffer, ANDROID_GKI_VFS_EXPORT_ONLY); +EXPORT_SYMBOL(__sync_dirty_buffer); int sync_dirty_buffer(struct buffer_head *bh) { diff --git a/fs/namei.c b/fs/namei.c index 297b2a774eaf..a462eccbf53d 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -2533,7 +2533,7 @@ int kern_path(const char *name, unsigned int flags, struct path *path) return filename_lookup(AT_FDCWD, getname_kernel(name), flags, path, NULL); } -EXPORT_SYMBOL_NS(kern_path, ANDROID_GKI_VFS_EXPORT_ONLY); +EXPORT_SYMBOL(kern_path); /** * vfs_path_lookup - lookup a file path relative to a dentry-vfsmount pair From fcaaaaae6d4c72eb4084dc136acd478ad09fe0f0 Mon Sep 17 00:00:00 2001 From: JaeMan Park Date: Thu, 13 Jan 2022 15:02:35 +0900 Subject: [PATCH 16/80] UPSTREAM: mac80211_hwsim: initialize ieee80211_tx_info at hw_scan_work In mac80211_hwsim, the probe_req frame is created and sent while scanning. It is sent with ieee80211_tx_info which is not initialized. Uninitialized ieee80211_tx_info can cause problems when using mac80211_hwsim with wmediumd. wmediumd checks the tx_rates field of ieee80211_tx_info and doesn't relay probe_req frame to other clients even if it is a broadcasting message. Call ieee80211_tx_prepare_skb() to initialize ieee80211_tx_info for the probe_req that is created by hw_scan_work in mac80211_hwsim. Signed-off-by: JaeMan Park Link: https://lore.kernel.org/r/20220113060235.546107-1-jaeman@google.com [fix memory leak] Signed-off-by: Johannes Berg (cherry picked from commit cacfddf82baf1470e5741edeecb187260868f195) Bug: 211353765 Change-Id: I7ffd0219b093ab18baf29aa0f9671d78ebd2f278 Signed-off-by: JaeMan Park Signed-off-by: Alistair Delva --- drivers/net/wireless/mac80211_hwsim.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c index e265f4c5c814..4697697c6318 100644 --- a/drivers/net/wireless/mac80211_hwsim.c +++ b/drivers/net/wireless/mac80211_hwsim.c @@ -2268,6 +2268,15 @@ static void hw_scan_work(struct work_struct *work) if (req->ie_len) skb_put_data(probe, req->ie, req->ie_len); + if (!ieee80211_tx_prepare_skb(hwsim->hw, + hwsim->hw_scan_vif, + probe, + hwsim->tmp_chan->band, + NULL)) { + kfree_skb(probe); + continue; + } + local_bh_disable(); mac80211_hwsim_tx_frame(hwsim->hw, probe, hwsim->tmp_chan); From 4cc8ec84be46225f4e61c4dfe77304a87caffb53 Mon Sep 17 00:00:00 2001 From: Woogeun Lee Date: Fri, 4 Mar 2022 18:07:23 +0900 Subject: [PATCH 17/80] ANDROID: ABI: update allowed list for galaxy Leaf changes summary: 3 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 1 Added function Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 2 Added variables 1 Added function: [A] 'function int __traceiter_kfree_skb(void*, sk_buff*, void*)' 2 Added variables: [A] 'tracepoint __tracepoint_kfree_skb' [A] 'hid_ll_driver usb_hid_driver' Bug: 222637795 Signed-off-by: Woogeun Lee Change-Id: I27eaff7ea3f504f337d67f8b6523737221b3f867 --- android/abi_gki_aarch64.xml | 29 ++++++++++++++++++++--------- android/abi_gki_aarch64_galaxy | 3 +++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 9a82dd40d12b..ab4758b1d3e8 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -591,6 +591,7 @@ + @@ -6200,6 +6201,7 @@ + @@ -6403,6 +6405,7 @@ + @@ -114121,8 +114124,8 @@ - - + + @@ -117103,6 +117106,12 @@ + + + + + + @@ -117632,6 +117641,7 @@ + @@ -126791,8 +126801,8 @@ - - + + @@ -130018,11 +130028,11 @@ - - - - - + + + + + @@ -142485,6 +142495,7 @@ + diff --git a/android/abi_gki_aarch64_galaxy b/android/abi_gki_aarch64_galaxy index f8d07776abc9..f35385449132 100644 --- a/android/abi_gki_aarch64_galaxy +++ b/android/abi_gki_aarch64_galaxy @@ -343,6 +343,7 @@ __traceiter_dwc3_readl __traceiter_dwc3_writel __traceiter_gpu_mem_total + __traceiter_kfree_skb __traceiter_sched_util_est_se_tp __traceiter_xdp_exception __tracepoint_android_rvh_account_irq @@ -494,6 +495,7 @@ __tracepoint_ipi_raise __tracepoint_irq_handler_entry __tracepoint_irq_handler_exit + __tracepoint_kfree_skb __tracepoint_pelt_cfs_tp __tracepoint_pelt_dl_tp __tracepoint_pelt_irq_tp @@ -4278,6 +4280,7 @@ usb_hcd_start_port_resume usb_hcd_unlink_urb_from_ep usb_hcds_loaded + usb_hid_driver usb_hub_clear_tt_buffer usb_hub_find_child usb_ifnum_to_if From 91fef75d480ee532f86badebe92ac7a4465042bc Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Tue, 22 Feb 2022 10:43:13 -0800 Subject: [PATCH 18/80] FROMGIT: f2fs: Restore rwsem lockdep support Lockdep uses lock class keys in its analysis. init_rwsem() instantiates one lock class key with each init_rwsem() user as follows: #define init_rwsem(sem) \ do { \ static struct lock_class_key __key; \ \ __init_rwsem((sem), #sem, &__key); \ } while (0) Commit e4544b63a7ee ("f2fs: move f2fs to use reader-unfair rwsems") reduced the number of lock class keys from one per init_rwsem() user to one per file in which init_f2fs_rwsem() is used. This causes the same lock class key to be associated with multiple f2fs rwsems and also triggers a number of false positive lockdep deadlock reports. Fix this by again instantiating one lock class key with each init_f2fs_rwsem() caller. Bug: 223346410 Cc: Tim Murray Reported-by: syzbot+0b9cadf5fc45a98a5083@syzkaller.appspotmail.com Fixes: e4544b63a7ee ("f2fs: move f2fs to use reader-unfair rwsems") Signed-off-by: Bart Van Assche (cherry picked from commit c7f91bd4102902384137dd5c50d04bfed27050dd git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git dev) Signed-off-by: Jaegeuk Kim Change-Id: I7ece9c98db2e84e54427a2c8155043cb15223ba3 --- fs/f2fs/f2fs.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index ae06162e6e31..b9fcd712e46a 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -2080,9 +2080,17 @@ static inline bool enabled_nat_bits(struct f2fs_sb_info *sbi, return (cpc) ? (cpc->reason & CP_UMOUNT) && set : set; } -static inline void init_f2fs_rwsem(struct f2fs_rwsem *sem) +#define init_f2fs_rwsem(sem) \ +do { \ + static struct lock_class_key __key; \ + \ + __init_f2fs_rwsem((sem), #sem, &__key); \ +} while (0) + +static inline void __init_f2fs_rwsem(struct f2fs_rwsem *sem, + const char *sem_name, struct lock_class_key *key) { - init_rwsem(&sem->internal_rwsem); + __init_rwsem(&sem->internal_rwsem, sem_name, key); init_waitqueue_head(&sem->read_waiters); } From 786bcb11099e8926ac5b5270de844d01c394ab89 Mon Sep 17 00:00:00 2001 From: Juhyung Park Date: Tue, 15 Feb 2022 17:27:21 +0900 Subject: [PATCH 19/80] FROMGIT: f2fs: quota: fix loop condition at f2fs_quota_sync() cnt should be passed to sb_has_quota_active() instead of type to check active quota properly. Moreover, when the type is -1, the compiler with enough inline knowledge can discard sb_has_quota_active() check altogether, causing a NULL pointer dereference at the following inode_lock(dqopt->files[cnt]): [ 2.796010] Unable to handle kernel NULL pointer dereference at virtual address 00000000000000a0 [ 2.796024] Mem abort info: [ 2.796025] ESR = 0x96000005 [ 2.796028] EC = 0x25: DABT (current EL), IL = 32 bits [ 2.796029] SET = 0, FnV = 0 [ 2.796031] EA = 0, S1PTW = 0 [ 2.796032] Data abort info: [ 2.796034] ISV = 0, ISS = 0x00000005 [ 2.796035] CM = 0, WnR = 0 [ 2.796046] user pgtable: 4k pages, 39-bit VAs, pgdp=00000003370d1000 [ 2.796048] [00000000000000a0] pgd=0000000000000000, pud=0000000000000000 [ 2.796051] Internal error: Oops: 96000005 [#1] PREEMPT SMP [ 2.796056] CPU: 7 PID: 640 Comm: f2fs_ckpt-259:7 Tainted: G S 5.4.179-arter97-r8-64666-g2f16e087f9d8 #1 [ 2.796057] Hardware name: Qualcomm Technologies, Inc. Lahaina MTP lemonadep (DT) [ 2.796059] pstate: 80c00005 (Nzcv daif +PAN +UAO) [ 2.796065] pc : down_write+0x28/0x70 [ 2.796070] lr : f2fs_quota_sync+0x100/0x294 [ 2.796071] sp : ffffffa3f48ffc30 [ 2.796073] x29: ffffffa3f48ffc30 x28: 0000000000000000 [ 2.796075] x27: ffffffa3f6d718b8 x26: ffffffa415fe9d80 [ 2.796077] x25: ffffffa3f7290048 x24: 0000000000000001 [ 2.796078] x23: 0000000000000000 x22: ffffffa3f7290000 [ 2.796080] x21: ffffffa3f72904a0 x20: ffffffa3f7290110 [ 2.796081] x19: ffffffa3f77a9800 x18: ffffffc020aae038 [ 2.796083] x17: ffffffa40e38e040 x16: ffffffa40e38e6d0 [ 2.796085] x15: ffffffa40e38e6cc x14: ffffffa40e38e6d0 [ 2.796086] x13: 00000000000004f6 x12: 00162c44ff493000 [ 2.796088] x11: 0000000000000400 x10: ffffffa40e38c948 [ 2.796090] x9 : 0000000000000000 x8 : 00000000000000a0 [ 2.796091] x7 : 0000000000000000 x6 : 0000d1060f00002a [ 2.796093] x5 : ffffffa3f48ff718 x4 : 000000000000000d [ 2.796094] x3 : 00000000060c0000 x2 : 0000000000000001 [ 2.796096] x1 : 0000000000000000 x0 : 00000000000000a0 [ 2.796098] Call trace: [ 2.796100] down_write+0x28/0x70 [ 2.796102] f2fs_quota_sync+0x100/0x294 [ 2.796104] block_operations+0x120/0x204 [ 2.796106] f2fs_write_checkpoint+0x11c/0x520 [ 2.796107] __checkpoint_and_complete_reqs+0x7c/0xd34 [ 2.796109] issue_checkpoint_thread+0x6c/0xb8 [ 2.796112] kthread+0x138/0x414 [ 2.796114] ret_from_fork+0x10/0x18 [ 2.796117] Code: aa0803e0 aa1f03e1 52800022 aa0103e9 (c8e97d02) [ 2.796120] ---[ end trace 96e942e8eb6a0b53 ]--- [ 2.800116] Kernel panic - not syncing: Fatal exception [ 2.800120] SMP: stopping secondary CPUs Bug: 223346410 Fixes: 9de71ede81e6 ("f2fs: quota: fix potential deadlock") Cc: # v5.15+ Signed-off-by: Juhyung Park Reviewed-by: Chao Yu (cherry picked from commit 680af5b824a52faa819167628665804a14f0e0df git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git dev) Signed-off-by: Jaegeuk Kim Change-Id: Ia3da7f094d433f74e336bff20baa74226a527258 --- fs/f2fs/super.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 408434e253c9..fc0f69fa702d 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2485,7 +2485,7 @@ int f2fs_quota_sync(struct super_block *sb, int type) struct f2fs_sb_info *sbi = F2FS_SB(sb); struct quota_info *dqopt = sb_dqopt(sb); int cnt; - int ret; + int ret = 0; /* * Now when everything is written we can discard the pagecache so @@ -2496,8 +2496,8 @@ int f2fs_quota_sync(struct super_block *sb, int type) if (type != -1 && cnt != type) continue; - if (!sb_has_quota_active(sb, type)) - return 0; + if (!sb_has_quota_active(sb, cnt)) + continue; inode_lock(dqopt->files[cnt]); From ae38f9954b6d739f7510fd0a20cb46fc7cb1e65c Mon Sep 17 00:00:00 2001 From: Liujie Xie Date: Mon, 7 Mar 2022 21:35:36 +0800 Subject: [PATCH 20/80] ANDROID: GKI: Update symbols to symbol list Update symbols to symbol list externed by oem modules. Leaf changes summary: 2 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 1 Added function Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 1 Added variable 1 Added function: [A] 'function int __traceiter_android_vh_drain_all_pages_bypass(void*, gfp_t, unsigned int, unsigned long int, int, unsigned long int, bool*)' 1 Added variable: [A] 'tracepoint __tracepoint_android_vh_drain_all_pages_bypass' Bug: 193384408 Signed-off-by: Liujie Xie Change-Id: I9c4e7076f7b58b11ed7566341e868866ed1e6f90 --- android/abi_gki_aarch64.xml | 71 +++++++++++++++++++++-------------- android/abi_gki_aarch64_oplus | 2 + 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index ab4758b1d3e8..ee4eba529ba0 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -426,6 +426,7 @@ + @@ -6017,6 +6018,7 @@ + @@ -113525,11 +113527,11 @@ - - - - - + + + + + @@ -114133,9 +114135,9 @@ - - - + + + @@ -114143,9 +114145,9 @@ - - - + + + @@ -116081,6 +116083,16 @@ + + + + + + + + + + @@ -117457,6 +117469,7 @@ + @@ -117573,7 +117586,7 @@ - + @@ -118143,9 +118156,9 @@ - - - + + + @@ -118262,9 +118275,9 @@ - - - + + + @@ -127039,14 +127052,14 @@ - - - + + + - - - + + + @@ -127726,8 +127739,8 @@ - - + + @@ -138111,11 +138124,11 @@ - + - - + + diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index de8b71a756f7..c01dfbccd138 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -2693,6 +2693,7 @@ __traceiter_android_vh_tune_scan_type __traceiter_android_vh_tune_swappiness __traceiter_android_vh_page_referenced_check_bypass + __traceiter_android_vh_drain_all_pages_bypass __traceiter_android_vh_ufs_compl_command __traceiter_android_vh_ufs_send_command __traceiter_android_vh_ufs_send_tm_command @@ -2896,6 +2897,7 @@ __tracepoint_android_vh_tune_scan_type __tracepoint_android_vh_tune_swappiness __tracepoint_android_vh_page_referenced_check_bypass + __tracepoint_android_vh_drain_all_pages_bypass __tracepoint_android_vh_ufs_compl_command __tracepoint_android_vh_ufs_send_command __tracepoint_android_vh_ufs_send_tm_command From 7b5fea2f46ac7babc4dc05dc606bcbec07a216bc Mon Sep 17 00:00:00 2001 From: wuweizhong Date: Wed, 9 Mar 2022 21:33:27 +0800 Subject: [PATCH 21/80] ANDROID: GKI: add allowed list file for xiaomi Leaf changes summary: 4 artifacts changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 4 Added functions Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 0 Added variable 4 Added functions: [A] 'function int wakeup_sources_read_lock()' [A] 'function void wakeup_sources_read_unlock(int)' [A] 'function wakeup_source* wakeup_sources_walk_next(wakeup_source*)' [A] 'function wakeup_source* wakeup_sources_walk_start()' Bug: 223539817 Change-Id: I8a57f4730130a4ee8675161284a20a983350140e Signed-off-by: wuweizhong --- android/abi_gki_aarch64.xml | 742 +++++++++++++++++---------------- android/abi_gki_aarch64_xiaomi | 6 + 2 files changed, 386 insertions(+), 362 deletions(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index ee4eba529ba0..fe8b0982b6d9 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -5774,6 +5774,10 @@ + + + + @@ -10274,24 +10278,24 @@ - + - + - + - + - + - + - + @@ -18171,42 +18175,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -20659,21 +20663,21 @@ - + - + - + - + - + - + @@ -22989,21 +22993,21 @@ - + - + - + - + - + - + @@ -26460,15 +26464,15 @@ - + - + - + - + @@ -26747,33 +26751,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -29538,12 +29542,12 @@ - + - + - + @@ -29660,21 +29664,21 @@ - + - + - + - + - + - + @@ -32139,15 +32143,15 @@ - + - + - + - + @@ -32679,30 +32683,30 @@ - + - + - + - + - + - + - + - + - + @@ -39457,30 +39461,30 @@ - + - + - + - + - + - + - + - + - + @@ -45320,9 +45324,9 @@ - + - + @@ -45472,21 +45476,21 @@ - + - + - + - + - + - + @@ -45726,15 +45730,15 @@ - + - + - + - + @@ -47050,7 +47054,7 @@ - + @@ -51201,54 +51205,54 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -51814,18 +51818,18 @@ - + - + - + - + - + @@ -52509,42 +52513,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -53410,12 +53414,12 @@ - + - + - + @@ -56233,21 +56237,21 @@ - + - + - + - + - + - + @@ -56730,15 +56734,15 @@ - + - + - + - + @@ -57692,36 +57696,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -60461,24 +60465,24 @@ - + - + - + - + - + - + - + @@ -68201,15 +68205,15 @@ - + - + - + - + @@ -70141,15 +70145,15 @@ - + - + - + - + @@ -79798,7 +79802,7 @@ - + @@ -80934,228 +80938,228 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -81169,15 +81173,15 @@ - + - + - + - + @@ -86537,21 +86541,21 @@ - + - + - + - + - + - + @@ -88766,9 +88770,9 @@ - + - + @@ -89142,21 +89146,21 @@ - + - + - + - + - + - + @@ -91656,15 +91660,15 @@ - + - + - + - + @@ -93144,24 +93148,24 @@ - + - + - + - + - + - + - + @@ -95982,36 +95986,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -99132,63 +99136,63 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -99467,18 +99471,18 @@ - + - + - + - + - + @@ -102748,15 +102752,15 @@ - + - + - + - + @@ -109018,27 +109022,27 @@ - + - + - + - + - + - + - + - + @@ -118080,12 +118084,12 @@ - - - - - - + + + + + + @@ -126998,10 +127002,10 @@ - - - - + + + + @@ -144387,6 +144391,20 @@ + + + + + + + + + + + + + + @@ -144571,15 +144589,15 @@ - - - - + + + + - - - + + + @@ -144603,19 +144621,19 @@ - - - + + + - - - - + + + + @@ -144628,13 +144646,13 @@ - - - + + + - - + + @@ -144652,9 +144670,9 @@ - - - + + + @@ -144677,14 +144695,14 @@ - - - + + + - - - + + + @@ -144705,8 +144723,8 @@ - - + + @@ -144714,9 +144732,9 @@ - - - + + + @@ -144729,8 +144747,8 @@ - - + + diff --git a/android/abi_gki_aarch64_xiaomi b/android/abi_gki_aarch64_xiaomi index b6b8934aa985..a162683958cd 100644 --- a/android/abi_gki_aarch64_xiaomi +++ b/android/abi_gki_aarch64_xiaomi @@ -194,3 +194,9 @@ #extend_reclaim.ko try_to_free_mem_cgroup_pages + +##required by xm_power_debug.ko module + wakeup_sources_read_lock + wakeup_sources_read_unlock + wakeup_sources_walk_start + wakeup_sources_walk_next From 8dbcaf63b2bfaeef5592a94d5070ef3693e4db01 Mon Sep 17 00:00:00 2001 From: Will Shiu Date: Tue, 8 Mar 2022 22:36:47 +0800 Subject: [PATCH 22/80] ANDROID: dm kcopyd: Use reserved memory for the copy buffer MediaTek needs to use specific reserved memory for merge buffer to ensure the data correctness during merge period. The specific path would be enabled only if bootarg "mtk_kcopyd_quirk" is well defined to be isolated for MediaTek platform only. The format of bootarg would be: mtk_kcopyd_quirk=mediatek,dm_ota where "dm_ota" is exact the name of the reserved memory. Bug: 223346425 Change-Id: I2b295ca8c0cea65146077324c58ac17c05fe0099 Signed-off-by: Will Shiu Signed-off-by: Stanley Chu Signed-off-by: Akilesh Kailash --- drivers/md/dm-kcopyd.c | 131 ++++++++++++++++++++++++++++++++++++-- drivers/md/dm-snap.c | 3 +- include/linux/dm-kcopyd.h | 1 + 3 files changed, 127 insertions(+), 8 deletions(-) diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 1bbe4a34ef4c..9594367a4ef0 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -39,6 +41,105 @@ static unsigned kcopyd_subjob_size_kb = DEFAULT_SUB_JOB_SIZE_KB; module_param(kcopyd_subjob_size_kb, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(kcopyd_subjob_size_kb, "Sub-job size for dm-kcopyd clients"); +static bool rsm_enabled; +static phys_addr_t rsm_mem_base, rsm_mem_size; + +#ifndef MODULE +static DEFINE_SPINLOCK(rsm_lock); +static int *rsm_mem; +static int rsm_page_cnt; +static int rsm_tbl_idx; +static struct reserved_mem *rmem; + +static void __init kcopyd_rsm_init(void) +{ + static struct device_node *rsm_node; + int ret = 0; + + if (!rsm_enabled) + return; + + rsm_node = of_find_compatible_node(NULL, NULL, "mediatek,dm_ota"); + if (!rsm_node) { + ret = -ENODEV; + goto out; + } + + rmem = of_reserved_mem_lookup(rsm_node); + if (!rmem) { + ret = -EINVAL; + goto out_put_node; + } + + rsm_mem_base = rmem->base; + rsm_mem_size = rmem->size; + rsm_page_cnt = rsm_mem_size / PAGE_SIZE; + rsm_mem = kcalloc(rsm_page_cnt, sizeof(int), GFP_KERNEL); + if (!rsm_mem) + ret = -ENOMEM; + +out_put_node: + of_node_put(rsm_node); +out: + if (ret) + pr_warn("kcopyd: failed to init rsm: %d", ret); +} + +static int __init kcopyd_rsm_enable(char *str) +{ + rsm_enabled = true; + + return 0; +} +early_param("mtk_kcopyd_quirk", kcopyd_rsm_enable); + +static void kcopyd_rsm_get_page(struct page **p) +{ + int i; + unsigned long flags; + + *p = NULL; + spin_lock_irqsave(&rsm_lock, flags); + for (i = 0 ; i < rsm_page_cnt ; i++) { + rsm_tbl_idx = (rsm_tbl_idx + 1 == rsm_page_cnt) ? 0 : rsm_tbl_idx + 1; + + if (rsm_mem[rsm_tbl_idx] == 0) { + rsm_mem[rsm_tbl_idx] = 1; + *p = virt_to_page(phys_to_virt(rsm_mem_base + PAGE_SIZE + * rsm_tbl_idx)); + break; + } + } + spin_unlock_irqrestore(&rsm_lock, flags); +} + +static void kcopyd_rsm_drop_page(struct page **p) +{ + u64 off; + unsigned long flags; + + if (*p) { + off = page_to_phys(*p) - rsm_mem_base; + spin_lock_irqsave(&rsm_lock, flags); + rsm_mem[off >> PAGE_SHIFT] = 0; + spin_unlock_irqrestore(&rsm_lock, flags); + *p = NULL; + } +} + +static void kcopyd_rsm_destroy(void) +{ + if (rsm_enabled) + kfree(rsm_mem); +} + +#else +#define kcopyd_rsm_destroy(...) +#define kcopyd_rsm_drop_page(...) +#define kcopyd_rsm_get_page(...) +#define kcopyd_rsm_init(...) +#endif + static unsigned dm_get_kcopyd_subjob_size(void) { unsigned sub_job_size_kb; @@ -211,7 +312,7 @@ static void wake(struct dm_kcopyd_client *kc) /* * Obtain one page for the use of kcopyd. */ -static struct page_list *alloc_pl(gfp_t gfp) +static struct page_list *alloc_pl(gfp_t gfp, unsigned long job_flags) { struct page_list *pl; @@ -219,7 +320,12 @@ static struct page_list *alloc_pl(gfp_t gfp) if (!pl) return NULL; - pl->page = alloc_page(gfp); + if (rsm_enabled && test_bit(DM_KCOPYD_SNAP_MERGE, &job_flags)) { + kcopyd_rsm_get_page(&pl->page); + } else { + pl->page = alloc_page(gfp); + } + if (!pl->page) { kfree(pl); return NULL; @@ -230,7 +336,14 @@ static struct page_list *alloc_pl(gfp_t gfp) static void free_pl(struct page_list *pl) { - __free_page(pl->page); + struct page *p = pl->page; + phys_addr_t pa = page_to_phys(p); + + if (rsm_enabled && pa >= rsm_mem_base && pa < rsm_mem_base + rsm_mem_size) + kcopyd_rsm_drop_page(&pl->page); + else + __free_page(pl->page); + kfree(pl); } @@ -258,14 +371,15 @@ static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl) } static int kcopyd_get_pages(struct dm_kcopyd_client *kc, - unsigned int nr, struct page_list **pages) + unsigned int nr, struct page_list **pages, + unsigned long job_flags) { struct page_list *pl; *pages = NULL; do { - pl = alloc_pl(__GFP_NOWARN | __GFP_NORETRY | __GFP_KSWAPD_RECLAIM); + pl = alloc_pl(__GFP_NOWARN | __GFP_NORETRY | __GFP_KSWAPD_RECLAIM, job_flags); if (unlikely(!pl)) { /* Use reserved pages */ pl = kc->pages; @@ -309,7 +423,7 @@ static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned nr_pages) struct page_list *pl = NULL, *next; for (i = 0; i < nr_pages; i++) { - next = alloc_pl(GFP_KERNEL); + next = alloc_pl(GFP_KERNEL, 0); if (!next) { if (pl) drop_pages(pl); @@ -395,6 +509,8 @@ int __init dm_kcopyd_init(void) zero_page_list.next = &zero_page_list; zero_page_list.page = ZERO_PAGE(0); + kcopyd_rsm_init(); + return 0; } @@ -402,6 +518,7 @@ void dm_kcopyd_exit(void) { kmem_cache_destroy(_job_cache); _job_cache = NULL; + kcopyd_rsm_destroy(); } /* @@ -586,7 +703,7 @@ static int run_pages_job(struct kcopyd_job *job) int r; unsigned nr_pages = dm_div_up(job->dests[0].count, PAGE_SIZE >> 9); - r = kcopyd_get_pages(job->kc, nr_pages, &job->pages); + r = kcopyd_get_pages(job->kc, nr_pages, &job->pages, job->flags); if (!r) { /* this job is ready for io */ push(&job->kc->io_jobs, job); diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c index 41735a25d50a..b2d8b37b202d 100644 --- a/drivers/md/dm-snap.c +++ b/drivers/md/dm-snap.c @@ -1117,7 +1117,8 @@ static void snapshot_merge_next_chunks(struct dm_snapshot *s) for (i = 0; i < linear_chunks; i++) __check_for_conflicting_io(s, old_chunk + i); - dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s); + dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 1 << DM_KCOPYD_SNAP_MERGE, + merge_callback, s); return; shut: diff --git a/include/linux/dm-kcopyd.h b/include/linux/dm-kcopyd.h index e42de7750c88..3a594ab9b563 100644 --- a/include/linux/dm-kcopyd.h +++ b/include/linux/dm-kcopyd.h @@ -21,6 +21,7 @@ #define DM_KCOPYD_IGNORE_ERROR 1 #define DM_KCOPYD_WRITE_SEQ 2 +#define DM_KCOPYD_SNAP_MERGE 3 struct dm_kcopyd_throttle { unsigned throttle; From cd6e5d5d7d0338fbfe58010f7dde0a521db6b681 Mon Sep 17 00:00:00 2001 From: Vijayanand Jitta Date: Fri, 4 Mar 2022 19:49:08 +0530 Subject: [PATCH 23/80] ANDROID: mm/slub: Fix Kasan issue with for_each_object_track In for_each_object_track we go through meta data of the slab object in function(fn), and as a result false postive out-of-bound access is reported by kasan. Fix this by wrapping that function call with metadata_access_enable/disable. Bug: 222651868 Fixes: ee8d2c7884a00 ("ANDROID: mm: add get_each_object_track function") Change-Id: Ifb4241a9c3e397a52759d467aa267d1297e297dd Signed-off-by: Vijayanand Jitta --- mm/slub.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/slub.c b/mm/slub.c index 229136a8ef70..135d1be1c2ac 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -599,7 +599,9 @@ unsigned long get_each_object_track(struct kmem_cache *s, slab_lock(page); for_each_object(p, s, page_address(page), page->objects) { t = get_track(s, p, alloc); + metadata_access_enable(); ret = fn(s, p, t, private); + metadata_access_disable(); if (ret < 0) break; num_track += 1; From 8248a3e758d97289484e96cf1e7f1dab82db06d9 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 14 Feb 2022 10:16:57 +0100 Subject: [PATCH 24/80] BACKPORT: sched: Fix yet more sched_fork() races Where commit 4ef0c5c6b5ba ("kernel/sched: Fix sched_fork() access an invalid sched_task_group") fixed a fork race vs cgroup, it opened up a race vs syscalls by not placing the task on the runqueue before it gets exposed through the pidhash. Commit 13765de8148f ("sched/fair: Fix fault in reweight_entity") is trying to fix a single instance of this, instead fix the whole class of issues, effectively reverting this commit. Change-Id: If8d8fc38f2ab1334c45e530aaaedfc7b2825a7e5 Fixes: 4ef0c5c6b5ba ("kernel/sched: Fix sched_fork() access an invalid sched_task_group") Reported-by: Linus Torvalds Signed-off-by: Peter Zijlstra (Intel) Tested-by: Tadeusz Struk Tested-by: Zhang Qiao Tested-by: Dietmar Eggemann Link: https://lkml.kernel.org/r/YgoeCbwj5mbCR0qA@hirez.programming.kicks-ass.net BUG: 221793230 (cherry picked from commit b1e8206582f9d680cff7d04828708c8b6ab32957) [quic_ashayj: Resolved minor merge conflicts] Signed-off-by: Ashay Jaiswal --- include/linux/sched/task.h | 4 ++-- kernel/fork.c | 13 ++++++++++++- kernel/sched/core.c | 35 ++++++++++++++++++++++------------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index 24cacb1ca654..198c9ae7ebc2 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -55,8 +55,8 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); extern void init_idle(struct task_struct *idle, int cpu); extern int sched_fork(unsigned long clone_flags, struct task_struct *p); -extern void sched_post_fork(struct task_struct *p, - struct kernel_clone_args *kargs); +extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs); +extern void sched_post_fork(struct task_struct *p); extern void sched_dead(struct task_struct *p); void __noreturn do_task_dead(void); diff --git a/kernel/fork.c b/kernel/fork.c index 66065e013958..a3c4cd5590de 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2249,6 +2249,17 @@ static __latent_entropy struct task_struct *copy_process( if (retval) goto bad_fork_put_pidfd; + /* + * Now that the cgroups are pinned, re-clone the parent cgroup and put + * the new task on the correct runqueue. All this *before* the task + * becomes visible. + * + * This isn't part of ->can_fork() because while the re-cloning is + * cgroup specific, it unconditionally needs to place the task on a + * runqueue. + */ + sched_cgroup_fork(p, args); + /* * From this point on we must avoid any synchronous user-space * communication until we take the tasklist-lock. In particular, we do @@ -2357,7 +2368,7 @@ static __latent_entropy struct task_struct *copy_process( write_unlock_irq(&tasklist_lock); proc_fork_connector(p); - sched_post_fork(p, args); + sched_post_fork(p); cgroup_post_fork(p, args); perf_event_fork(p); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 15f51cccb8bd..ab29f97f4866 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -880,9 +880,8 @@ int tg_nop(struct task_group *tg, void *data) } #endif -static void set_load_weight(struct task_struct *p) +static void set_load_weight(struct task_struct *p, bool update_load) { - bool update_load = !(READ_ONCE(p->state) & TASK_NEW); int prio = p->static_prio - MAX_RT_PRIO; struct load_weight *load = &p->se.load; @@ -3485,7 +3484,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) p->static_prio = NICE_TO_PRIO(0); p->prio = p->normal_prio = p->static_prio; - set_load_weight(p); + set_load_weight(p, false); /* * We don't need the reset flag anymore after the fork. It has @@ -3504,6 +3503,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) init_entity_runnable_average(&p->se); trace_android_rvh_finish_prio_fork(p); + #ifdef CONFIG_SCHED_INFO if (likely(sched_info_on())) memset(&p->sched_info, 0, sizeof(p->sched_info)); @@ -3519,18 +3519,24 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) return 0; } -void sched_post_fork(struct task_struct *p, struct kernel_clone_args *kargs) +void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) { unsigned long flags; -#ifdef CONFIG_CGROUP_SCHED - struct task_group *tg; -#endif + /* + * Because we're not yet on the pid-hash, p->pi_lock isn't strictly + * required yet, but lockdep gets upset if rules are violated. + */ raw_spin_lock_irqsave(&p->pi_lock, flags); #ifdef CONFIG_CGROUP_SCHED - tg = container_of(kargs->cset->subsys[cpu_cgrp_id], - struct task_group, css); - p->sched_task_group = autogroup_task_group(p, tg); + if (1) { + struct task_group *tg; + + tg = container_of(kargs->cset->subsys[cpu_cgrp_id], + struct task_group, css); + tg = autogroup_task_group(p, tg); + p->sched_task_group = tg; + } #endif rseq_migrate(p); /* @@ -3541,7 +3547,10 @@ void sched_post_fork(struct task_struct *p, struct kernel_clone_args *kargs) if (p->sched_class->task_fork) p->sched_class->task_fork(p); raw_spin_unlock_irqrestore(&p->pi_lock, flags); +} +void sched_post_fork(struct task_struct *p) +{ uclamp_post_fork(p); } @@ -5253,7 +5262,7 @@ void set_user_nice(struct task_struct *p, long nice) put_prev_task(rq, p); p->static_prio = NICE_TO_PRIO(nice); - set_load_weight(p); + set_load_weight(p, true); old_prio = p->prio; p->prio = effective_prio(p); @@ -5427,7 +5436,7 @@ static void __setscheduler_params(struct task_struct *p, */ p->rt_priority = attr->sched_priority; p->normal_prio = normal_prio(p); - set_load_weight(p); + set_load_weight(p, true); } /* @@ -7570,7 +7579,7 @@ void __init sched_init(void) atomic_set(&rq->nr_iowait, 0); } - set_load_weight(&init_task); + set_load_weight(&init_task, false); /* * The boot idle thread does lazy MMU switching as well: From fe75d583870fe3ced95c4aab05be06c3ed1d1d86 Mon Sep 17 00:00:00 2001 From: Naina Mehta Date: Fri, 4 Mar 2022 14:54:41 +0530 Subject: [PATCH 25/80] ANDROID: fault: Add vendor hook for TLB conflict Add android_vh_handle_tlb_conf vendor hook to gracefully handle TLB conflict. Leaf changes summary: 1 artifact changed Changed leaf types summary: 0 leaf type changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 0 Added function Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 1 Added variable 1 Added variable: [A] 'tracepoint __tracepoint_android_vh_handle_tlb_conf' Bug: 222637802 Change-Id: I3d6814cd950e77e1a15ce444764078692724cd0a Signed-off-by: Naina Mehta --- android/abi_gki_aarch64.xml | 2 ++ android/abi_gki_aarch64_qcom | 1 + arch/arm64/include/asm/esr.h | 1 + arch/arm64/mm/fault.c | 6 +++++- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/fault.h | 4 ++++ 6 files changed, 14 insertions(+), 1 deletion(-) diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index fe8b0982b6d9..caaf70ce713b 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -6049,6 +6049,7 @@ + @@ -117500,6 +117501,7 @@ + diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index cd8a90f237a6..bcfc4c9ef782 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -2688,6 +2688,7 @@ __tracepoint_android_vh_ftrace_size_check __tracepoint_android_vh_gic_resume __tracepoint_android_vh_gpio_block_read + __tracepoint_android_vh_handle_tlb_conf __tracepoint_android_vh_iommu_setup_dma_ops __tracepoint_android_vh_ipi_stop __tracepoint_android_vh_jiffies_update diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 8f59bbeba7a7..f2b6a8f90d57 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -112,6 +112,7 @@ #define ESR_ELx_FSC_ACCESS (0x08) #define ESR_ELx_FSC_FAULT (0x04) #define ESR_ELx_FSC_PERM (0x0C) +#define ESR_ELx_FSC_TLBCONF (0x30) /* ISS field definitions for Data Aborts */ #define ESR_ELx_ISV_SHIFT (24) diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 37806735f722..cbec7bc70095 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -711,7 +711,11 @@ static int do_alignment_fault(unsigned long far, unsigned int esr, static int do_bad(unsigned long far, unsigned int esr, struct pt_regs *regs) { - return 1; /* "fault" */ + unsigned long addr = untagged_addr(far); + int ret = 1; + + trace_android_vh_handle_tlb_conf(addr, esr, &ret); + return ret; } static int do_sea(unsigned long far, unsigned int esr, struct pt_regs *regs) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 6492e896e61f..3e26777a54c2 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -392,3 +392,4 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_free_proc); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_thread_release); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_has_work_ilocked); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_read_done); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_handle_tlb_conf); diff --git a/include/trace/hooks/fault.h b/include/trace/hooks/fault.h index 1daf2d250f31..2c4176001ead 100644 --- a/include/trace/hooks/fault.h +++ b/include/trace/hooks/fault.h @@ -29,6 +29,10 @@ DECLARE_RESTRICTED_HOOK(android_rvh_do_sp_pc_abort, TP_ARGS(regs, esr, addr, user), TP_CONDITION(!user)); +DECLARE_HOOK(android_vh_handle_tlb_conf, + TP_PROTO(unsigned long addr, unsigned int esr, int *ret), + TP_ARGS(addr, esr, ret)); + /* macro versions of hooks are no longer required */ #endif /* _TRACE_HOOK_FAULT_H */ From 4ebb639f0d1ce574c9f6541c2b91acb0457c4f5d Mon Sep 17 00:00:00 2001 From: Minchan Kim Date: Wed, 9 Mar 2022 08:54:37 -0800 Subject: [PATCH 26/80] ANDROID: mm: page_pinner: fix build warning Fix build warning below. > All warnings (new ones prefixed by >>): > > >> mm/page_pinner.c:304:22: warning: variable 'page_pinner' set but not used [-Wunused-but-set-variable] > struct page_pinner *page_pinner; > ^ > 1 warning generated. > Fixes: ddc4a4879735 ("ANDROID: mm: page_pinner: introduce failure_tracking feature") Reported-by: kernel test robot Signed-off-by: Minchan Kim Change-Id: I02cfd98f9c33745920f883d5e2e2bc2730f662a2 --- mm/page_pinner.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/mm/page_pinner.c b/mm/page_pinner.c index f4a141aafac6..8bccb54fbbd3 100644 --- a/mm/page_pinner.c +++ b/mm/page_pinner.c @@ -328,7 +328,6 @@ void __dump_page_pinner(struct page *page) void __page_pinner_migration_failed(struct page *page) { struct page_ext *page_ext = lookup_page_ext(page); - struct page_pinner *page_pinner; struct captured_pinner record; unsigned long flags; unsigned int idx; @@ -336,7 +335,6 @@ void __page_pinner_migration_failed(struct page *page) if (unlikely(!page_ext)) return; - page_pinner = get_page_pinner(page_ext); if (!test_bit(PAGE_EXT_PINNER_MIGRATION_FAILED, &page_ext->flags)) return; From f3ca80cced4e447653b51be876c026c337d84702 Mon Sep 17 00:00:00 2001 From: Lee Jones Date: Fri, 11 Mar 2022 14:19:59 +0000 Subject: [PATCH 27/80] ANDROID: dm-bow: Protect Ranges fetched and erased from the RB tree Bug: 195565510 Change-Id: Ic87a27d3ffe51b5290a7ff7a87fddd8305fd42af Signed-off-by: Lee Jones --- drivers/md/dm-bow.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/md/dm-bow.c b/drivers/md/dm-bow.c index cfd1fa63ff97..e666ca01d220 100644 --- a/drivers/md/dm-bow.c +++ b/drivers/md/dm-bow.c @@ -236,6 +236,7 @@ static void set_type(struct bow_context *bc, struct bow_range **br, int type) (*br)->type = type; + mutex_lock(&bc->ranges_lock); if (next->type == type) { if (type == TRIMMED) list_del(&next->trimmed_list); @@ -249,6 +250,7 @@ static void set_type(struct bow_context *bc, struct bow_range **br, int type) rb_erase(&(*br)->node, &bc->ranges); kfree(*br); } + mutex_unlock(&bc->ranges_lock); *br = NULL; } @@ -599,6 +601,7 @@ static void dm_bow_dtr(struct dm_target *ti) struct bow_context *bc = (struct bow_context *) ti->private; struct kobject *kobj; + mutex_lock(&bc->ranges_lock); while (rb_first(&bc->ranges)) { struct bow_range *br = container_of(rb_first(&bc->ranges), struct bow_range, node); @@ -606,6 +609,8 @@ static void dm_bow_dtr(struct dm_target *ti) rb_erase(&br->node, &bc->ranges); kfree(br); } + mutex_unlock(&bc->ranges_lock); + if (bc->workqueue) destroy_workqueue(bc->workqueue); if (bc->bufio) @@ -1181,6 +1186,7 @@ static void dm_bow_tablestatus(struct dm_target *ti, char *result, return; } + mutex_lock(&bc->ranges_lock); for (i = rb_first(&bc->ranges); i; i = rb_next(i)) { struct bow_range *br = container_of(i, struct bow_range, node); @@ -1188,11 +1194,11 @@ static void dm_bow_tablestatus(struct dm_target *ti, char *result, readable_type[br->type], (unsigned long long)br->sector); if (result >= end) - return; + goto unlock; result += scnprintf(result, end - result, "\n"); if (result >= end) - return; + goto unlock; if (br->type == TRIMMED) ++trimmed_range_count; @@ -1214,19 +1220,22 @@ static void dm_bow_tablestatus(struct dm_target *ti, char *result, if (!rb_next(i)) { scnprintf(result, end - result, "\nERROR: Last range not of type TOP"); - return; + goto unlock; } if (br->sector > range_top(br)) { scnprintf(result, end - result, "\nERROR: sectors out of order"); - return; + goto unlock; } } if (trimmed_range_count != trimmed_list_length) scnprintf(result, end - result, "\nERROR: not all trimmed ranges in trimmed list"); + +unlock: + mutex_unlock(&bc->ranges_lock); } static void dm_bow_status(struct dm_target *ti, status_type_t type, From f27f62fecd57fb318f3f544ced091b68cd249077 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 11 May 2021 22:35:17 +0200 Subject: [PATCH 28/80] UPSTREAM: bpf: Add kconfig knob for disabling unpriv bpf by default commit 08389d888287c3823f80b0216766b71e17f0aba5 upstream. Add a kconfig knob which allows for unprivileged bpf to be disabled by default. If set, the knob sets /proc/sys/kernel/unprivileged_bpf_disabled to value of 2. This still allows a transition of 2 -> {0,1} through an admin. Similarly, this also still keeps 1 -> {1} behavior intact, so that once set to permanently disabled, it cannot be undone aside from a reboot. We've also added extra2 with max of 2 for the procfs handler, so that an admin still has a chance to toggle between 0 <-> 2. Either way, as an additional alternative, applications can make use of CAP_BPF that we added a while ago. Bug: 215557547 Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/74ec548079189e4e4dffaeb42b8987bb3c852eee.1620765074.git.daniel@iogearbox.net Cc: Salvatore Bonaccorso Signed-off-by: Greg Kroah-Hartman (cherry picked from commit 8c15bfb36a442d63aec74a3379cb7a197f3e5f99) Signed-off-by: Greg Kroah-Hartman Change-Id: I6d80853f0bd2c8618d956d967681c97b931a6137 --- Documentation/admin-guide/sysctl/kernel.rst | 17 +++++++++--- init/Kconfig | 10 +++++++ kernel/bpf/syscall.c | 3 ++- kernel/sysctl.c | 29 +++++++++++++++++---- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst index d4b32cc32bb7..7d5e8a67c775 100644 --- a/Documentation/admin-guide/sysctl/kernel.rst +++ b/Documentation/admin-guide/sysctl/kernel.rst @@ -1457,11 +1457,22 @@ unprivileged_bpf_disabled ========================= Writing 1 to this entry will disable unprivileged calls to ``bpf()``; -once disabled, calling ``bpf()`` without ``CAP_SYS_ADMIN`` will return -``-EPERM``. +once disabled, calling ``bpf()`` without ``CAP_SYS_ADMIN`` or ``CAP_BPF`` +will return ``-EPERM``. Once set to 1, this can't be cleared from the +running kernel anymore. -Once set, this can't be cleared. +Writing 2 to this entry will also disable unprivileged calls to ``bpf()``, +however, an admin can still change this setting later on, if needed, by +writing 0 or 1 to this entry. +If ``BPF_UNPRIV_DEFAULT_OFF`` is enabled in the kernel config, then this +entry will default to 2 instead of 0. + += ============================================================= +0 Unprivileged calls to ``bpf()`` are enabled +1 Unprivileged calls to ``bpf()`` are disabled without recovery +2 Unprivileged calls to ``bpf()`` are disabled += ============================================================= watchdog ======== diff --git a/init/Kconfig b/init/Kconfig index 4413af4302ea..c08acd288f9f 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1738,6 +1738,16 @@ config BPF_JIT_DEFAULT_ON def_bool ARCH_WANT_DEFAULT_BPF_JIT || BPF_JIT_ALWAYS_ON depends on HAVE_EBPF_JIT && BPF_JIT +config BPF_UNPRIV_DEFAULT_OFF + bool "Disable unprivileged BPF by default" + depends on BPF_SYSCALL + help + Disables unprivileged BPF by default by setting the corresponding + /proc/sys/kernel/unprivileged_bpf_disabled knob to 2. An admin can + still reenable it by setting it to 0 later on, or permanently + disable it by setting it to 1 (from which no other transition to + 0 is possible anymore). + source "kernel/bpf/preload/Kconfig" config USERFAULTFD diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 7a043aef894c..27f86a042bbf 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -52,7 +52,8 @@ static DEFINE_SPINLOCK(map_idr_lock); static DEFINE_IDR(link_idr); static DEFINE_SPINLOCK(link_idr_lock); -int sysctl_unprivileged_bpf_disabled __read_mostly; +int sysctl_unprivileged_bpf_disabled __read_mostly = + IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0; static const struct bpf_map_ops * const bpf_map_types[] = { #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 7aed10c7dea9..77228869268d 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -236,7 +236,27 @@ static int bpf_stats_handler(struct ctl_table *table, int write, mutex_unlock(&bpf_stats_enabled_mutex); return ret; } -#endif + +static int bpf_unpriv_handler(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, unpriv_enable = *(int *)table->data; + bool locked_state = unpriv_enable == 1; + struct ctl_table tmp = *table; + + if (write && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + tmp.data = &unpriv_enable; + ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos); + if (write && !ret) { + if (locked_state && unpriv_enable != 1) + return -EPERM; + *(int *)table->data = unpriv_enable; + } + return ret; +} +#endif /* CONFIG_BPF_SYSCALL && CONFIG_SYSCTL */ /* * /proc/sys support @@ -2629,10 +2649,9 @@ static struct ctl_table kern_table[] = { .data = &sysctl_unprivileged_bpf_disabled, .maxlen = sizeof(sysctl_unprivileged_bpf_disabled), .mode = 0644, - /* only handle a transition from default "0" to "1" */ - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ONE, - .extra2 = SYSCTL_ONE, + .proc_handler = bpf_unpriv_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &two, }, { .procname = "bpf_stats_enabled", From 414a6076acfba80c731876a87661be7cc709bf31 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 Oct 2021 14:01:46 +0200 Subject: [PATCH 29/80] UPSTREAM: x86,bugs: Unconditionally allow spectre_v2=retpoline,amd commit f8a66d608a3e471e1202778c2a36cbdc96bae73b upstream. Currently Linux prevents usage of retpoline,amd on !AMD hardware, this is unfriendly and gets in the way of testing. Remove this restriction. Bug: 215557547 Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Borislav Petkov Acked-by: Josh Poimboeuf Tested-by: Alexei Starovoitov Link: https://lore.kernel.org/r/20211026120310.487348118@infradead.org Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I9f683ceb14f75bb1c5ede1b75ea7a446e2bcd5f6 --- arch/x86/kernel/cpu/bugs.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index d41b70fe4918..d4e7a8755834 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -845,13 +845,6 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) return SPECTRE_V2_CMD_AUTO; } - if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD && - boot_cpu_data.x86_vendor != X86_VENDOR_HYGON && - boot_cpu_data.x86_vendor != X86_VENDOR_AMD) { - pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n"); - return SPECTRE_V2_CMD_AUTO; - } - spec_v2_print_cond(mitigation_options[i].option, mitigation_options[i].secure); return cmd; From cc9e9aa4e03572564d1ad8b478f13b53181dea8c Mon Sep 17 00:00:00 2001 From: "Peter Zijlstra (Intel)" Date: Wed, 16 Feb 2022 20:57:00 +0100 Subject: [PATCH 30/80] UPSTREAM: x86/speculation: Rename RETPOLINE_AMD to RETPOLINE_LFENCE commit d45476d9832409371537013ebdd8dc1a7781f97a upstream. The RETPOLINE_AMD name is unfortunate since it isn't necessarily AMD only, in fact Hygon also uses it. Furthermore it will likely be sufficient for some Intel processors. Therefore rename the thing to RETPOLINE_LFENCE to better describe what it is. Add the spectre_v2=retpoline,lfence option as an alias to spectre_v2=retpoline,amd to preserve existing setups. However, the output of /sys/devices/system/cpu/vulnerabilities/spectre_v2 will be changed. [ bp: Fix typos, massage. ] Bug: 215557547 Co-developed-by: Josh Poimboeuf Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Borislav Petkov Reviewed-by: Thomas Gleixner [fllinden@amazon.com: backported to 5.10] Signed-off-by: Frank van der Linden Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: If3529d497a73590e869797d3f376c72d5414a318 --- arch/x86/include/asm/cpufeatures.h | 2 +- arch/x86/include/asm/nospec-branch.h | 12 +++++----- arch/x86/kernel/cpu/bugs.c | 29 +++++++++++++++--------- tools/arch/x86/include/asm/cpufeatures.h | 2 +- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index dad350d42ecf..3b407f46f1a0 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -204,7 +204,7 @@ #define X86_FEATURE_SME ( 7*32+10) /* AMD Secure Memory Encryption */ #define X86_FEATURE_PTI ( 7*32+11) /* Kernel Page Table Isolation enabled */ #define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ -#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */ +#define X86_FEATURE_RETPOLINE_LFENCE ( 7*32+13) /* "" Use LFENCE for Spectre variant 2 */ #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */ #define X86_FEATURE_CDP_L2 ( 7*32+15) /* Code and Data Prioritization L2 */ #define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */ diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index cb9ad6b73973..fb6899cb22b0 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -82,7 +82,7 @@ #ifdef CONFIG_RETPOLINE ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \ __stringify(jmp __x86_retpoline_\reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE #else jmp *%\reg #endif @@ -92,7 +92,7 @@ #ifdef CONFIG_RETPOLINE ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \ __stringify(call __x86_retpoline_\reg), X86_FEATURE_RETPOLINE, \ - __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_AMD + __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_LFENCE #else call *%\reg #endif @@ -134,7 +134,7 @@ "lfence;\n" \ ANNOTATE_RETPOLINE_SAFE \ "call *%[thunk_target]\n", \ - X86_FEATURE_RETPOLINE_AMD) + X86_FEATURE_RETPOLINE_LFENCE) # define THUNK_TARGET(addr) [thunk_target] "r" (addr) @@ -164,7 +164,7 @@ "lfence;\n" \ ANNOTATE_RETPOLINE_SAFE \ "call *%[thunk_target]\n", \ - X86_FEATURE_RETPOLINE_AMD) + X86_FEATURE_RETPOLINE_LFENCE) # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) #endif @@ -176,8 +176,8 @@ /* The Spectre V2 mitigation variants */ enum spectre_v2_mitigation { SPECTRE_V2_NONE, - SPECTRE_V2_RETPOLINE_GENERIC, - SPECTRE_V2_RETPOLINE_AMD, + SPECTRE_V2_RETPOLINE, + SPECTRE_V2_LFENCE, SPECTRE_V2_IBRS_ENHANCED, }; diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index d4e7a8755834..b0b50240cb05 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -627,7 +627,7 @@ enum spectre_v2_mitigation_cmd { SPECTRE_V2_CMD_FORCE, SPECTRE_V2_CMD_RETPOLINE, SPECTRE_V2_CMD_RETPOLINE_GENERIC, - SPECTRE_V2_CMD_RETPOLINE_AMD, + SPECTRE_V2_CMD_RETPOLINE_LFENCE, }; enum spectre_v2_user_cmd { @@ -787,8 +787,8 @@ set_mode: static const char * const spectre_v2_strings[] = { [SPECTRE_V2_NONE] = "Vulnerable", - [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline", - [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline", + [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", + [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS", }; @@ -800,7 +800,8 @@ static const struct { { "off", SPECTRE_V2_CMD_NONE, false }, { "on", SPECTRE_V2_CMD_FORCE, true }, { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, - { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false }, + { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, + { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, { "auto", SPECTRE_V2_CMD_AUTO, false }, }; @@ -838,13 +839,19 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) } if ((cmd == SPECTRE_V2_CMD_RETPOLINE || - cmd == SPECTRE_V2_CMD_RETPOLINE_AMD || + cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) && !IS_ENABLED(CONFIG_RETPOLINE)) { pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option); return SPECTRE_V2_CMD_AUTO; } + if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE) && + !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { + pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", mitigation_options[i].option); + return SPECTRE_V2_CMD_AUTO; + } + spec_v2_print_cond(mitigation_options[i].option, mitigation_options[i].secure); return cmd; @@ -879,9 +886,9 @@ static void __init spectre_v2_select_mitigation(void) if (IS_ENABLED(CONFIG_RETPOLINE)) goto retpoline_auto; break; - case SPECTRE_V2_CMD_RETPOLINE_AMD: + case SPECTRE_V2_CMD_RETPOLINE_LFENCE: if (IS_ENABLED(CONFIG_RETPOLINE)) - goto retpoline_amd; + goto retpoline_lfence; break; case SPECTRE_V2_CMD_RETPOLINE_GENERIC: if (IS_ENABLED(CONFIG_RETPOLINE)) @@ -898,17 +905,17 @@ static void __init spectre_v2_select_mitigation(void) retpoline_auto: if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { - retpoline_amd: + retpoline_lfence: if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n"); goto retpoline_generic; } - mode = SPECTRE_V2_RETPOLINE_AMD; - setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD); + mode = SPECTRE_V2_LFENCE; + setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); setup_force_cpu_cap(X86_FEATURE_RETPOLINE); } else { retpoline_generic: - mode = SPECTRE_V2_RETPOLINE_GENERIC; + mode = SPECTRE_V2_RETPOLINE; setup_force_cpu_cap(X86_FEATURE_RETPOLINE); } diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index dad350d42ecf..b58730cc12e8 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -204,7 +204,7 @@ #define X86_FEATURE_SME ( 7*32+10) /* AMD Secure Memory Encryption */ #define X86_FEATURE_PTI ( 7*32+11) /* Kernel Page Table Isolation enabled */ #define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ -#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */ +#define X86_FEATURE_RETPOLINE_LFENCE ( 7*32+13) /* "" Use LFENCEs for Spectre variant 2 */ #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */ #define X86_FEATURE_CDP_L2 ( 7*32+15) /* Code and Data Prioritization L2 */ #define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */ From 1c3e98581b931ab1377cc694143b379c86470545 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 16 Feb 2022 20:57:01 +0100 Subject: [PATCH 31/80] UPSTREAM: x86/speculation: Add eIBRS + Retpoline options commit 1e19da8522c81bf46b335f84137165741e0d82b7 upstream. Thanks to the chaps at VUsec it is now clear that eIBRS is not sufficient, therefore allow enabling of retpolines along with eIBRS. Add spectre_v2=eibrs, spectre_v2=eibrs,lfence and spectre_v2=eibrs,retpoline options to explicitly pick your preferred means of mitigation. Since there's new mitigations there's also user visible changes in /sys/devices/system/cpu/vulnerabilities/spectre_v2 to reflect these new mitigations. [ bp: Massage commit message, trim error messages, do more precise eIBRS mode checking. ] Bug: 215557547 Co-developed-by: Josh Poimboeuf Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Borislav Petkov Reviewed-by: Patrick Colp Reviewed-by: Thomas Gleixner Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I6c81d2e9bf8cd2dbe246935aaf31cdbbe7f9cad3 --- arch/x86/include/asm/nospec-branch.h | 4 +- arch/x86/kernel/cpu/bugs.c | 147 +++++++++++++++++++-------- 2 files changed, 106 insertions(+), 45 deletions(-) diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index fb6899cb22b0..4d0f5386e637 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -178,7 +178,9 @@ enum spectre_v2_mitigation { SPECTRE_V2_NONE, SPECTRE_V2_RETPOLINE, SPECTRE_V2_LFENCE, - SPECTRE_V2_IBRS_ENHANCED, + SPECTRE_V2_EIBRS, + SPECTRE_V2_EIBRS_RETPOLINE, + SPECTRE_V2_EIBRS_LFENCE, }; /* The indirect branch speculation control variants */ diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index b0b50240cb05..3a5c6724ea06 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -628,6 +628,9 @@ enum spectre_v2_mitigation_cmd { SPECTRE_V2_CMD_RETPOLINE, SPECTRE_V2_CMD_RETPOLINE_GENERIC, SPECTRE_V2_CMD_RETPOLINE_LFENCE, + SPECTRE_V2_CMD_EIBRS, + SPECTRE_V2_CMD_EIBRS_RETPOLINE, + SPECTRE_V2_CMD_EIBRS_LFENCE, }; enum spectre_v2_user_cmd { @@ -700,6 +703,13 @@ spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd) return SPECTRE_V2_USER_CMD_AUTO; } +static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode) +{ + return (mode == SPECTRE_V2_EIBRS || + mode == SPECTRE_V2_EIBRS_RETPOLINE || + mode == SPECTRE_V2_EIBRS_LFENCE); +} + static void __init spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) { @@ -767,7 +777,7 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) */ if (!boot_cpu_has(X86_FEATURE_STIBP) || !smt_possible || - spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) + spectre_v2_in_eibrs_mode(spectre_v2_enabled)) return; /* @@ -789,7 +799,9 @@ static const char * const spectre_v2_strings[] = { [SPECTRE_V2_NONE] = "Vulnerable", [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", - [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS", + [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS", + [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE", + [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines", }; static const struct { @@ -803,6 +815,9 @@ static const struct { { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, + { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, + { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, + { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, { "auto", SPECTRE_V2_CMD_AUTO, false }, }; @@ -840,15 +855,29 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) if ((cmd == SPECTRE_V2_CMD_RETPOLINE || cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || - cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) && + cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || + cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && !IS_ENABLED(CONFIG_RETPOLINE)) { - pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option); + pr_err("%s selected but not compiled in. Switching to AUTO select\n", + mitigation_options[i].option); return SPECTRE_V2_CMD_AUTO; } - if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE) && + if ((cmd == SPECTRE_V2_CMD_EIBRS || + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || + cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && + !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { + pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n", + mitigation_options[i].option); + return SPECTRE_V2_CMD_AUTO; + } + + if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || + cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { - pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", mitigation_options[i].option); + pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", + mitigation_options[i].option); return SPECTRE_V2_CMD_AUTO; } @@ -857,6 +886,25 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) return cmd; } +static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) +{ + if (!IS_ENABLED(CONFIG_RETPOLINE)) { + pr_err("Kernel not compiled with retpoline; no mitigation available!"); + return SPECTRE_V2_NONE; + } + + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || + boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { + if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { + pr_err("LFENCE not serializing, switching to generic retpoline\n"); + return SPECTRE_V2_RETPOLINE; + } + return SPECTRE_V2_LFENCE; + } + + return SPECTRE_V2_RETPOLINE; +} + static void __init spectre_v2_select_mitigation(void) { enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); @@ -877,49 +925,60 @@ static void __init spectre_v2_select_mitigation(void) case SPECTRE_V2_CMD_FORCE: case SPECTRE_V2_CMD_AUTO: if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { - mode = SPECTRE_V2_IBRS_ENHANCED; - /* Force it so VMEXIT will restore correctly */ - x86_spec_ctrl_base |= SPEC_CTRL_IBRS; - wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); - goto specv2_set_mode; + mode = SPECTRE_V2_EIBRS; + break; } - if (IS_ENABLED(CONFIG_RETPOLINE)) - goto retpoline_auto; + + mode = spectre_v2_select_retpoline(); break; + case SPECTRE_V2_CMD_RETPOLINE_LFENCE: - if (IS_ENABLED(CONFIG_RETPOLINE)) - goto retpoline_lfence; - break; - case SPECTRE_V2_CMD_RETPOLINE_GENERIC: - if (IS_ENABLED(CONFIG_RETPOLINE)) - goto retpoline_generic; - break; - case SPECTRE_V2_CMD_RETPOLINE: - if (IS_ENABLED(CONFIG_RETPOLINE)) - goto retpoline_auto; - break; - } - pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!"); - return; - -retpoline_auto: - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || - boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { - retpoline_lfence: - if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { - pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n"); - goto retpoline_generic; - } mode = SPECTRE_V2_LFENCE; - setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); - setup_force_cpu_cap(X86_FEATURE_RETPOLINE); - } else { - retpoline_generic: + break; + + case SPECTRE_V2_CMD_RETPOLINE_GENERIC: mode = SPECTRE_V2_RETPOLINE; - setup_force_cpu_cap(X86_FEATURE_RETPOLINE); + break; + + case SPECTRE_V2_CMD_RETPOLINE: + mode = spectre_v2_select_retpoline(); + break; + + case SPECTRE_V2_CMD_EIBRS: + mode = SPECTRE_V2_EIBRS; + break; + + case SPECTRE_V2_CMD_EIBRS_LFENCE: + mode = SPECTRE_V2_EIBRS_LFENCE; + break; + + case SPECTRE_V2_CMD_EIBRS_RETPOLINE: + mode = SPECTRE_V2_EIBRS_RETPOLINE; + break; + } + + if (spectre_v2_in_eibrs_mode(mode)) { + /* Force it so VMEXIT will restore correctly */ + x86_spec_ctrl_base |= SPEC_CTRL_IBRS; + wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); + } + + switch (mode) { + case SPECTRE_V2_NONE: + case SPECTRE_V2_EIBRS: + break; + + case SPECTRE_V2_LFENCE: + case SPECTRE_V2_EIBRS_LFENCE: + setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); + fallthrough; + + case SPECTRE_V2_RETPOLINE: + case SPECTRE_V2_EIBRS_RETPOLINE: + setup_force_cpu_cap(X86_FEATURE_RETPOLINE); + break; } -specv2_set_mode: spectre_v2_enabled = mode; pr_info("%s\n", spectre_v2_strings[mode]); @@ -945,7 +1004,7 @@ specv2_set_mode: * the CPU supports Enhanced IBRS, kernel might un-intentionally not * enable IBRS around firmware calls. */ - if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) { + if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) { setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); pr_info("Enabling Restricted Speculation for firmware calls\n"); } @@ -1621,7 +1680,7 @@ static ssize_t tsx_async_abort_show_state(char *buf) static char *stibp_state(void) { - if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) + if (spectre_v2_in_eibrs_mode(spectre_v2_enabled)) return ""; switch (spectre_v2_user_stibp) { From 3883503747eeb304f8089d10a2c57c6f857ff2bc Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 16 Feb 2022 20:57:02 +0100 Subject: [PATCH 32/80] UPSTREAM: Documentation/hw-vuln: Update spectre doc commit 5ad3eb1132453b9795ce5fd4572b1c18b292cca9 upstream. Update the doc with the new fun. [ bp: Massage commit message. ] Bug: 215557547 Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Borislav Petkov Reviewed-by: Thomas Gleixner [fllinden@amazon.com: backported to 5.10] Signed-off-by: Frank van der Linden Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ief29f666e293df823aab26e7adea6f95466ffa96 --- Documentation/admin-guide/hw-vuln/spectre.rst | 42 +++++++++++++------ .../admin-guide/kernel-parameters.txt | 8 +++- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/Documentation/admin-guide/hw-vuln/spectre.rst b/Documentation/admin-guide/hw-vuln/spectre.rst index e05e581af5cf..4776c112fe7d 100644 --- a/Documentation/admin-guide/hw-vuln/spectre.rst +++ b/Documentation/admin-guide/hw-vuln/spectre.rst @@ -131,6 +131,19 @@ steer its indirect branch speculations to gadget code, and measure the speculative execution's side effects left in level 1 cache to infer the victim's data. +Yet another variant 2 attack vector is for the attacker to poison the +Branch History Buffer (BHB) to speculatively steer an indirect branch +to a specific Branch Target Buffer (BTB) entry, even if the entry isn't +associated with the source address of the indirect branch. Specifically, +the BHB might be shared across privilege levels even in the presence of +Enhanced IBRS. + +Currently the only known real-world BHB attack vector is via +unprivileged eBPF. Therefore, it's highly recommended to not enable +unprivileged eBPF, especially when eIBRS is used (without retpolines). +For a full mitigation against BHB attacks, it's recommended to use +retpolines (or eIBRS combined with retpolines). + Attack scenarios ---------------- @@ -364,13 +377,15 @@ The possible values in this file are: - Kernel status: - ==================================== ================================= - 'Not affected' The processor is not vulnerable - 'Vulnerable' Vulnerable, no mitigation - 'Mitigation: Full generic retpoline' Software-focused mitigation - 'Mitigation: Full AMD retpoline' AMD-specific software mitigation - 'Mitigation: Enhanced IBRS' Hardware-focused mitigation - ==================================== ================================= + ======================================== ================================= + 'Not affected' The processor is not vulnerable + 'Mitigation: None' Vulnerable, no mitigation + 'Mitigation: Retpolines' Use Retpoline thunks + 'Mitigation: LFENCE' Use LFENCE instructions + 'Mitigation: Enhanced IBRS' Hardware-focused mitigation + 'Mitigation: Enhanced IBRS + Retpolines' Hardware-focused + Retpolines + 'Mitigation: Enhanced IBRS + LFENCE' Hardware-focused + LFENCE + ======================================== ================================= - Firmware status: Show if Indirect Branch Restricted Speculation (IBRS) is used to protect against Spectre variant 2 attacks when calling firmware (x86 only). @@ -584,12 +599,13 @@ kernel command line. Specific mitigations can also be selected manually: - retpoline - replace indirect branches - retpoline,generic - google's original retpoline - retpoline,amd - AMD-specific minimal thunk + retpoline auto pick between generic,lfence + retpoline,generic Retpolines + retpoline,lfence LFENCE; indirect branch + retpoline,amd alias for retpoline,lfence + eibrs enhanced IBRS + eibrs,retpoline enhanced IBRS + Retpolines + eibrs,lfence enhanced IBRS + LFENCE Not specifying this option is equivalent to spectre_v2=auto. diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 38614bd62e19..0ab7d4f9dd15 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5046,8 +5046,12 @@ Specific mitigations can also be selected manually: retpoline - replace indirect branches - retpoline,generic - google's original retpoline - retpoline,amd - AMD-specific minimal thunk + retpoline,generic - Retpolines + retpoline,lfence - LFENCE; indirect branch + retpoline,amd - alias for retpoline,lfence + eibrs - enhanced IBRS + eibrs,retpoline - enhanced IBRS + Retpolines + eibrs,lfence - enhanced IBRS + LFENCE Not specifying this option is equivalent to spectre_v2=auto. From bd02dc432905d48c400caf29806f58b87dfaf255 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Fri, 18 Feb 2022 11:49:08 -0800 Subject: [PATCH 33/80] UPSTREAM: x86/speculation: Include unprivileged eBPF status in Spectre v2 mitigation reporting commit 44a3918c8245ab10c6c9719dd12e7a8d291980d8 upstream. With unprivileged eBPF enabled, eIBRS (without retpoline) is vulnerable to Spectre v2 BHB-based attacks. When both are enabled, print a warning message and report it in the 'spectre_v2' sysfs vulnerabilities file. Bug: 215557547 Signed-off-by: Josh Poimboeuf Signed-off-by: Borislav Petkov Reviewed-by: Thomas Gleixner [fllinden@amazon.com: backported to 5.10] Signed-off-by: Frank van der Linden Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ie9ee8c137729aadb4f9ef2be346a86d71eca363b --- arch/x86/kernel/cpu/bugs.c | 35 +++++++++++++++++++++++++++++------ include/linux/bpf.h | 12 ++++++++++++ kernel/sysctl.c | 7 +++++++ 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 3a5c6724ea06..e97ab5ca9358 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -613,6 +614,16 @@ static inline const char *spectre_v2_module_string(void) static inline const char *spectre_v2_module_string(void) { return ""; } #endif +#define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" + +#ifdef CONFIG_BPF_SYSCALL +void unpriv_ebpf_notify(int new_state) +{ + if (spectre_v2_enabled == SPECTRE_V2_EIBRS && !new_state) + pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); +} +#endif + static inline bool match_option(const char *arg, int arglen, const char *opt) { int len = strlen(opt); @@ -957,6 +968,9 @@ static void __init spectre_v2_select_mitigation(void) break; } + if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) + pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); + if (spectre_v2_in_eibrs_mode(mode)) { /* Force it so VMEXIT will restore correctly */ x86_spec_ctrl_base |= SPEC_CTRL_IBRS; @@ -1710,6 +1724,20 @@ static char *ibpb_state(void) return ""; } +static ssize_t spectre_v2_show_state(char *buf) +{ + if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) + return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); + + return sprintf(buf, "%s%s%s%s%s%s\n", + spectre_v2_strings[spectre_v2_enabled], + ibpb_state(), + boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", + stibp_state(), + boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", + spectre_v2_module_string()); +} + static ssize_t srbds_show_state(char *buf) { return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]); @@ -1735,12 +1763,7 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); case X86_BUG_SPECTRE_V2: - return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled], - ibpb_state(), - boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", - stibp_state(), - boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", - spectre_v2_module_string()); + return spectre_v2_show_state(buf); case X86_BUG_SPEC_STORE_BYPASS: return sprintf(buf, "%s\n", ssb_strings[ssb_mode]); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 20db1b8e9763..121004ac573e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1491,6 +1491,12 @@ struct bpf_prog *bpf_prog_by_id(u32 id); struct bpf_link *bpf_link_by_id(u32 id); const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); + +static inline bool unprivileged_ebpf_enabled(void) +{ + return !sysctl_unprivileged_bpf_disabled; +} + #else /* !CONFIG_BPF_SYSCALL */ static inline struct bpf_prog *bpf_prog_get(u32 ufd) { @@ -1685,6 +1691,12 @@ bpf_base_func_proto(enum bpf_func_id func_id) { return NULL; } + +static inline bool unprivileged_ebpf_enabled(void) +{ + return false; +} + #endif /* CONFIG_BPF_SYSCALL */ static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 77228869268d..a4006f6558d1 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -237,6 +237,10 @@ static int bpf_stats_handler(struct ctl_table *table, int write, return ret; } +void __weak unpriv_ebpf_notify(int new_state) +{ +} + static int bpf_unpriv_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { @@ -254,6 +258,9 @@ static int bpf_unpriv_handler(struct ctl_table *table, int write, return -EPERM; *(int *)table->data = unpriv_enable; } + + unpriv_ebpf_notify(unpriv_enable); + return ret; } #endif /* CONFIG_BPF_SYSCALL && CONFIG_SYSCTL */ From c4188388a3c7544fe29722b8c20abf77110087de Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Mon, 28 Feb 2022 11:23:15 -0600 Subject: [PATCH 34/80] UPSTREAM: x86/speculation: Use generic retpoline by default on AMD commit 244d00b5dd4755f8df892c86cab35fb2cfd4f14b upstream. AMD retpoline may be susceptible to speculation. The speculation execution window for an incorrect indirect branch prediction using LFENCE/JMP sequence may potentially be large enough to allow exploitation using Spectre V2. By default, don't use retpoline,lfence on AMD. Instead, use the generic retpoline. Bug: 215557547 Signed-off-by: Kim Phillips Signed-off-by: Borislav Petkov Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ia055414ec9ab24eeebb21e799cc220cce8a1f3f6 --- arch/x86/kernel/cpu/bugs.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index e97ab5ca9358..935a4812a442 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -904,15 +904,6 @@ static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) return SPECTRE_V2_NONE; } - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || - boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { - if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { - pr_err("LFENCE not serializing, switching to generic retpoline\n"); - return SPECTRE_V2_RETPOLINE; - } - return SPECTRE_V2_LFENCE; - } - return SPECTRE_V2_RETPOLINE; } From f1b1f893b481248532f29e5fccc8868ca2062ca6 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Mon, 28 Feb 2022 11:23:16 -0600 Subject: [PATCH 35/80] UPSTREAM: x86/speculation: Update link to AMD speculation whitepaper commit e9b6013a7ce31535b04b02ba99babefe8a8599fa upstream. Update the link to the "Software Techniques for Managing Speculation on AMD Processors" whitepaper. Bug: 215557547 Signed-off-by: Kim Phillips Signed-off-by: Borislav Petkov Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I65521b935a50147f7c694f00f292add8bf2385bc --- Documentation/admin-guide/hw-vuln/spectre.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/hw-vuln/spectre.rst b/Documentation/admin-guide/hw-vuln/spectre.rst index 4776c112fe7d..3a6e1fcad3c0 100644 --- a/Documentation/admin-guide/hw-vuln/spectre.rst +++ b/Documentation/admin-guide/hw-vuln/spectre.rst @@ -60,8 +60,8 @@ privileged data touched during the speculative execution. Spectre variant 1 attacks take advantage of speculative execution of conditional branches, while Spectre variant 2 attacks use speculative execution of indirect branches to leak privileged memory. -See :ref:`[1] ` :ref:`[5] ` :ref:`[7] ` -:ref:`[10] ` :ref:`[11] `. +See :ref:`[1] ` :ref:`[5] ` :ref:`[6] ` +:ref:`[7] ` :ref:`[10] ` :ref:`[11] `. Spectre variant 1 (Bounds Check Bypass) --------------------------------------- @@ -746,7 +746,7 @@ AMD white papers: .. _spec_ref6: -[6] `Software techniques for managing speculation on AMD processors `_. +[6] `Software techniques for managing speculation on AMD processors `_. ARM white papers: From 54a2bd029f34e575926e647293b86f8fce0622be Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Fri, 25 Feb 2022 14:31:49 -0800 Subject: [PATCH 36/80] UPSTREAM: x86/speculation: Warn about Spectre v2 LFENCE mitigation commit eafd987d4a82c7bb5aa12f0e3b4f8f3dea93e678 upstream. With: f8a66d608a3e ("x86,bugs: Unconditionally allow spectre_v2=retpoline,amd") it became possible to enable the LFENCE "retpoline" on Intel. However, Intel doesn't recommend it, as it has some weaknesses compared to retpoline. Now AMD doesn't recommend it either. It can still be left available as a cmdline option. It's faster than retpoline but is weaker in certain scenarios -- particularly SMT, but even non-SMT may be vulnerable in some cases. So just unconditionally warn if the user requests it on the cmdline. [ bp: Massage commit message. ] Bug: 215557547 Signed-off-by: Josh Poimboeuf Signed-off-by: Borislav Petkov Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I6f3a4c7855097b7d3248b5f73d22259d16bbbfb8 --- arch/x86/kernel/cpu/bugs.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 935a4812a442..bade6afeb169 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -614,6 +614,7 @@ static inline const char *spectre_v2_module_string(void) static inline const char *spectre_v2_module_string(void) { return ""; } #endif +#define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" #ifdef CONFIG_BPF_SYSCALL @@ -935,6 +936,7 @@ static void __init spectre_v2_select_mitigation(void) break; case SPECTRE_V2_CMD_RETPOLINE_LFENCE: + pr_err(SPECTRE_V2_LFENCE_MSG); mode = SPECTRE_V2_LFENCE; break; @@ -1717,6 +1719,9 @@ static char *ibpb_state(void) static ssize_t spectre_v2_show_state(char *buf) { + if (spectre_v2_enabled == SPECTRE_V2_LFENCE) + return sprintf(buf, "Vulnerable: LFENCE\n"); + if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); From 9362cd2b478d2247ee54707a7723f204f94ebc32 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Fri, 25 Feb 2022 14:32:28 -0800 Subject: [PATCH 37/80] UPSTREAM: x86/speculation: Warn about eIBRS + LFENCE + Unprivileged eBPF + SMT commit 0de05d056afdb00eca8c7bbb0c79a3438daf700c upstream. The commit 44a3918c8245 ("x86/speculation: Include unprivileged eBPF status in Spectre v2 mitigation reporting") added a warning for the "eIBRS + unprivileged eBPF" combination, which has been shown to be vulnerable against Spectre v2 BHB-based attacks. However, there's no warning about the "eIBRS + LFENCE retpoline + unprivileged eBPF" combo. The LFENCE adds more protection by shortening the speculation window after a mispredicted branch. That makes an attack significantly more difficult, even with unprivileged eBPF. So at least for now the logic doesn't warn about that combination. But if you then add SMT into the mix, the SMT attack angle weakens the effectiveness of the LFENCE considerably. So extend the "eIBRS + unprivileged eBPF" warning to also include the "eIBRS + LFENCE + unprivileged eBPF + SMT" case. [ bp: Massage commit message. ] Bug: 215557547 Suggested-by: Alyssa Milburn Signed-off-by: Josh Poimboeuf Signed-off-by: Borislav Petkov Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I1d99e035c6fc53fac367b0423056dc75237ed430 --- arch/x86/kernel/cpu/bugs.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index bade6afeb169..78b9514a3844 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -616,12 +616,27 @@ static inline const char *spectre_v2_module_string(void) { return ""; } #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" +#define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" #ifdef CONFIG_BPF_SYSCALL void unpriv_ebpf_notify(int new_state) { - if (spectre_v2_enabled == SPECTRE_V2_EIBRS && !new_state) + if (new_state) + return; + + /* Unprivileged eBPF is enabled */ + + switch (spectre_v2_enabled) { + case SPECTRE_V2_EIBRS: pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); + break; + case SPECTRE_V2_EIBRS_LFENCE: + if (sched_smt_active()) + pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); + break; + default: + break; + } } #endif @@ -1081,6 +1096,10 @@ void cpu_bugs_smt_update(void) { mutex_lock(&spec_ctrl_mutex); + if (sched_smt_active() && unprivileged_ebpf_enabled() && + spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) + pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); + switch (spectre_v2_user_stibp) { case SPECTRE_V2_USER_NONE: break; @@ -1723,7 +1742,11 @@ static ssize_t spectre_v2_show_state(char *buf) return sprintf(buf, "Vulnerable: LFENCE\n"); if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) - return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); + return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); + + if (sched_smt_active() && unprivileged_ebpf_enabled() && + spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) + return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled], From 5a64a668020b8658ed89026b397763c3a5c40c75 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 11 Feb 2022 16:45:54 +0000 Subject: [PATCH 38/80] UPSTREAM: ARM: report Spectre v2 status through sysfs commit 9dd78194a3722fa6712192cdd4f7032d45112a9a upstream. As per other architectures, add support for reporting the Spectre vulnerability status via sysfs CPU. Bug: 215557547 Acked-by: Catalin Marinas Signed-off-by: Russell King (Oracle) Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I426186db1fa21e590de65b70ed531fbf086b176d --- arch/arm/include/asm/spectre.h | 28 +++++++ arch/arm/kernel/Makefile | 2 + arch/arm/kernel/spectre.c | 54 +++++++++++++ arch/arm/mm/Kconfig | 1 + arch/arm/mm/proc-v7-bugs.c | 143 ++++++++++++++++++++++++--------- 5 files changed, 188 insertions(+), 40 deletions(-) create mode 100644 arch/arm/include/asm/spectre.h create mode 100644 arch/arm/kernel/spectre.c diff --git a/arch/arm/include/asm/spectre.h b/arch/arm/include/asm/spectre.h new file mode 100644 index 000000000000..8a9019e08dba --- /dev/null +++ b/arch/arm/include/asm/spectre.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_SPECTRE_H +#define __ASM_SPECTRE_H + +enum { + SPECTRE_UNAFFECTED, + SPECTRE_MITIGATED, + SPECTRE_VULNERABLE, +}; + +enum { + __SPECTRE_V2_METHOD_BPIALL, + __SPECTRE_V2_METHOD_ICIALLU, + __SPECTRE_V2_METHOD_SMC, + __SPECTRE_V2_METHOD_HVC, +}; + +enum { + SPECTRE_V2_METHOD_BPIALL = BIT(__SPECTRE_V2_METHOD_BPIALL), + SPECTRE_V2_METHOD_ICIALLU = BIT(__SPECTRE_V2_METHOD_ICIALLU), + SPECTRE_V2_METHOD_SMC = BIT(__SPECTRE_V2_METHOD_SMC), + SPECTRE_V2_METHOD_HVC = BIT(__SPECTRE_V2_METHOD_HVC), +}; + +void spectre_v2_update_state(unsigned int state, unsigned int methods); + +#endif diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index 89e5d864e923..79588b562353 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -106,4 +106,6 @@ endif obj-$(CONFIG_HAVE_ARM_SMCCC) += smccc-call.o +obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += spectre.o + extra-y := $(head-y) vmlinux.lds diff --git a/arch/arm/kernel/spectre.c b/arch/arm/kernel/spectre.c new file mode 100644 index 000000000000..6f6dd1cfd099 --- /dev/null +++ b/arch/arm/kernel/spectre.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include + +ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "Mitigation: __user pointer sanitization\n"); +} + +static unsigned int spectre_v2_state; +static unsigned int spectre_v2_methods; + +void spectre_v2_update_state(unsigned int state, unsigned int method) +{ + if (state > spectre_v2_state) + spectre_v2_state = state; + spectre_v2_methods |= method; +} + +ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const char *method; + + if (spectre_v2_state == SPECTRE_UNAFFECTED) + return sprintf(buf, "%s\n", "Not affected"); + + if (spectre_v2_state != SPECTRE_MITIGATED) + return sprintf(buf, "%s\n", "Vulnerable"); + + switch (spectre_v2_methods) { + case SPECTRE_V2_METHOD_BPIALL: + method = "Branch predictor hardening"; + break; + + case SPECTRE_V2_METHOD_ICIALLU: + method = "I-cache invalidation"; + break; + + case SPECTRE_V2_METHOD_SMC: + case SPECTRE_V2_METHOD_HVC: + method = "Firmware call"; + break; + + default: + method = "Multiple mitigations"; + break; + } + + return sprintf(buf, "Mitigation: %s\n", method); +} diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 423a97dd2f57..0804bd5e619f 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -833,6 +833,7 @@ config CPU_BPREDICT_DISABLE config CPU_SPECTRE bool + select GENERIC_CPU_VULNERABILITIES config HARDEN_BRANCH_PREDICTOR bool "Harden the branch predictor against aliasing attacks" if EXPERT diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c index 114c05ab4dd9..e438e59bb63e 100644 --- a/arch/arm/mm/proc-v7-bugs.c +++ b/arch/arm/mm/proc-v7-bugs.c @@ -6,8 +6,35 @@ #include #include #include +#include #include +#ifdef CONFIG_ARM_PSCI +static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, + ARM_SMCCC_ARCH_WORKAROUND_1, &res); + + switch ((int)res.a0) { + case SMCCC_RET_SUCCESS: + return SPECTRE_MITIGATED; + + case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: + return SPECTRE_UNAFFECTED; + + default: + return SPECTRE_VULNERABLE; + } +} +#else +static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) +{ + return SPECTRE_VULNERABLE; +} +#endif + #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn); @@ -36,13 +63,60 @@ static void __maybe_unused call_hvc_arch_workaround_1(void) arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); } -static void cpu_v7_spectre_init(void) +static unsigned int spectre_v2_install_workaround(unsigned int method) { const char *spectre_v2_method = NULL; int cpu = smp_processor_id(); if (per_cpu(harden_branch_predictor_fn, cpu)) - return; + return SPECTRE_MITIGATED; + + switch (method) { + case SPECTRE_V2_METHOD_BPIALL: + per_cpu(harden_branch_predictor_fn, cpu) = + harden_branch_predictor_bpiall; + spectre_v2_method = "BPIALL"; + break; + + case SPECTRE_V2_METHOD_ICIALLU: + per_cpu(harden_branch_predictor_fn, cpu) = + harden_branch_predictor_iciallu; + spectre_v2_method = "ICIALLU"; + break; + + case SPECTRE_V2_METHOD_HVC: + per_cpu(harden_branch_predictor_fn, cpu) = + call_hvc_arch_workaround_1; + cpu_do_switch_mm = cpu_v7_hvc_switch_mm; + spectre_v2_method = "hypervisor"; + break; + + case SPECTRE_V2_METHOD_SMC: + per_cpu(harden_branch_predictor_fn, cpu) = + call_smc_arch_workaround_1; + cpu_do_switch_mm = cpu_v7_smc_switch_mm; + spectre_v2_method = "firmware"; + break; + } + + if (spectre_v2_method) + pr_info("CPU%u: Spectre v2: using %s workaround\n", + smp_processor_id(), spectre_v2_method); + + return SPECTRE_MITIGATED; +} +#else +static unsigned int spectre_v2_install_workaround(unsigned int method) +{ + pr_info("CPU%u: Spectre V2: workarounds disabled by configuration\n"); + + return SPECTRE_VULNERABLE; +} +#endif + +static void cpu_v7_spectre_v2_init(void) +{ + unsigned int state, method = 0; switch (read_cpuid_part()) { case ARM_CPU_PART_CORTEX_A8: @@ -51,68 +125,57 @@ static void cpu_v7_spectre_init(void) case ARM_CPU_PART_CORTEX_A17: case ARM_CPU_PART_CORTEX_A73: case ARM_CPU_PART_CORTEX_A75: - per_cpu(harden_branch_predictor_fn, cpu) = - harden_branch_predictor_bpiall; - spectre_v2_method = "BPIALL"; + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_BPIALL; break; case ARM_CPU_PART_CORTEX_A15: case ARM_CPU_PART_BRAHMA_B15: - per_cpu(harden_branch_predictor_fn, cpu) = - harden_branch_predictor_iciallu; - spectre_v2_method = "ICIALLU"; + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_ICIALLU; break; -#ifdef CONFIG_ARM_PSCI case ARM_CPU_PART_BRAHMA_B53: /* Requires no workaround */ + state = SPECTRE_UNAFFECTED; break; + default: /* Other ARM CPUs require no workaround */ - if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) + if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) { + state = SPECTRE_UNAFFECTED; break; - fallthrough; - /* Cortex A57/A72 require firmware workaround */ - case ARM_CPU_PART_CORTEX_A57: - case ARM_CPU_PART_CORTEX_A72: { - struct arm_smccc_res res; + } - arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, - ARM_SMCCC_ARCH_WORKAROUND_1, &res); - if ((int)res.a0 != 0) - return; + fallthrough; + + /* Cortex A57/A72 require firmware workaround */ + case ARM_CPU_PART_CORTEX_A57: + case ARM_CPU_PART_CORTEX_A72: + state = spectre_v2_get_cpu_fw_mitigation_state(); + if (state != SPECTRE_MITIGATED) + break; switch (arm_smccc_1_1_get_conduit()) { case SMCCC_CONDUIT_HVC: - per_cpu(harden_branch_predictor_fn, cpu) = - call_hvc_arch_workaround_1; - cpu_do_switch_mm = cpu_v7_hvc_switch_mm; - spectre_v2_method = "hypervisor"; + method = SPECTRE_V2_METHOD_HVC; break; case SMCCC_CONDUIT_SMC: - per_cpu(harden_branch_predictor_fn, cpu) = - call_smc_arch_workaround_1; - cpu_do_switch_mm = cpu_v7_smc_switch_mm; - spectre_v2_method = "firmware"; + method = SPECTRE_V2_METHOD_SMC; break; default: + state = SPECTRE_VULNERABLE; break; } } -#endif - } - if (spectre_v2_method) - pr_info("CPU%u: Spectre v2: using %s workaround\n", - smp_processor_id(), spectre_v2_method); + if (state == SPECTRE_MITIGATED) + state = spectre_v2_install_workaround(method); + + spectre_v2_update_state(state, method); } -#else -static void cpu_v7_spectre_init(void) -{ -} -#endif static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned, u32 mask, const char *msg) @@ -142,16 +205,16 @@ static bool check_spectre_auxcr(bool *warned, u32 bit) void cpu_v7_ca8_ibe(void) { if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6))) - cpu_v7_spectre_init(); + cpu_v7_spectre_v2_init(); } void cpu_v7_ca15_ibe(void) { if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0))) - cpu_v7_spectre_init(); + cpu_v7_spectre_v2_init(); } void cpu_v7_bugs_init(void) { - cpu_v7_spectre_init(); + cpu_v7_spectre_v2_init(); } From 3bfcb356df0dad96d1c9ebdc0d46f769046cf8b9 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 11 Feb 2022 19:46:15 +0000 Subject: [PATCH 39/80] UPSTREAM: ARM: early traps initialisation commit 04e91b7324760a377a725e218b5ee783826d30f5 upstream. Provide a couple of helpers to copy the vectors and stubs, and also to flush the copied vectors and stubs. Bug: 215557547 Acked-by: Catalin Marinas Signed-off-by: Russell King (Oracle) Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Id60ef859e431a093597199294e4f57efdde84867 --- arch/arm/kernel/traps.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 17d5a785df28..e9ad5f844781 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -806,10 +806,22 @@ static inline void __init kuser_init(void *vectors) } #endif +#ifndef CONFIG_CPU_V7M +static void copy_from_lma(void *vma, void *lma_start, void *lma_end) +{ + memcpy(vma, lma_start, lma_end - lma_start); +} + +static void flush_vectors(void *vma, size_t offset, size_t size) +{ + unsigned long start = (unsigned long)vma + offset; + unsigned long end = start + size; + + flush_icache_range(start, end); +} + void __init early_trap_init(void *vectors_base) { -#ifndef CONFIG_CPU_V7M - unsigned long vectors = (unsigned long)vectors_base; extern char __stubs_start[], __stubs_end[]; extern char __vectors_start[], __vectors_end[]; unsigned i; @@ -830,17 +842,20 @@ void __init early_trap_init(void *vectors_base) * into the vector page, mapped at 0xffff0000, and ensure these * are visible to the instruction stream. */ - memcpy((void *)vectors, __vectors_start, __vectors_end - __vectors_start); - memcpy((void *)vectors + 0x1000, __stubs_start, __stubs_end - __stubs_start); + copy_from_lma(vectors_base, __vectors_start, __vectors_end); + copy_from_lma(vectors_base + 0x1000, __stubs_start, __stubs_end); kuser_init(vectors_base); - flush_icache_range(vectors, vectors + PAGE_SIZE * 2); + flush_vectors(vectors_base, 0, PAGE_SIZE * 2); +} #else /* ifndef CONFIG_CPU_V7M */ +void __init early_trap_init(void *vectors_base) +{ /* * on V7-M there is no need to copy the vector table to a dedicated * memory area. The address is configurable and so a table in the kernel * image can be used. */ -#endif } +#endif From 5a41f364e7c21ce411a71788cda721f46533f6fa Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 11 Feb 2022 19:49:50 +0000 Subject: [PATCH 40/80] UPSTREAM: ARM: use LOADADDR() to get load address of sections commit 8d9d651ff2270a632e9dc497b142db31e8911315 upstream. Use the linker's LOADADDR() macro to get the load address of the sections, and provide a macro to set the start and end symbols. Bug: 215557547 Acked-by: Catalin Marinas Signed-off-by: Russell King (Oracle) Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I464f69bdc337ce004d521f20eed7261faaa2f951 --- arch/arm/include/asm/vmlinux.lds.h | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/arch/arm/include/asm/vmlinux.lds.h b/arch/arm/include/asm/vmlinux.lds.h index 4a91428c324d..e02710d17cf9 100644 --- a/arch/arm/include/asm/vmlinux.lds.h +++ b/arch/arm/include/asm/vmlinux.lds.h @@ -26,6 +26,11 @@ #define ARM_MMU_DISCARD(x) x #endif +/* Set start/end symbol names to the LMA for the section */ +#define ARM_LMA(sym, section) \ + sym##_start = LOADADDR(section); \ + sym##_end = LOADADDR(section) + SIZEOF(section) + #define PROC_INFO \ . = ALIGN(4); \ __proc_info_begin = .; \ @@ -110,19 +115,19 @@ * only thing that matters is their relative offsets */ #define ARM_VECTORS \ - __vectors_start = .; \ + __vectors_lma = .; \ .vectors 0xffff0000 : AT(__vectors_start) { \ *(.vectors) \ } \ - . = __vectors_start + SIZEOF(.vectors); \ - __vectors_end = .; \ + ARM_LMA(__vectors, .vectors); \ + . = __vectors_lma + SIZEOF(.vectors); \ \ - __stubs_start = .; \ - .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_start) { \ + __stubs_lma = .; \ + .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_lma) { \ *(.stubs) \ } \ - . = __stubs_start + SIZEOF(.stubs); \ - __stubs_end = .; \ + ARM_LMA(__stubs, .stubs); \ + . = __stubs_lma + SIZEOF(.stubs); \ \ PROVIDE(vector_fiq_offset = vector_fiq - ADDR(.vectors)); From afbbe4048fb5908a42cf5cb473aaa997ae01f4f9 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Thu, 10 Feb 2022 16:05:45 +0000 Subject: [PATCH 41/80] UPSTREAM: ARM: Spectre-BHB workaround commit b9baf5c8c5c356757f4f9d8180b5e9d234065bc3 upstream. Workaround the Spectre BHB issues for Cortex-A15, Cortex-A57, Cortex-A72, Cortex-A73 and Cortex-A75. We also include Brahma B15 as well to be safe, which is affected by Spectre V2 in the same ways as Cortex-A15. Bug: 215557547 Reviewed-by: Catalin Marinas Signed-off-by: Russell King (Oracle) [changes due to lack of SYSTEM_FREEING_INITMEM - gregkh] Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ic8fcc01c10a8d8ad1293e6d5d8a5f2da592234c7 --- arch/arm/include/asm/assembler.h | 10 ++++ arch/arm/include/asm/spectre.h | 4 ++ arch/arm/include/asm/vmlinux.lds.h | 18 +++++-- arch/arm/kernel/entry-armv.S | 79 +++++++++++++++++++++++++++--- arch/arm/kernel/entry-common.S | 24 +++++++++ arch/arm/kernel/spectre.c | 4 ++ arch/arm/kernel/traps.c | 38 ++++++++++++++ arch/arm/mm/Kconfig | 10 ++++ arch/arm/mm/proc-v7-bugs.c | 76 ++++++++++++++++++++++++++++ 9 files changed, 254 insertions(+), 9 deletions(-) diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 761e88c73955..5cce98ae030e 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -107,6 +107,16 @@ .endm #endif +#if __LINUX_ARM_ARCH__ < 7 + .macro dsb, args + mcr p15, 0, r0, c7, c10, 4 + .endm + + .macro isb, args + mcr p15, 0, r0, c7, r5, 4 + .endm +#endif + .macro asm_trace_hardirqs_off, save=1 #if defined(CONFIG_TRACE_IRQFLAGS) .if \save diff --git a/arch/arm/include/asm/spectre.h b/arch/arm/include/asm/spectre.h index 8a9019e08dba..d1fa5607d3aa 100644 --- a/arch/arm/include/asm/spectre.h +++ b/arch/arm/include/asm/spectre.h @@ -14,6 +14,7 @@ enum { __SPECTRE_V2_METHOD_ICIALLU, __SPECTRE_V2_METHOD_SMC, __SPECTRE_V2_METHOD_HVC, + __SPECTRE_V2_METHOD_LOOP8, }; enum { @@ -21,8 +22,11 @@ enum { SPECTRE_V2_METHOD_ICIALLU = BIT(__SPECTRE_V2_METHOD_ICIALLU), SPECTRE_V2_METHOD_SMC = BIT(__SPECTRE_V2_METHOD_SMC), SPECTRE_V2_METHOD_HVC = BIT(__SPECTRE_V2_METHOD_HVC), + SPECTRE_V2_METHOD_LOOP8 = BIT(__SPECTRE_V2_METHOD_LOOP8), }; void spectre_v2_update_state(unsigned int state, unsigned int methods); +int spectre_bhb_update_vectors(unsigned int method); + #endif diff --git a/arch/arm/include/asm/vmlinux.lds.h b/arch/arm/include/asm/vmlinux.lds.h index e02710d17cf9..0ef21bfae9f6 100644 --- a/arch/arm/include/asm/vmlinux.lds.h +++ b/arch/arm/include/asm/vmlinux.lds.h @@ -116,11 +116,23 @@ */ #define ARM_VECTORS \ __vectors_lma = .; \ - .vectors 0xffff0000 : AT(__vectors_start) { \ - *(.vectors) \ + OVERLAY 0xffff0000 : NOCROSSREFS AT(__vectors_lma) { \ + .vectors { \ + *(.vectors) \ + } \ + .vectors.bhb.loop8 { \ + *(.vectors.bhb.loop8) \ + } \ + .vectors.bhb.bpiall { \ + *(.vectors.bhb.bpiall) \ + } \ } \ ARM_LMA(__vectors, .vectors); \ - . = __vectors_lma + SIZEOF(.vectors); \ + ARM_LMA(__vectors_bhb_loop8, .vectors.bhb.loop8); \ + ARM_LMA(__vectors_bhb_bpiall, .vectors.bhb.bpiall); \ + . = __vectors_lma + SIZEOF(.vectors) + \ + SIZEOF(.vectors.bhb.loop8) + \ + SIZEOF(.vectors.bhb.bpiall); \ \ __stubs_lma = .; \ .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_lma) { \ diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 1c9e6d1452c5..01f5e6ab9334 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -1007,12 +1007,11 @@ vector_\name: sub lr, lr, #\correction .endif - @ - @ Save r0, lr_ (parent PC) and spsr_ - @ (parent CPSR) - @ + @ Save r0, lr_ (parent PC) stmia sp, {r0, lr} @ save r0, lr - mrs lr, spsr + + @ Save spsr_ (parent CPSR) +2: mrs lr, spsr str lr, [sp, #8] @ save spsr @ @@ -1033,6 +1032,44 @@ vector_\name: movs pc, lr @ branch to handler in SVC mode ENDPROC(vector_\name) +#ifdef CONFIG_HARDEN_BRANCH_HISTORY + .subsection 1 + .align 5 +vector_bhb_loop8_\name: + .if \correction + sub lr, lr, #\correction + .endif + + @ Save r0, lr_ (parent PC) + stmia sp, {r0, lr} + + @ bhb workaround + mov r0, #8 +1: b . + 4 + subs r0, r0, #1 + bne 1b + dsb + isb + b 2b +ENDPROC(vector_bhb_loop8_\name) + +vector_bhb_bpiall_\name: + .if \correction + sub lr, lr, #\correction + .endif + + @ Save r0, lr_ (parent PC) + stmia sp, {r0, lr} + + @ bhb workaround + mcr p15, 0, r0, c7, c5, 6 @ BPIALL + @ isb not needed due to "movs pc, lr" in the vector stub + @ which gives a "context synchronisation". + b 2b +ENDPROC(vector_bhb_bpiall_\name) + .previous +#endif + .align 2 @ handler addresses follow this label 1: @@ -1041,6 +1078,10 @@ ENDPROC(vector_\name) .section .stubs, "ax", %progbits @ This must be the first word .word vector_swi +#ifdef CONFIG_HARDEN_BRANCH_HISTORY + .word vector_bhb_loop8_swi + .word vector_bhb_bpiall_swi +#endif vector_rst: ARM( swi SYS_ERROR0 ) @@ -1155,8 +1196,10 @@ vector_addrexcptn: * FIQ "NMI" handler *----------------------------------------------------------------------------- * Handle a FIQ using the SVC stack allowing FIQ act like NMI on x86 - * systems. + * systems. This must be the last vector stub, so lets place it in its own + * subsection. */ + .subsection 2 vector_stub fiq, FIQ_MODE, 4 .long __fiq_usr @ 0 (USR_26 / USR_32) @@ -1189,6 +1232,30 @@ vector_addrexcptn: W(b) vector_irq W(b) vector_fiq +#ifdef CONFIG_HARDEN_BRANCH_HISTORY + .section .vectors.bhb.loop8, "ax", %progbits +.L__vectors_bhb_loop8_start: + W(b) vector_rst + W(b) vector_bhb_loop8_und + W(ldr) pc, .L__vectors_bhb_loop8_start + 0x1004 + W(b) vector_bhb_loop8_pabt + W(b) vector_bhb_loop8_dabt + W(b) vector_addrexcptn + W(b) vector_bhb_loop8_irq + W(b) vector_bhb_loop8_fiq + + .section .vectors.bhb.bpiall, "ax", %progbits +.L__vectors_bhb_bpiall_start: + W(b) vector_rst + W(b) vector_bhb_bpiall_und + W(ldr) pc, .L__vectors_bhb_bpiall_start + 0x1008 + W(b) vector_bhb_bpiall_pabt + W(b) vector_bhb_bpiall_dabt + W(b) vector_addrexcptn + W(b) vector_bhb_bpiall_irq + W(b) vector_bhb_bpiall_fiq +#endif + .data .align 2 diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 271cb8a1eba1..bd619da73c84 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -162,6 +162,29 @@ ENDPROC(ret_from_fork) *----------------------------------------------------------------------------- */ + .align 5 +#ifdef CONFIG_HARDEN_BRANCH_HISTORY +ENTRY(vector_bhb_loop8_swi) + sub sp, sp, #PT_REGS_SIZE + stmia sp, {r0 - r12} + mov r8, #8 +1: b 2f +2: subs r8, r8, #1 + bne 1b + dsb + isb + b 3f +ENDPROC(vector_bhb_loop8_swi) + + .align 5 +ENTRY(vector_bhb_bpiall_swi) + sub sp, sp, #PT_REGS_SIZE + stmia sp, {r0 - r12} + mcr p15, 0, r8, c7, c5, 6 @ BPIALL + isb + b 3f +ENDPROC(vector_bhb_bpiall_swi) +#endif .align 5 ENTRY(vector_swi) #ifdef CONFIG_CPU_V7M @@ -169,6 +192,7 @@ ENTRY(vector_swi) #else sub sp, sp, #PT_REGS_SIZE stmia sp, {r0 - r12} @ Calling r0 - r12 +3: ARM( add r8, sp, #S_PC ) ARM( stmdb r8, {sp, lr}^ ) @ Calling sp, lr THUMB( mov r8, sp ) diff --git a/arch/arm/kernel/spectre.c b/arch/arm/kernel/spectre.c index 6f6dd1cfd099..ade967f18d06 100644 --- a/arch/arm/kernel/spectre.c +++ b/arch/arm/kernel/spectre.c @@ -45,6 +45,10 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, method = "Firmware call"; break; + case SPECTRE_V2_METHOD_LOOP8: + method = "History overwrite"; + break; + default: method = "Multiple mitigations"; break; diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index e9ad5f844781..2d9e72ad1b0f 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -820,6 +821,43 @@ static void flush_vectors(void *vma, size_t offset, size_t size) flush_icache_range(start, end); } +#ifdef CONFIG_HARDEN_BRANCH_HISTORY +int spectre_bhb_update_vectors(unsigned int method) +{ + extern char __vectors_bhb_bpiall_start[], __vectors_bhb_bpiall_end[]; + extern char __vectors_bhb_loop8_start[], __vectors_bhb_loop8_end[]; + void *vec_start, *vec_end; + + if (system_state > SYSTEM_SCHEDULING) { + pr_err("CPU%u: Spectre BHB workaround too late - system vulnerable\n", + smp_processor_id()); + return SPECTRE_VULNERABLE; + } + + switch (method) { + case SPECTRE_V2_METHOD_LOOP8: + vec_start = __vectors_bhb_loop8_start; + vec_end = __vectors_bhb_loop8_end; + break; + + case SPECTRE_V2_METHOD_BPIALL: + vec_start = __vectors_bhb_bpiall_start; + vec_end = __vectors_bhb_bpiall_end; + break; + + default: + pr_err("CPU%u: unknown Spectre BHB state %d\n", + smp_processor_id(), method); + return SPECTRE_VULNERABLE; + } + + copy_from_lma(vectors_page, vec_start, vec_end); + flush_vectors(vectors_page, 0, vec_end - vec_start); + + return SPECTRE_MITIGATED; +} +#endif + void __init early_trap_init(void *vectors_base) { extern char __stubs_start[], __stubs_end[]; diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 0804bd5e619f..c6bf34a33849 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -854,6 +854,16 @@ config HARDEN_BRANCH_PREDICTOR If unsure, say Y. +config HARDEN_BRANCH_HISTORY + bool "Harden Spectre style attacks against branch history" if EXPERT + depends on CPU_SPECTRE + default y + help + Speculation attacks against some high-performance processors can + make use of branch history to influence future speculation. When + taking an exception, a sequence of branches overwrites the branch + history, or branch history is invalidated. + config TLS_REG_EMUL bool select NEED_KUSER_HELPERS diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c index e438e59bb63e..c226feab2457 100644 --- a/arch/arm/mm/proc-v7-bugs.c +++ b/arch/arm/mm/proc-v7-bugs.c @@ -177,6 +177,81 @@ static void cpu_v7_spectre_v2_init(void) spectre_v2_update_state(state, method); } +#ifdef CONFIG_HARDEN_BRANCH_HISTORY +static int spectre_bhb_method; + +static const char *spectre_bhb_method_name(int method) +{ + switch (method) { + case SPECTRE_V2_METHOD_LOOP8: + return "loop"; + + case SPECTRE_V2_METHOD_BPIALL: + return "BPIALL"; + + default: + return "unknown"; + } +} + +static int spectre_bhb_install_workaround(int method) +{ + if (spectre_bhb_method != method) { + if (spectre_bhb_method) { + pr_err("CPU%u: Spectre BHB: method disagreement, system vulnerable\n", + smp_processor_id()); + + return SPECTRE_VULNERABLE; + } + + if (spectre_bhb_update_vectors(method) == SPECTRE_VULNERABLE) + return SPECTRE_VULNERABLE; + + spectre_bhb_method = method; + } + + pr_info("CPU%u: Spectre BHB: using %s workaround\n", + smp_processor_id(), spectre_bhb_method_name(method)); + + return SPECTRE_MITIGATED; +} +#else +static int spectre_bhb_install_workaround(int method) +{ + return SPECTRE_VULNERABLE; +} +#endif + +static void cpu_v7_spectre_bhb_init(void) +{ + unsigned int state, method = 0; + + switch (read_cpuid_part()) { + case ARM_CPU_PART_CORTEX_A15: + case ARM_CPU_PART_BRAHMA_B15: + case ARM_CPU_PART_CORTEX_A57: + case ARM_CPU_PART_CORTEX_A72: + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_LOOP8; + break; + + case ARM_CPU_PART_CORTEX_A73: + case ARM_CPU_PART_CORTEX_A75: + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_BPIALL; + break; + + default: + state = SPECTRE_UNAFFECTED; + break; + } + + if (state == SPECTRE_MITIGATED) + state = spectre_bhb_install_workaround(method); + + spectre_v2_update_state(state, method); +} + static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned, u32 mask, const char *msg) { @@ -217,4 +292,5 @@ void cpu_v7_ca15_ibe(void) void cpu_v7_bugs_init(void) { cpu_v7_spectre_v2_init(); + cpu_v7_spectre_bhb_init(); } From bd2376838d2ff624ca76f02c8220e76f1e9b0d86 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Mon, 7 Mar 2022 19:28:32 +0000 Subject: [PATCH 42/80] UPSTREAM: ARM: include unprivileged BPF status in Spectre V2 reporting commit 25875aa71dfefd1959f07e626c4d285b88b27ac2 upstream. The mitigations for Spectre-BHB are only applied when an exception is taken, but when unprivileged BPF is enabled, userspace can load BPF programs that can be used to exploit the problem. When unprivileged BPF is enabled, report the vulnerable status via the spectre_v2 sysfs file. Bug: 215557547 Signed-off-by: Russell King (Oracle) Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ibf6c9ed7dc6e2b269feecde562ecaca2856c68ef --- arch/arm/kernel/spectre.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/arm/kernel/spectre.c b/arch/arm/kernel/spectre.c index ade967f18d06..e7fea962d632 100644 --- a/arch/arm/kernel/spectre.c +++ b/arch/arm/kernel/spectre.c @@ -1,9 +1,19 @@ // SPDX-License-Identifier: GPL-2.0-only +#include #include #include #include +static bool _unprivileged_ebpf_enabled(void) +{ +#ifdef CONFIG_BPF_SYSCALL + return !sysctl_unprivileged_bpf_disabled; +#else + return false +#endif +} + ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) { @@ -31,6 +41,9 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, if (spectre_v2_state != SPECTRE_MITIGATED) return sprintf(buf, "%s\n", "Vulnerable"); + if (_unprivileged_ebpf_enabled()) + return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); + switch (spectre_v2_methods) { case SPECTRE_V2_METHOD_BPIALL: method = "Branch predictor hardening"; From a4e68d43f5ed1c79952373d90dcfddb423046b50 Mon Sep 17 00:00:00 2001 From: Emmanuel Gil Peyrot Date: Tue, 8 Mar 2022 20:18:20 +0100 Subject: [PATCH 43/80] UPSTREAM: ARM: fix build error when BPF_SYSCALL is disabled commit 330f4c53d3c2d8b11d86ec03a964b86dc81452f5 upstream. It was missing a semicolon. Bug: 215557547 Signed-off-by: Emmanuel Gil Peyrot Reviewed-by: Nathan Chancellor Fixes: 25875aa71dfe ("ARM: include unprivileged BPF status in Spectre V2 reporting"). Signed-off-by: Linus Torvalds Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ifb28334a7ddaf7976ccb19839f225341e1c504d6 --- arch/arm/kernel/spectre.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/kernel/spectre.c b/arch/arm/kernel/spectre.c index e7fea962d632..0dcefc36fb7a 100644 --- a/arch/arm/kernel/spectre.c +++ b/arch/arm/kernel/spectre.c @@ -10,7 +10,7 @@ static bool _unprivileged_ebpf_enabled(void) #ifdef CONFIG_BPF_SYSCALL return !sysctl_unprivileged_bpf_disabled; #else - return false + return false; #endif } From 57bc1e13f00812eab2e104583a1f820c1fae3d1c Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Wed, 9 Mar 2022 19:08:42 +0000 Subject: [PATCH 44/80] UPSTREAM: ARM: fix co-processor register typo commit 33970b031dc4653cc9dc80f2886976706c4c8ef1 upstream. In the recent Spectre BHB patches, there was a typo that is only exposed in certain configurations: mcr p15,0,XX,c7,r5,4 should have been mcr p15,0,XX,c7,c5,4 Bug: 215557547 Reported-by: kernel test robot Fixes: b9baf5c8c5c3 ("ARM: Spectre-BHB workaround") Signed-off-by: Russell King (Oracle) Acked-by: Catalin Marinas Signed-off-by: Linus Torvalds Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ic7ef143df05901125d0d936d5b70000320b9debe --- arch/arm/include/asm/assembler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 5cce98ae030e..8ff20f9b06ba 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -113,7 +113,7 @@ .endm .macro isb, args - mcr p15, 0, r0, c7, r5, 4 + mcr p15, 0, r0, c7, c5, 4 .endm #endif From fdf3cb8a00b079559852389677cc8cf9b834a69c Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 9 Mar 2022 15:07:27 -0700 Subject: [PATCH 45/80] UPSTREAM: ARM: Do not use NOCROSSREFS directive with ld.lld commit 36168e387fa7d0f1fe0cd5cf76c8cea7aee714fa upstream. ld.lld does not support the NOCROSSREFS directive at the moment, which breaks the build after commit b9baf5c8c5c3 ("ARM: Spectre-BHB workaround"): ld.lld: error: ./arch/arm/kernel/vmlinux.lds:34: AT expected, but got NOCROSSREFS Support for this directive will eventually be implemented, at which point a version check can be added. To avoid breaking the build in the meantime, just define NOCROSSREFS to nothing when using ld.lld, with a link to the issue for tracking. Bug: 215557547 Cc: stable@vger.kernel.org Fixes: b9baf5c8c5c3 ("ARM: Spectre-BHB workaround") Link: https://github.com/ClangBuiltLinux/linux/issues/1609 Signed-off-by: Nathan Chancellor Signed-off-by: Linus Torvalds Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I6c1ddb098c415e848b5422e768c483a04005079a --- arch/arm/include/asm/vmlinux.lds.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/arm/include/asm/vmlinux.lds.h b/arch/arm/include/asm/vmlinux.lds.h index 0ef21bfae9f6..fad45c884e98 100644 --- a/arch/arm/include/asm/vmlinux.lds.h +++ b/arch/arm/include/asm/vmlinux.lds.h @@ -26,6 +26,14 @@ #define ARM_MMU_DISCARD(x) x #endif +/* + * ld.lld does not support NOCROSSREFS: + * https://github.com/ClangBuiltLinux/linux/issues/1609 + */ +#ifdef CONFIG_LD_IS_LLD +#define NOCROSSREFS +#endif + /* Set start/end symbol names to the LMA for the section */ #define ARM_LMA(sym, section) \ sym##_start = LOADADDR(section); \ From 1ea0d915884d54905d09c186d0722bf61d60d727 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Thu, 10 Mar 2022 10:22:14 +0000 Subject: [PATCH 46/80] UPSTREAM: ARM: fix build warning in proc-v7-bugs.c commit b1a384d2cbccb1eb3f84765020d25e2c1929706e upstream. The kernel test robot discovered that building without HARDEN_BRANCH_PREDICTOR issues a warning due to a missing argument to pr_info(). Add the missing argument. Bug: 215557547 Reported-by: kernel test robot Fixes: 9dd78194a372 ("ARM: report Spectre v2 status through sysfs") Signed-off-by: Russell King (Oracle) Signed-off-by: Linus Torvalds Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ic68494938f6a01595db5a4b386d5d59552b2c1bd --- arch/arm/mm/proc-v7-bugs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c index c226feab2457..06dbfb968182 100644 --- a/arch/arm/mm/proc-v7-bugs.c +++ b/arch/arm/mm/proc-v7-bugs.c @@ -108,7 +108,8 @@ static unsigned int spectre_v2_install_workaround(unsigned int method) #else static unsigned int spectre_v2_install_workaround(unsigned int method) { - pr_info("CPU%u: Spectre V2: workarounds disabled by configuration\n"); + pr_info("CPU%u: Spectre V2: workarounds disabled by configuration\n", + smp_processor_id()); return SPECTRE_VULNERABLE; } From 56186c7e4ace39193c2c439304c16a700e1ef251 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Fri, 11 Mar 2022 11:49:12 -0800 Subject: [PATCH 47/80] UPSTREAM: ARM: Spectre-BHB: provide empty stub for non-config commit 68453767131a5deec1e8f9ac92a9042f929e585d upstream. When CONFIG_GENERIC_CPU_VULNERABILITIES is not set, references to spectre_v2_update_state() cause a build error, so provide an empty stub for that function when the Kconfig option is not set. Fixes this build error: arm-linux-gnueabi-ld: arch/arm/mm/proc-v7-bugs.o: in function `cpu_v7_bugs_init': proc-v7-bugs.c:(.text+0x52): undefined reference to `spectre_v2_update_state' arm-linux-gnueabi-ld: proc-v7-bugs.c:(.text+0x82): undefined reference to `spectre_v2_update_state' Bug: 215557547 Fixes: b9baf5c8c5c3 ("ARM: Spectre-BHB workaround") Signed-off-by: Randy Dunlap Reported-by: kernel test robot Cc: Russell King Cc: Catalin Marinas Cc: linux-arm-kernel@lists.infradead.org Cc: patches@armlinux.org.uk Acked-by: Russell King (Oracle) Signed-off-by: Linus Torvalds Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I68a933128ef95a7ce0efc7fcda6fa34ac9b829b7 --- arch/arm/include/asm/spectre.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm/include/asm/spectre.h b/arch/arm/include/asm/spectre.h index d1fa5607d3aa..85f9e538fb32 100644 --- a/arch/arm/include/asm/spectre.h +++ b/arch/arm/include/asm/spectre.h @@ -25,7 +25,13 @@ enum { SPECTRE_V2_METHOD_LOOP8 = BIT(__SPECTRE_V2_METHOD_LOOP8), }; +#ifdef CONFIG_GENERIC_CPU_VULNERABILITIES void spectre_v2_update_state(unsigned int state, unsigned int methods); +#else +static inline void spectre_v2_update_state(unsigned int state, + unsigned int methods) +{} +#endif int spectre_bhb_update_vectors(unsigned int method); From a40cd2375581e722538e18a898110842f7ac4253 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 11 Mar 2022 17:13:17 +0000 Subject: [PATCH 48/80] UPSTREAM: ARM: fix Thumb2 regression with Spectre BHB commit 6c7cb60bff7aec24b834343ff433125f469886a3 upstream. When building for Thumb2, the vectors make use of a local label. Sadly, the Spectre BHB code also uses a local label with the same number which results in the Thumb2 reference pointing at the wrong place. Fix this by changing the number used for the Spectre BHB local label. Bug: 215557547 Fixes: b9baf5c8c5c3 ("ARM: Spectre-BHB workaround") Tested-by: Nathan Chancellor Signed-off-by: Russell King (Oracle) Signed-off-by: Linus Torvalds Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I2d1a926b165cc25cb5ac4482a1591f2ee189585d --- arch/arm/kernel/entry-armv.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 01f5e6ab9334..ee8358fc72ba 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -1045,9 +1045,9 @@ vector_bhb_loop8_\name: @ bhb workaround mov r0, #8 -1: b . + 4 +3: b . + 4 subs r0, r0, #1 - bne 1b + bne 3b dsb isb b 2b From 803ff1161c131f8300db84d8551dfa545817abbd Mon Sep 17 00:00:00 2001 From: Ramji Jiyani Date: Mon, 2 Aug 2021 22:04:45 +0000 Subject: [PATCH 49/80] UPSTREAM: binder: Add invalid handle info in user error log In the case of a failed transaction, only the thread and process id are logged. Add the handle info for the reference to the target node in user error log to aid debugging. Acked-by: Todd Kjos Signed-off-by: Ramji Jiyani Link: https://lore.kernel.org/r/20210802220446.1938347-1-ramjiyani@google.com Signed-off-by: Greg Kroah-Hartman Bug: 186045972 (cherry picked from commit 1ae14df56cc3e87d56f3c159803a289021f8ef7d) Signed-off-by: Ramji Jiyani Change-Id: I874bab8f32bafcf6a5fb38669c60d1e968f50f59 --- drivers/android/binder.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index cd688e896239..7d0a1713a513 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2705,8 +2705,8 @@ static void binder_transaction(struct binder_proc *proc, ref->node, &target_proc, &return_error); } else { - binder_user_error("%d:%d got transaction to invalid handle\n", - proc->pid, thread->pid); + binder_user_error("%d:%d got transaction to invalid handle, %u\n", + proc->pid, thread->pid, tr->target.handle); return_error = BR_FAILED_REPLY; } binder_proc_unlock(proc); From 51eded5d1b5feb42e8132ea0dc49bf82f4504725 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 4 Feb 2021 22:56:13 +0900 Subject: [PATCH 50/80] UPSTREAM: arm64: cputype: Add CPU implementor & types for the Apple M1 cores commit 11ecdad722daafcac09c4859dddf31b3d46449bc upstream. The implementor will be used to condition the FIQ support quirk. The specific CPU types are not used at the moment, but let's add them for documentation purposes. Bug: 215557547 Acked-by: Will Deacon Signed-off-by: Hector Martin Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I3246164b569f7aeca6bdaa7ee5938e45c68153df --- arch/arm64/include/asm/cputype.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index f9adcf962699..e8fdc10395b6 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -59,6 +59,7 @@ #define ARM_CPU_IMP_NVIDIA 0x4E #define ARM_CPU_IMP_FUJITSU 0x46 #define ARM_CPU_IMP_HISI 0x48 +#define ARM_CPU_IMP_APPLE 0x61 #define ARM_CPU_PART_AEM_V8 0xD0F #define ARM_CPU_PART_FOUNDATION 0xD00 @@ -102,6 +103,9 @@ #define HISI_CPU_PART_TSV110 0xD01 +#define APPLE_CPU_PART_M1_ICESTORM 0x022 +#define APPLE_CPU_PART_M1_FIRESTORM 0x023 + #define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) #define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) #define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) @@ -133,6 +137,8 @@ #define MIDR_NVIDIA_CARMEL MIDR_CPU_MODEL(ARM_CPU_IMP_NVIDIA, NVIDIA_CPU_PART_CARMEL) #define MIDR_FUJITSU_A64FX MIDR_CPU_MODEL(ARM_CPU_IMP_FUJITSU, FUJITSU_CPU_PART_A64FX) #define MIDR_HISI_TSV110 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV110) +#define MIDR_APPLE_M1_ICESTORM MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM) +#define MIDR_APPLE_M1_FIRESTORM MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM) /* Fujitsu Erratum 010001 affects A64FX 1.0 and 1.1, (v0r0 and v1r0) */ #define MIDR_FUJITSU_ERRATUM_010001 MIDR_FUJITSU_A64FX From 6d1f2678e2c063b6d57de29e6b6f6f4a296ae3d9 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Mon, 24 Jan 2022 08:45:37 +0530 Subject: [PATCH 51/80] UPSTREAM: arm64: Add Cortex-X2 CPU part definition commit 72bb9dcb6c33cfac80282713c2b4f2b254cd24d1 upstream. Add the CPU Partnumbers for the new Arm designs. Bug: 215557547 Cc: Will Deacon Cc: Suzuki Poulose Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/1642994138-25887-2-git-send-email-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Icd167e5cd6b4087f0a02c11d3dae9d603926e105 --- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index e8fdc10395b6..999b9149f856 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -75,6 +75,7 @@ #define ARM_CPU_PART_CORTEX_A77 0xD0D #define ARM_CPU_PART_CORTEX_A510 0xD46 #define ARM_CPU_PART_CORTEX_A710 0xD47 +#define ARM_CPU_PART_CORTEX_X2 0xD48 #define ARM_CPU_PART_NEOVERSE_N2 0xD49 #define APM_CPU_PART_POTENZA 0x000 @@ -118,6 +119,7 @@ #define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77) #define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510) #define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710) +#define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2) #define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) From 2e2eef400bce8e05e4dbdb03753e83de3ac5e31b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 17 Oct 2021 13:42:25 +0100 Subject: [PATCH 52/80] UPSTREAM: arm64: Add HWCAP for self-synchronising virtual counter commit fee29f008aa3f2aff01117f28b57b1145d92cb9b upstream. Since userspace can make use of the CNTVSS_EL0 instruction, expose it via a HWCAP. Bug: 215557547 Suggested-by: Will Deacon Acked-by: Will Deacon Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20211017124225.3018098-18-maz@kernel.org Signed-off-by: Will Deacon Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I8d2253c57bb5690135392b84d55bfb854a780b00 --- Documentation/arm64/cpu-feature-registers.rst | 12 ++++++++++-- Documentation/arm64/elf_hwcaps.rst | 4 ++++ arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 3 ++- arch/arm64/kernel/cpuinfo.c | 1 + 6 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Documentation/arm64/cpu-feature-registers.rst b/Documentation/arm64/cpu-feature-registers.rst index 328e0c454fbd..9f9b8fd06089 100644 --- a/Documentation/arm64/cpu-feature-registers.rst +++ b/Documentation/arm64/cpu-feature-registers.rst @@ -235,7 +235,15 @@ infrastructure: | DPB | [3-0] | y | +------------------------------+---------+---------+ - 6) ID_AA64MMFR2_EL1 - Memory model feature register 2 + 6) ID_AA64MMFR0_EL1 - Memory model feature register 0 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | ECV | [63-60] | y | + +------------------------------+---------+---------+ + + 7) ID_AA64MMFR2_EL1 - Memory model feature register 2 +------------------------------+---------+---------+ | Name | bits | visible | @@ -243,7 +251,7 @@ infrastructure: | AT | [35-32] | y | +------------------------------+---------+---------+ - 7) ID_AA64ZFR0_EL1 - SVE feature ID register 0 + 8) ID_AA64ZFR0_EL1 - SVE feature ID register 0 +------------------------------+---------+---------+ | Name | bits | visible | diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst index bbd9cf54db6c..95e66bd7dd17 100644 --- a/Documentation/arm64/elf_hwcaps.rst +++ b/Documentation/arm64/elf_hwcaps.rst @@ -245,6 +245,10 @@ HWCAP2_MTE Functionality implied by ID_AA64PFR1_EL1.MTE == 0b0010, as described by Documentation/arm64/memory-tagging-extension.rst. +HWCAP2_ECV + + Functionality implied by ID_AA64MMFR0_EL1.ECV == 0b0001. + 4. Unused AT_HWCAP bits ----------------------- diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 9a5498c2c8ee..0922dd55f90d 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -105,6 +105,7 @@ #define KERNEL_HWCAP_RNG __khwcap2_feature(RNG) #define KERNEL_HWCAP_BTI __khwcap2_feature(BTI) #define KERNEL_HWCAP_MTE __khwcap2_feature(MTE) +#define KERNEL_HWCAP_ECV __khwcap2_feature(ECV) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index b8f41aa234ee..7b23b16f21ce 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -75,5 +75,6 @@ #define HWCAP2_RNG (1 << 16) #define HWCAP2_BTI (1 << 17) #define HWCAP2_MTE (1 << 18) +#define HWCAP2_ECV (1 << 19) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 7db63bd14a54..78964d55d8bb 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -280,7 +280,7 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = { }; static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ECV_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ECV_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_FGT_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EXS_SHIFT, 4, 0), /* @@ -2434,6 +2434,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { #ifdef CONFIG_ARM64_MTE HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_MTE, CAP_HWCAP, KERNEL_HWCAP_MTE), #endif /* CONFIG_ARM64_MTE */ + HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_ECV_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV), {}, }; diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 8ce33742ad6a..3d32459dca7e 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -94,6 +94,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_RNG] = "rng", [KERNEL_HWCAP_BTI] = "bti", [KERNEL_HWCAP_MTE] = "mte", + [KERNEL_HWCAP_ECV] = "ecv", }; #ifdef CONFIG_COMPAT From 68bc555a237a8eff106710b98b34f65835c42e98 Mon Sep 17 00:00:00 2001 From: Joey Gouly Date: Fri, 10 Dec 2021 16:54:31 +0000 Subject: [PATCH 53/80] UPSTREAM: arm64: add ID_AA64ISAR2_EL1 sys register commit 9e45365f1469ef2b934f9d035975dbc9ad352116 upstream. This is a new ID register, introduced in 8.7. Bug: 215557547 Signed-off-by: Joey Gouly Cc: Will Deacon Cc: Marc Zyngier Cc: James Morse Cc: Alexandru Elisei Cc: Suzuki K Poulose Cc: Reiji Watanabe Acked-by: Marc Zyngier Link: https://lore.kernel.org/r/20211210165432.8106-3-joey.gouly@arm.com Signed-off-by: Catalin Marinas Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I7f7e7d027ed9ede262d3b9b04cd58cf0234bc36c --- arch/arm64/include/asm/cpu.h | 1 + arch/arm64/include/asm/sysreg.h | 15 +++++++++++++++ arch/arm64/kernel/cpufeature.c | 9 +++++++++ arch/arm64/kernel/cpuinfo.c | 1 + arch/arm64/kvm/sys_regs.c | 2 +- 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h index f4e01aa0f442..70dd1d6c8736 100644 --- a/arch/arm64/include/asm/cpu.h +++ b/arch/arm64/include/asm/cpu.h @@ -50,6 +50,7 @@ struct cpuinfo_arm64 { u64 reg_id_aa64dfr1; u64 reg_id_aa64isar0; u64 reg_id_aa64isar1; + u64 reg_id_aa64isar2; u64 reg_id_aa64mmfr0; u64 reg_id_aa64mmfr1; u64 reg_id_aa64mmfr2; diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index c41e6f7fcab2..fa0ea612c63a 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -180,6 +180,7 @@ #define SYS_ID_AA64ISAR0_EL1 sys_reg(3, 0, 0, 6, 0) #define SYS_ID_AA64ISAR1_EL1 sys_reg(3, 0, 0, 6, 1) +#define SYS_ID_AA64ISAR2_EL1 sys_reg(3, 0, 0, 6, 2) #define SYS_ID_AA64MMFR0_EL1 sys_reg(3, 0, 0, 7, 0) #define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1) @@ -753,6 +754,20 @@ #define ID_AA64ISAR1_GPI_NI 0x0 #define ID_AA64ISAR1_GPI_IMP_DEF 0x1 +/* id_aa64isar2 */ +#define ID_AA64ISAR2_RPRES_SHIFT 4 +#define ID_AA64ISAR2_WFXT_SHIFT 0 + +#define ID_AA64ISAR2_RPRES_8BIT 0x0 +#define ID_AA64ISAR2_RPRES_12BIT 0x1 +/* + * Value 0x1 has been removed from the architecture, and is + * reserved, but has not yet been removed from the ARM ARM + * as of ARM DDI 0487G.b. + */ +#define ID_AA64ISAR2_WFXT_NI 0x0 +#define ID_AA64ISAR2_WFXT_SUPPORTED 0x2 + /* id_aa64pfr0 */ #define ID_AA64PFR0_CSV3_SHIFT 60 #define ID_AA64PFR0_CSV2_SHIFT 56 diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 78964d55d8bb..38887d49a71f 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -226,6 +226,10 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { ARM64_FTR_END, }; +static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { + ARM64_FTR_END, +}; + static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0), @@ -633,6 +637,7 @@ static const struct __ftr_reg_entry { ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0), ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1, &id_aa64isar1_override), + ARM64_FTR_REG(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2), /* Op1 = 0, CRn = 0, CRm = 7 */ ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0), @@ -920,6 +925,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1); init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0); init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1); + init_cpu_ftr_reg(SYS_ID_AA64ISAR2_EL1, info->reg_id_aa64isar2); init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0); init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1); init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2); @@ -1135,6 +1141,8 @@ void update_cpu_features(int cpu, info->reg_id_aa64isar0, boot->reg_id_aa64isar0); taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu, info->reg_id_aa64isar1, boot->reg_id_aa64isar1); + taint |= check_update_ftr_reg(SYS_ID_AA64ISAR2_EL1, cpu, + info->reg_id_aa64isar2, boot->reg_id_aa64isar2); /* * Differing PARange support is fine as long as all peripherals and @@ -1245,6 +1253,7 @@ u64 __read_sysreg_by_encoding(u32 sys_id) read_sysreg_case(SYS_ID_AA64MMFR2_EL1); read_sysreg_case(SYS_ID_AA64ISAR0_EL1); read_sysreg_case(SYS_ID_AA64ISAR1_EL1); + read_sysreg_case(SYS_ID_AA64ISAR2_EL1); read_sysreg_case(SYS_CNTFRQ_EL0); read_sysreg_case(SYS_CTR_EL0); diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 3d32459dca7e..b88944bcd915 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -391,6 +391,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) info->reg_id_aa64dfr1 = read_cpuid(ID_AA64DFR1_EL1); info->reg_id_aa64isar0 = read_cpuid(ID_AA64ISAR0_EL1); info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1); + info->reg_id_aa64isar2 = read_cpuid(ID_AA64ISAR2_EL1); info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1); info->reg_id_aa64mmfr1 = read_cpuid(ID_AA64MMFR1_EL1); info->reg_id_aa64mmfr2 = read_cpuid(ID_AA64MMFR2_EL1); diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 47b05731b16c..187576986aef 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1447,7 +1447,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { /* CRm=6 */ ID_SANITISED(ID_AA64ISAR0_EL1), ID_SANITISED(ID_AA64ISAR1_EL1), - ID_UNALLOCATED(6,2), + ID_SANITISED(ID_AA64ISAR2_EL1), ID_UNALLOCATED(6,3), ID_UNALLOCATED(6,4), ID_UNALLOCATED(6,5), From 18c4e4fa56e55a5a8cf0c479011525cf35e3a644 Mon Sep 17 00:00:00 2001 From: Joey Gouly Date: Fri, 10 Dec 2021 16:54:30 +0000 Subject: [PATCH 54/80] UPSTREAM: arm64: cpufeature: add HWCAP for FEAT_AFP commit 5c13f042e73200b50573ace63e1a6b94e2917616 upstream. Add a new HWCAP to detect the Alternate Floating-point Behaviour feature (FEAT_AFP), introduced in Armv8.7. Also expose this to userspace in the ID_AA64MMFR1_EL1 feature register. Bug: 215557547 Signed-off-by: Joey Gouly Cc: Will Deacon Acked-by: Marc Zyngier Link: https://lore.kernel.org/r/20211210165432.8106-2-joey.gouly@arm.com Signed-off-by: Catalin Marinas Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ifb7685f6d97cb469262a3766daab71d09f293b1b --- Documentation/arm64/cpu-feature-registers.rst | 9 +++++++++ Documentation/arm64/elf_hwcaps.rst | 4 ++++ arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/asm/sysreg.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 2 ++ arch/arm64/kernel/cpuinfo.c | 1 + 7 files changed, 19 insertions(+) diff --git a/Documentation/arm64/cpu-feature-registers.rst b/Documentation/arm64/cpu-feature-registers.rst index 9f9b8fd06089..1b19d20c2dbd 100644 --- a/Documentation/arm64/cpu-feature-registers.rst +++ b/Documentation/arm64/cpu-feature-registers.rst @@ -275,6 +275,15 @@ infrastructure: | SVEVer | [3-0] | y | +------------------------------+---------+---------+ + 8) ID_AA64MMFR1_EL1 - Memory model feature register 1 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | AFP | [47-44] | y | + +------------------------------+---------+---------+ + + Appendix I: Example ------------------- diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst index 95e66bd7dd17..77b6da64cfad 100644 --- a/Documentation/arm64/elf_hwcaps.rst +++ b/Documentation/arm64/elf_hwcaps.rst @@ -249,6 +249,10 @@ HWCAP2_ECV Functionality implied by ID_AA64MMFR0_EL1.ECV == 0b0001. +HWCAP2_AFP + + Functionality implied by ID_AA64MFR1_EL1.AFP == 0b0001. + 4. Unused AT_HWCAP bits ----------------------- diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 0922dd55f90d..f20fbec36669 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -106,6 +106,7 @@ #define KERNEL_HWCAP_BTI __khwcap2_feature(BTI) #define KERNEL_HWCAP_MTE __khwcap2_feature(MTE) #define KERNEL_HWCAP_ECV __khwcap2_feature(ECV) +#define KERNEL_HWCAP_AFP __khwcap2_feature(AFP) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index fa0ea612c63a..253b69b7d027 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -867,6 +867,7 @@ #endif /* id_aa64mmfr1 */ +#define ID_AA64MMFR1_AFP_SHIFT 44 #define ID_AA64MMFR1_ETS_SHIFT 36 #define ID_AA64MMFR1_TWED_SHIFT 32 #define ID_AA64MMFR1_XNX_SHIFT 28 diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 7b23b16f21ce..180da7396549 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -76,5 +76,6 @@ #define HWCAP2_BTI (1 << 17) #define HWCAP2_MTE (1 << 18) #define HWCAP2_ECV (1 << 19) +#define HWCAP2_AFP (1 << 20) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 38887d49a71f..2e27e0d915e9 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -330,6 +330,7 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { }; static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_AFP_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_ETS_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_TWED_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_XNX_SHIFT, 4, 0), @@ -2444,6 +2445,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_MTE, CAP_HWCAP, KERNEL_HWCAP_MTE), #endif /* CONFIG_ARM64_MTE */ HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_ECV_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV), + HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_AFP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP), {}, }; diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index b88944bcd915..25300eb04cef 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -95,6 +95,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_BTI] = "bti", [KERNEL_HWCAP_MTE] = "mte", [KERNEL_HWCAP_ECV] = "ecv", + [KERNEL_HWCAP_AFP] = "afp", }; #ifdef CONFIG_COMPAT From 81ec26aafe54937d0dfcd60f42a9ae5776b30c3b Mon Sep 17 00:00:00 2001 From: Joey Gouly Date: Fri, 10 Dec 2021 16:54:32 +0000 Subject: [PATCH 55/80] UPSTREAM: arm64: cpufeature: add HWCAP for FEAT_RPRES commit 1175011a7d0030d49dc9c10bde36f08f26d0a8ee upstream. Add a new HWCAP to detect the Increased precision of Reciprocal Estimate and Reciprocal Square Root Estimate feature (FEAT_RPRES), introduced in Armv8.7. Also expose this to userspace in the ID_AA64ISAR2_EL1 feature register. Bug: 215557547 Signed-off-by: Joey Gouly Cc: Will Deacon Cc: Jonathan Corbet Acked-by: Marc Zyngier Link: https://lore.kernel.org/r/20211210165432.8106-4-joey.gouly@arm.com Signed-off-by: Catalin Marinas Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I51e6c854cc1d319ccf6f11966e1c014ae82cb4cf --- Documentation/arm64/cpu-feature-registers.rst | 8 ++++++++ Documentation/arm64/elf_hwcaps.rst | 4 ++++ arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 2 ++ arch/arm64/kernel/cpuinfo.c | 1 + 6 files changed, 17 insertions(+) diff --git a/Documentation/arm64/cpu-feature-registers.rst b/Documentation/arm64/cpu-feature-registers.rst index 1b19d20c2dbd..749ae970c319 100644 --- a/Documentation/arm64/cpu-feature-registers.rst +++ b/Documentation/arm64/cpu-feature-registers.rst @@ -283,6 +283,14 @@ infrastructure: | AFP | [47-44] | y | +------------------------------+---------+---------+ + 9) ID_AA64ISAR2_EL1 - Instruction set attribute register 2 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | RPRES | [7-4] | y | + +------------------------------+---------+---------+ + Appendix I: Example ------------------- diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst index 77b6da64cfad..e88d245d426d 100644 --- a/Documentation/arm64/elf_hwcaps.rst +++ b/Documentation/arm64/elf_hwcaps.rst @@ -253,6 +253,10 @@ HWCAP2_AFP Functionality implied by ID_AA64MFR1_EL1.AFP == 0b0001. +HWCAP2_RPRES + + Functionality implied by ID_AA64ISAR2_EL1.RPRES == 0b0001. + 4. Unused AT_HWCAP bits ----------------------- diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index f20fbec36669..6422147ea612 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -107,6 +107,7 @@ #define KERNEL_HWCAP_MTE __khwcap2_feature(MTE) #define KERNEL_HWCAP_ECV __khwcap2_feature(ECV) #define KERNEL_HWCAP_AFP __khwcap2_feature(AFP) +#define KERNEL_HWCAP_RPRES __khwcap2_feature(RPRES) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 180da7396549..f03731847d9d 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -77,5 +77,6 @@ #define HWCAP2_MTE (1 << 18) #define HWCAP2_ECV (1 << 19) #define HWCAP2_AFP (1 << 20) +#define HWCAP2_RPRES (1 << 21) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 2e27e0d915e9..3aa0489262b1 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -227,6 +227,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_RPRES_SHIFT, 4, 0), ARM64_FTR_END, }; @@ -2446,6 +2447,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { #endif /* CONFIG_ARM64_MTE */ HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_ECV_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV), HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_AFP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP), + HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_RPRES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES), {}, }; diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 25300eb04cef..54af35fadd90 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -96,6 +96,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_MTE] = "mte", [KERNEL_HWCAP_ECV] = "ecv", [KERNEL_HWCAP_AFP] = "afp", + [KERNEL_HWCAP_RPRES] = "rpres", }; #ifdef CONFIG_COMPAT From 17867c11a27e498d7b49bbd9eadcfcb4b790c12f Mon Sep 17 00:00:00 2001 From: James Morse Date: Wed, 17 Nov 2021 15:15:26 +0000 Subject: [PATCH 56/80] UPSTREAM: arm64: entry.S: Add ventry overflow sanity checks commit 4330e2c5c04c27bebf89d34e0bc14e6943413067 upstream. Subsequent patches add even more code to the ventry slots. Ensure kernels that overflow a ventry slot don't get built. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I03d4525ca81ff83c726a9a43258604b9e165fadc --- arch/arm64/kernel/entry.S | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 2dd31309f7e6..90483c5cf6e6 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -62,6 +62,7 @@ .macro kernel_ventry, el, label, regsize = 64 .align 7 +.Lventry_start\@: #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 .if \el == 0 alternative_if ARM64_UNMAP_KERNEL_AT_EL0 @@ -120,6 +121,7 @@ alternative_else_nop_endif mrs x0, tpidrro_el0 #endif b el\()\el\()_\label +.org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm .macro tramp_alias, dst, sym @@ -929,6 +931,7 @@ alternative_else_nop_endif add x30, x30, #(1b - tramp_vectors) isb ret +.org 1b + 128 // Did we overflow the ventry slot? .endm .macro tramp_exit, regsize = 64 From c8b567d8881a443fd7708c928da96bcb39644968 Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 16 Nov 2021 15:00:51 +0000 Subject: [PATCH 57/80] UPSTREAM: arm64: spectre: Rename spectre_v4_patch_fw_mitigation_conduit commit 1b33d4860deaecf1d8eec3061b7e7ed7ab0bae8d upstream. The spectre-v4 sequence includes an SMC from the assembly entry code. spectre_v4_patch_fw_mitigation_conduit is the patching callback that generates an HVC or SMC depending on the SMCCC conduit type. As this isn't specific to spectre-v4, rename it smccc_patch_fw_mitigation_conduit so it can be re-used. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ie99c947a2d728bfb55738dda3833b2efeabb6e2e --- arch/arm64/kernel/entry.S | 2 +- arch/arm64/kernel/proton-pack.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 90483c5cf6e6..3664b28e8d07 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -143,7 +143,7 @@ alternative_cb_end tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@ mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2 mov w1, #\state -alternative_cb spectre_v4_patch_fw_mitigation_conduit +alternative_cb smccc_patch_fw_mitigation_conduit nop // Patched to SMC/HVC #0 alternative_cb_end .L__asm_ssbd_skip\@: diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 902e4084c477..9394f21d7566 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -554,9 +554,9 @@ void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt, * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction * to call into firmware to adjust the mitigation state. */ -void __init spectre_v4_patch_fw_mitigation_conduit(struct alt_instr *alt, - __le32 *origptr, - __le32 *updptr, int nr_inst) +void __init smccc_patch_fw_mitigation_conduit(struct alt_instr *alt, + __le32 *origptr, + __le32 *updptr, int nr_inst) { u32 insn; From 32ba6d5d6105679e02b19e2f6d4f3a7c83eb3174 Mon Sep 17 00:00:00 2001 From: James Morse Date: Wed, 24 Nov 2021 15:36:12 +0000 Subject: [PATCH 58/80] UPSTREAM: arm64: entry: Make the trampoline cleanup optional commit d739da1694a0eaef0358a42b76904b611539b77b upstream. Subsequent patches will add additional sets of vectors that use the same tricks as the kpti vectors to reach the full-fat vectors. The full-fat vectors contain some cleanup for kpti that is patched in by alternatives when kpti is in use. Once there are additional vectors, the cleanup will be needed in more cases. But on big/little systems, the cleanup would be harmful if no trampoline vector were in use. Instead of forcing CPUs that don't need a trampoline vector to use one, make the trampoline cleanup optional. Entry at the top of the vectors will skip the cleanup. The trampoline vectors can then skip the first instruction, triggering the cleanup to run. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ia62262831514064e298fd796e260969a8a46a0f8 --- arch/arm64/kernel/entry.S | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 3664b28e8d07..1c0741915b6b 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -65,14 +65,18 @@ .Lventry_start\@: #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 .if \el == 0 -alternative_if ARM64_UNMAP_KERNEL_AT_EL0 + /* + * This must be the first instruction of the EL0 vector entries. It is + * skipped by the trampoline vectors, to trigger the cleanup. + */ + b .Lskip_tramp_vectors_cleanup\@ .if \regsize == 64 mrs x30, tpidrro_el0 msr tpidrro_el0, xzr .else mov x30, xzr .endif -alternative_else_nop_endif +.Lskip_tramp_vectors_cleanup\@: .endif #endif @@ -928,7 +932,7 @@ alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM prfm plil1strm, [x30, #(1b - tramp_vectors)] alternative_else_nop_endif msr vbar_el1, x30 - add x30, x30, #(1b - tramp_vectors) + add x30, x30, #(1b - tramp_vectors + 4) isb ret .org 1b + 128 // Did we overflow the ventry slot? From bb8baaf3af43951ea0b77f6e10e4cfe483aafa37 Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 23 Nov 2021 18:41:43 +0000 Subject: [PATCH 59/80] UPSTREAM: arm64: entry: Free up another register on kpti's tramp_exit path commit 03aff3a77a58b5b52a77e00537a42090ad57b80b upstream. Kpti stashes x30 in far_el1 while it uses x30 for all its work. Making the vectors a per-cpu data structure will require a second register. Allow tramp_exit two registers before it unmaps the kernel, by leaving x30 on the stack, and stashing x29 in far_el1. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ib61b44ff4dcd83c18888fb60150c563a577518bf --- arch/arm64/kernel/entry.S | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 1c0741915b6b..edc4566368cd 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -456,14 +456,16 @@ alternative_else_nop_endif ldp x24, x25, [sp, #16 * 12] ldp x26, x27, [sp, #16 * 13] ldp x28, x29, [sp, #16 * 14] - ldr lr, [sp, #S_LR] - add sp, sp, #S_FRAME_SIZE // restore sp .if \el == 0 -alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 +alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 + ldr lr, [sp, #S_LR] + add sp, sp, #S_FRAME_SIZE // restore sp + eret +alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 bne 4f - msr far_el1, x30 + msr far_el1, x29 tramp_alias x30, tramp_exit_native br x30 4: @@ -471,6 +473,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 br x30 #endif .else + ldr lr, [sp, #S_LR] + add sp, sp, #S_FRAME_SIZE // restore sp + /* Ensure any device/NC reads complete */ alternative_insn nop, "dmb sy", ARM64_WORKAROUND_1508412 @@ -941,10 +946,12 @@ alternative_else_nop_endif .macro tramp_exit, regsize = 64 adr x30, tramp_vectors msr vbar_el1, x30 - tramp_unmap_kernel x30 + ldr lr, [sp, #S_LR] + tramp_unmap_kernel x29 .if \regsize == 64 - mrs x30, far_el1 + mrs x29, far_el1 .endif + add sp, sp, #S_FRAME_SIZE // restore sp eret sb .endm From 6e48449c91d048625526dfd67f4ba4f46fb15147 Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 23 Nov 2021 15:43:31 +0000 Subject: [PATCH 60/80] UPSTREAM: arm64: entry: Move the trampoline data page before the text page commit c091fb6ae059cda563b2a4d93fdbc548ef34e1d6 upstream. The trampoline code has a data page that holds the address of the vectors, which is unmapped when running in user-space. This ensures that with CONFIG_RANDOMIZE_BASE, the randomised address of the kernel can't be discovered until after the kernel has been mapped. If the trampoline text page is extended to include multiple sets of vectors, it will be larger than a single page, making it tricky to find the data page without knowing the size of the trampoline text pages, which will vary with PAGE_SIZE. Move the data page to appear before the text page. This allows the data page to be found without knowing the size of the trampoline text pages. 'tramp_vectors' is used to refer to the beginning of the .entry.tramp.text section, do that explicitly. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ia72d8b1946c6c63226be2f43c75f1924d027946e --- arch/arm64/include/asm/fixmap.h | 2 +- arch/arm64/kernel/entry.S | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 4335800201c9..0aabc0253b18 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -62,8 +62,8 @@ enum fixed_addresses { #endif /* CONFIG_ACPI_APEI_GHES */ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - FIX_ENTRY_TRAMP_DATA, FIX_ENTRY_TRAMP_TEXT, + FIX_ENTRY_TRAMP_DATA, #define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT)) #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ __end_of_permanent_fixed_addresses, diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index edc4566368cd..90fd24554181 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -911,6 +911,11 @@ alternative_else_nop_endif */ .endm + .macro tramp_data_page dst + adr \dst, .entry.tramp.text + sub \dst, \dst, PAGE_SIZE + .endm + .macro tramp_ventry, regsize = 64 .align 7 1: @@ -927,7 +932,7 @@ alternative_else_nop_endif 2: tramp_map_kernel x30 #ifdef CONFIG_RANDOMIZE_BASE - adr x30, tramp_vectors + PAGE_SIZE + tramp_data_page x30 alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003 ldr x30, [x30] #else @@ -1081,7 +1086,7 @@ SYM_CODE_START(__sdei_asm_entry_trampoline) 1: str x4, [x1, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)] #ifdef CONFIG_RANDOMIZE_BASE - adr x4, tramp_vectors + PAGE_SIZE + tramp_data_page x4 add x4, x4, #:lo12:__sdei_asm_trampoline_next_handler ldr x4, [x4] #else From 0defb52ce6620b96fb3d1314e3ad3f7377cd0d09 Mon Sep 17 00:00:00 2001 From: James Morse Date: Wed, 24 Nov 2021 11:40:18 +0000 Subject: [PATCH 61/80] UPSTREAM: arm64: entry: Allow tramp_alias to access symbols after the 4K boundary commit 6c5bf79b69f911560fbf82214c0971af6e58e682 upstream. Systems using kpti enter and exit the kernel through a trampoline mapping that is always mapped, even when the kernel is not. tramp_valias is a macro to find the address of a symbol in the trampoline mapping. Adding extra sets of vectors will expand the size of the entry.tramp.text section to beyond 4K. tramp_valias will be unable to generate addresses for symbols beyond 4K as it uses the 12 bit immediate of the add instruction. As there are now two registers available when tramp_alias is called, use the extra register to avoid the 4K limit of the 12 bit immediate. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ic7a70f79baeacb3c8d12c396009968e563a2e69a --- arch/arm64/kernel/entry.S | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 90fd24554181..3c32ebc6006b 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -128,9 +128,12 @@ .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm - .macro tramp_alias, dst, sym + .macro tramp_alias, dst, sym, tmp mov_q \dst, TRAMP_VALIAS - add \dst, \dst, #(\sym - .entry.tramp.text) + adr_l \tmp, \sym + add \dst, \dst, \tmp + adr_l \tmp, .entry.tramp.text + sub \dst, \dst, \tmp .endm /* @@ -466,10 +469,10 @@ alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 bne 4f msr far_el1, x29 - tramp_alias x30, tramp_exit_native + tramp_alias x30, tramp_exit_native, x29 br x30 4: - tramp_alias x30, tramp_exit_compat + tramp_alias x30, tramp_exit_compat, x29 br x30 #endif .else @@ -1228,7 +1231,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline + tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3 br x5 #endif SYM_CODE_END(__sdei_asm_handler) From 00d8bb6b906c56fae93b04c5195c33504eb198b3 Mon Sep 17 00:00:00 2001 From: James Morse Date: Wed, 24 Nov 2021 13:40:09 +0000 Subject: [PATCH 62/80] UPSTREAM: arm64: entry: Don't assume tramp_vectors is the start of the vectors commit ed50da7764535f1e24432ded289974f2bf2b0c5a upstream. The tramp_ventry macro uses tramp_vectors as the address of the vectors when calculating which ventry in the 'full fat' vectors to branch to. While there is one set of tramp_vectors, this will be true. Adding multiple sets of vectors will break this assumption. Move the generation of the vectors to a macro, and pass the start of the vectors as an argument to tramp_ventry. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Idff2a8dcbb088edb40ea0714212ec966d4c065da --- arch/arm64/kernel/entry.S | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 3c32ebc6006b..3e632674261d 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -919,7 +919,7 @@ alternative_else_nop_endif sub \dst, \dst, PAGE_SIZE .endm - .macro tramp_ventry, regsize = 64 + .macro tramp_ventry, vector_start, regsize .align 7 1: .if \regsize == 64 @@ -942,10 +942,10 @@ alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003 ldr x30, =vectors #endif alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM - prfm plil1strm, [x30, #(1b - tramp_vectors)] + prfm plil1strm, [x30, #(1b - \vector_start)] alternative_else_nop_endif msr vbar_el1, x30 - add x30, x30, #(1b - tramp_vectors + 4) + add x30, x30, #(1b - \vector_start + 4) isb ret .org 1b + 128 // Did we overflow the ventry slot? @@ -964,19 +964,21 @@ alternative_else_nop_endif sb .endm - .align 11 -SYM_CODE_START_NOALIGN(tramp_vectors) + .macro generate_tramp_vector +.Lvector_start\@: .space 0x400 - tramp_ventry - tramp_ventry - tramp_ventry - tramp_ventry + .rept 4 + tramp_ventry .Lvector_start\@, 64 + .endr + .rept 4 + tramp_ventry .Lvector_start\@, 32 + .endr + .endm - tramp_ventry 32 - tramp_ventry 32 - tramp_ventry 32 - tramp_ventry 32 + .align 11 +SYM_CODE_START_NOALIGN(tramp_vectors) + generate_tramp_vector SYM_CODE_END(tramp_vectors) SYM_CODE_START(tramp_exit_native) From e6408b96a01ad8c1b86e5de01991f014728590e9 Mon Sep 17 00:00:00 2001 From: James Morse Date: Thu, 18 Nov 2021 14:02:30 +0000 Subject: [PATCH 63/80] UPSTREAM: arm64: entry: Move trampoline macros out of ifdef'd section commit 13d7a08352a83ef2252aeb464a5e08dfc06b5dfd upstream. The macros for building the kpti trampoline are all behind CONFIG_UNMAP_KERNEL_AT_EL0, and in a region that outputs to the .entry.tramp.text section. Move the macros out so they can be used to generate other kinds of trampoline. Only the symbols need to be guarded by CONFIG_UNMAP_KERNEL_AT_EL0 and appear in the .entry.tramp.text section. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I7ce53531765df5ac4eb9a4d814ab561a6df76931 --- arch/arm64/kernel/entry.S | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 3e632674261d..d5c2ddad8d11 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -875,12 +875,6 @@ SYM_CODE_END(ret_to_user) .popsection // .entry.text -#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 -/* - * Exception vectors trampoline. - */ - .pushsection ".entry.tramp.text", "ax" - // Move from tramp_pg_dir to swapper_pg_dir .macro tramp_map_kernel, tmp mrs \tmp, ttbr1_el1 @@ -976,6 +970,11 @@ alternative_else_nop_endif .endr .endm +#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +/* + * Exception vectors trampoline. + */ + .pushsection ".entry.tramp.text", "ax" .align 11 SYM_CODE_START_NOALIGN(tramp_vectors) generate_tramp_vector From 158c87e50aaf4334b88e8c19f3cbbd792b884049 Mon Sep 17 00:00:00 2001 From: James Morse Date: Thu, 18 Nov 2021 13:16:23 +0000 Subject: [PATCH 64/80] UPSTREAM: arm64: entry: Make the kpti trampoline's kpti sequence optional commit c47e4d04ba0f1ea17353d85d45f611277507e07a upstream. Spectre-BHB needs to add sequences to the vectors. Having one global set of vectors is a problem for big/little systems where the sequence is costly on cpus that are not vulnerable. Making the vectors per-cpu in the style of KVM's bh_harden_hyp_vecs requires the vectors to be generated by macros. Make the kpti re-mapping of the kernel optional, so the macros can be used without kpti. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I2c20a42404a0cf79df9f9ccc69328ed986125208 --- arch/arm64/kernel/entry.S | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index d5c2ddad8d11..879e3bb2dc43 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -913,9 +913,10 @@ alternative_else_nop_endif sub \dst, \dst, PAGE_SIZE .endm - .macro tramp_ventry, vector_start, regsize + .macro tramp_ventry, vector_start, regsize, kpti .align 7 1: + .if \kpti == 1 .if \regsize == 64 msr tpidrro_el0, x30 // Restored in kernel_ventry .endif @@ -938,9 +939,14 @@ alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003 alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM prfm plil1strm, [x30, #(1b - \vector_start)] alternative_else_nop_endif + msr vbar_el1, x30 - add x30, x30, #(1b - \vector_start + 4) isb + .else + ldr x30, =vectors + .endif // \kpti == 1 + + add x30, x30, #(1b - \vector_start + 4) ret .org 1b + 128 // Did we overflow the ventry slot? .endm @@ -958,15 +964,15 @@ alternative_else_nop_endif sb .endm - .macro generate_tramp_vector + .macro generate_tramp_vector, kpti .Lvector_start\@: .space 0x400 .rept 4 - tramp_ventry .Lvector_start\@, 64 + tramp_ventry .Lvector_start\@, 64, \kpti .endr .rept 4 - tramp_ventry .Lvector_start\@, 32 + tramp_ventry .Lvector_start\@, 32, \kpti .endr .endm @@ -977,7 +983,7 @@ alternative_else_nop_endif .pushsection ".entry.tramp.text", "ax" .align 11 SYM_CODE_START_NOALIGN(tramp_vectors) - generate_tramp_vector + generate_tramp_vector kpti=1 SYM_CODE_END(tramp_vectors) SYM_CODE_START(tramp_exit_native) From 0db372ec4bf4319b8c925e5a4bdfd19324debf7a Mon Sep 17 00:00:00 2001 From: James Morse Date: Thu, 18 Nov 2021 15:04:32 +0000 Subject: [PATCH 65/80] UPSTREAM: arm64: entry: Allow the trampoline text to occupy multiple pages commit a9c406e6462ff14956d690de7bbe5131a5677dc9 upstream. Adding a second set of vectors to .entry.tramp.text will make it larger than a single 4K page. Allow the trampoline text to occupy up to three pages by adding two more fixmap slots. Previous changes to tramp_valias allowed it to reach beyond a single page. Bug: 215557547 Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ibd280a2b20b8e0ea42ab76b86afa84920cc25eee --- arch/arm64/include/asm/fixmap.h | 6 ++++-- arch/arm64/include/asm/sections.h | 5 +++++ arch/arm64/kernel/entry.S | 2 +- arch/arm64/kernel/vmlinux.lds.S | 2 +- arch/arm64/mm/mmu.c | 12 +++++++++--- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 0aabc0253b18..daff882883f9 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -62,9 +62,11 @@ enum fixed_addresses { #endif /* CONFIG_ACPI_APEI_GHES */ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - FIX_ENTRY_TRAMP_TEXT, + FIX_ENTRY_TRAMP_TEXT3, + FIX_ENTRY_TRAMP_TEXT2, + FIX_ENTRY_TRAMP_TEXT1, FIX_ENTRY_TRAMP_DATA, -#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT)) +#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1)) #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ __end_of_permanent_fixed_addresses, diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h index e4ad9db53af1..552891e626e5 100644 --- a/arch/arm64/include/asm/sections.h +++ b/arch/arm64/include/asm/sections.h @@ -22,4 +22,9 @@ extern char __irqentry_text_start[], __irqentry_text_end[]; extern char __mmuoff_data_start[], __mmuoff_data_end[]; extern char __entry_tramp_text_start[], __entry_tramp_text_end[]; +static inline size_t entry_tramp_text_size(void) +{ + return __entry_tramp_text_end - __entry_tramp_text_start; +} + #endif /* __ASM_SECTIONS_H */ diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 879e3bb2dc43..73553e52151c 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -909,7 +909,7 @@ alternative_else_nop_endif .endm .macro tramp_data_page dst - adr \dst, .entry.tramp.text + adr_l \dst, .entry.tramp.text sub \dst, \dst, PAGE_SIZE .endm diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index b00961be177d..786a635b3581 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -330,7 +330,7 @@ ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1)) <= SZ_4K, "Hibernate exit text too big or misaligned") #endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 -ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) == PAGE_SIZE, +ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) <= 3*PAGE_SIZE, "Entry trampoline text too big") #endif #ifdef CONFIG_KVM diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 542f14c61fb3..09867e36ae58 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -593,6 +593,8 @@ early_param("rodata", parse_rodata); #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 static int __init map_entry_trampoline(void) { + int i; + pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start); @@ -601,11 +603,15 @@ static int __init map_entry_trampoline(void) /* Map only the text into the trampoline page table */ memset(tramp_pg_dir, 0, PGD_SIZE); - __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE, - prot, __pgd_pgtable_alloc, 0); + __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, + entry_tramp_text_size(), prot, + __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS); /* Map both the text and data into the kernel page table */ - __set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot); + for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++) + __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, + pa_start + i * PAGE_SIZE, prot); + if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { extern char __entry_tramp_data_start[]; From 2a90cf9af267c6b6c62df532398067a0f6ef659f Mon Sep 17 00:00:00 2001 From: James Morse Date: Wed, 24 Nov 2021 15:03:15 +0000 Subject: [PATCH 66/80] UPSTREAM: arm64: entry: Add non-kpti __bp_harden_el1_vectors for mitigations commit aff65393fa1401e034656e349abd655cfe272de0 upstream. kpti is an optional feature, for systems not using kpti a set of vectors for the spectre-bhb mitigations is needed. Add another set of vectors, __bp_harden_el1_vectors, that will be used if a mitigation is needed and kpti is not in use. The EL1 ventries are repeated verbatim as there is no additional work needed for entry from EL1. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I0e72c5c1c366d3906bc1525e24a367989dfc98af --- arch/arm64/kernel/entry.S | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 73553e52151c..cf0d7145871f 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -916,10 +916,11 @@ alternative_else_nop_endif .macro tramp_ventry, vector_start, regsize, kpti .align 7 1: - .if \kpti == 1 .if \regsize == 64 msr tpidrro_el0, x30 // Restored in kernel_ventry .endif + + .if \kpti == 1 /* * Defend against branch aliasing attacks by pushing a dummy * entry onto the return stack and using a RET instruction to @@ -1006,6 +1007,38 @@ SYM_DATA_END(__entry_tramp_data_start) #endif /* CONFIG_RANDOMIZE_BASE */ #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ +/* + * Exception vectors for spectre mitigations on entry from EL1 when + * kpti is not in use. + */ + .macro generate_el1_vector +.Lvector_start\@: + kernel_ventry 1, sync_invalid // Synchronous EL1t + kernel_ventry 1, irq_invalid // IRQ EL1t + kernel_ventry 1, fiq_invalid // FIQ EL1t + kernel_ventry 1, error_invalid // Error EL1t + + kernel_ventry 1, sync // Synchronous EL1h + kernel_ventry 1, irq // IRQ EL1h + kernel_ventry 1, fiq_invalid // FIQ EL1h + kernel_ventry 1, error // Error EL1h + + .rept 4 + tramp_ventry .Lvector_start\@, 64, kpti=0 + .endr + .rept 4 + tramp_ventry .Lvector_start\@, 32, kpti=0 + .endr + .endm + + .pushsection ".entry.text", "ax" + .align 11 +SYM_CODE_START(__bp_harden_el1_vectors) + generate_el1_vector +SYM_CODE_END(__bp_harden_el1_vectors) + .popsection + + /* * Register switch for AArch64. The callee-saved registers need to be saved * and restored. On entry: From e322fe26a1a80f65ad84e74f6ef1985df995a4f1 Mon Sep 17 00:00:00 2001 From: James Morse Date: Thu, 18 Nov 2021 13:59:46 +0000 Subject: [PATCH 67/80] UPSTREAM: arm64: entry: Add vectors that have the bhb mitigation sequences commit ba2689234be92024e5635d30fe744f4853ad97db upstream. Some CPUs affected by Spectre-BHB need a sequence of branches, or a firmware call to be run before any indirect branch. This needs to go in the vectors. No CPU needs both. While this can be patched in, it would run on all CPUs as there is a single set of vectors. If only one part of a big/little combination is affected, the unaffected CPUs have to run the mitigation too. Create extra vectors that include the sequence. Subsequent patches will allow affected CPUs to select this set of vectors. Later patches will modify the loop count to match what the CPU requires. Bug: 215557547 Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I2c9addc82ad9b42fff4a4d87d57c3c9f431f8c20 --- arch/arm64/include/asm/assembler.h | 24 ++++++++++++++ arch/arm64/include/asm/vectors.h | 34 +++++++++++++++++++ arch/arm64/kernel/entry.S | 53 +++++++++++++++++++++++++----- include/linux/arm-smccc.h | 5 +++ 4 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 arch/arm64/include/asm/vectors.h diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 4eed9601d9d4..107e0f4651a3 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -801,4 +801,28 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU #endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */ + .macro __mitigate_spectre_bhb_loop tmp +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY + mov \tmp, #32 +.Lspectre_bhb_loop\@: + b . + 4 + subs \tmp, \tmp, #1 + b.ne .Lspectre_bhb_loop\@ + sb +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + .endm + + /* Save/restores x0-x3 to the stack */ + .macro __mitigate_spectre_bhb_fw +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 +alternative_cb smccc_patch_fw_mitigation_conduit + nop // Patched to SMC/HVC #0 +alternative_cb_end + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + .endm #endif /* __ASM_ASSEMBLER_H */ diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h new file mode 100644 index 000000000000..16ca74260375 --- /dev/null +++ b/arch/arm64/include/asm/vectors.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2022 ARM Ltd. + */ +#ifndef __ASM_VECTORS_H +#define __ASM_VECTORS_H + +/* + * Note: the order of this enum corresponds to two arrays in entry.S: + * tramp_vecs and __bp_harden_el1_vectors. By default the canonical + * 'full fat' vectors are used directly. + */ +enum arm64_bp_harden_el1_vectors { +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY + /* + * Perform the BHB loop mitigation, before branching to the canonical + * vectors. + */ + EL1_VECTOR_BHB_LOOP, + + /* + * Make the SMC call for firmware mitigation, before branching to the + * canonical vectors. + */ + EL1_VECTOR_BHB_FW, +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + + /* + * Remap the kernel before branching to the canonical vectors. + */ + EL1_VECTOR_KPTI, +}; + +#endif /* __ASM_VECTORS_H */ diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index cf0d7145871f..0b9b182e0be3 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -913,13 +913,26 @@ alternative_else_nop_endif sub \dst, \dst, PAGE_SIZE .endm - .macro tramp_ventry, vector_start, regsize, kpti + +#define BHB_MITIGATION_NONE 0 +#define BHB_MITIGATION_LOOP 1 +#define BHB_MITIGATION_FW 2 + + .macro tramp_ventry, vector_start, regsize, kpti, bhb .align 7 1: .if \regsize == 64 msr tpidrro_el0, x30 // Restored in kernel_ventry .endif + .if \bhb == BHB_MITIGATION_LOOP + /* + * This sequence must appear before the first indirect branch. i.e. the + * ret out of tramp_ventry. It appears here because x30 is free. + */ + __mitigate_spectre_bhb_loop x30 + .endif // \bhb == BHB_MITIGATION_LOOP + .if \kpti == 1 /* * Defend against branch aliasing attacks by pushing a dummy @@ -947,6 +960,15 @@ alternative_else_nop_endif ldr x30, =vectors .endif // \kpti == 1 + .if \bhb == BHB_MITIGATION_FW + /* + * The firmware sequence must appear before the first indirect branch. + * i.e. the ret out of tramp_ventry. But it also needs the stack to be + * mapped to save/restore the registers the SMC clobbers. + */ + __mitigate_spectre_bhb_fw + .endif // \bhb == BHB_MITIGATION_FW + add x30, x30, #(1b - \vector_start + 4) ret .org 1b + 128 // Did we overflow the ventry slot? @@ -954,6 +976,9 @@ alternative_else_nop_endif .macro tramp_exit, regsize = 64 adr x30, tramp_vectors +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY + add x30, x30, SZ_4K +#endif msr vbar_el1, x30 ldr lr, [sp, #S_LR] tramp_unmap_kernel x29 @@ -965,26 +990,32 @@ alternative_else_nop_endif sb .endm - .macro generate_tramp_vector, kpti + .macro generate_tramp_vector, kpti, bhb .Lvector_start\@: .space 0x400 .rept 4 - tramp_ventry .Lvector_start\@, 64, \kpti + tramp_ventry .Lvector_start\@, 64, \kpti, \bhb .endr .rept 4 - tramp_ventry .Lvector_start\@, 32, \kpti + tramp_ventry .Lvector_start\@, 32, \kpti, \bhb .endr .endm #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 /* * Exception vectors trampoline. + * The order must match __bp_harden_el1_vectors and the + * arm64_bp_harden_el1_vectors enum. */ .pushsection ".entry.tramp.text", "ax" .align 11 SYM_CODE_START_NOALIGN(tramp_vectors) - generate_tramp_vector kpti=1 +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE SYM_CODE_END(tramp_vectors) SYM_CODE_START(tramp_exit_native) @@ -1011,7 +1042,7 @@ SYM_DATA_END(__entry_tramp_data_start) * Exception vectors for spectre mitigations on entry from EL1 when * kpti is not in use. */ - .macro generate_el1_vector + .macro generate_el1_vector, bhb .Lvector_start\@: kernel_ventry 1, sync_invalid // Synchronous EL1t kernel_ventry 1, irq_invalid // IRQ EL1t @@ -1024,17 +1055,21 @@ SYM_DATA_END(__entry_tramp_data_start) kernel_ventry 1, error // Error EL1h .rept 4 - tramp_ventry .Lvector_start\@, 64, kpti=0 + tramp_ventry .Lvector_start\@, 64, 0, \bhb .endr .rept 4 - tramp_ventry .Lvector_start\@, 32, kpti=0 + tramp_ventry .Lvector_start\@, 32, 0, \bhb .endr .endm +/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */ .pushsection ".entry.text", "ax" .align 11 SYM_CODE_START(__bp_harden_el1_vectors) - generate_el1_vector +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY + generate_el1_vector bhb=BHB_MITIGATION_LOOP + generate_el1_vector bhb=BHB_MITIGATION_FW +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ SYM_CODE_END(__bp_harden_el1_vectors) .popsection diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h index 62c54234576c..c07a4ef9897a 100644 --- a/include/linux/arm-smccc.h +++ b/include/linux/arm-smccc.h @@ -87,6 +87,11 @@ ARM_SMCCC_SMC_32, \ 0, 0x7fff) +#define ARM_SMCCC_ARCH_WORKAROUND_3 \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + 0, 0x3fff) + #define SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED 1 /* Paravirtualised time calls (defined by ARM DEN0057A) */ From 28837e415dd6b2b09bd677328aad23eddc3e5a1b Mon Sep 17 00:00:00 2001 From: James Morse Date: Thu, 25 Nov 2021 14:25:34 +0000 Subject: [PATCH 68/80] UPSTREAM: arm64: entry: Add macro for reading symbol addresses from the trampoline commit b28a8eebe81c186fdb1a0078263b30576c8e1f42 upstream. The trampoline code needs to use the address of symbols in the wider kernel, e.g. vectors. PC-relative addressing wouldn't work as the trampoline code doesn't run at the address the linker expected. tramp_ventry uses a literal pool, unless CONFIG_RANDOMIZE_BASE is set, in which case it uses the data page as a literal pool because the data page can be unmapped when running in user-space, which is required for CPUs vulnerable to meltdown. Pull this logic out as a macro, instead of adding a third copy of it. Bug: 215557547 Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I1d11f54bb867a6312069bc3fae6f4ceb4ab1fd20 --- arch/arm64/kernel/entry.S | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 0b9b182e0be3..b8212b6416a3 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -913,6 +913,15 @@ alternative_else_nop_endif sub \dst, \dst, PAGE_SIZE .endm + .macro tramp_data_read_var dst, var +#ifdef CONFIG_RANDOMIZE_BASE + tramp_data_page \dst + add \dst, \dst, #:lo12:__entry_tramp_data_\var + ldr \dst, [\dst] +#else + ldr \dst, =\var +#endif + .endm #define BHB_MITIGATION_NONE 0 #define BHB_MITIGATION_LOOP 1 @@ -943,13 +952,8 @@ alternative_else_nop_endif b . 2: tramp_map_kernel x30 -#ifdef CONFIG_RANDOMIZE_BASE - tramp_data_page x30 alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003 - ldr x30, [x30] -#else - ldr x30, =vectors -#endif + tramp_data_read_var x30, vectors alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM prfm plil1strm, [x30, #(1b - \vector_start)] alternative_else_nop_endif @@ -1032,7 +1036,12 @@ SYM_CODE_END(tramp_exit_compat) .pushsection ".rodata", "a" .align PAGE_SHIFT SYM_DATA_START(__entry_tramp_data_start) +__entry_tramp_data_vectors: .quad vectors +#ifdef CONFIG_ARM_SDE_INTERFACE +__entry_tramp_data___sdei_asm_handler: + .quad __sdei_asm_handler +#endif /* CONFIG_ARM_SDE_INTERFACE */ SYM_DATA_END(__entry_tramp_data_start) .popsection // .rodata #endif /* CONFIG_RANDOMIZE_BASE */ @@ -1163,13 +1172,7 @@ SYM_CODE_START(__sdei_asm_entry_trampoline) */ 1: str x4, [x1, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)] -#ifdef CONFIG_RANDOMIZE_BASE - tramp_data_page x4 - add x4, x4, #:lo12:__sdei_asm_trampoline_next_handler - ldr x4, [x4] -#else - ldr x4, =__sdei_asm_handler -#endif + tramp_data_read_var x4, __sdei_asm_handler br x4 SYM_CODE_END(__sdei_asm_entry_trampoline) NOKPROBE(__sdei_asm_entry_trampoline) @@ -1192,13 +1195,6 @@ SYM_CODE_END(__sdei_asm_exit_trampoline) NOKPROBE(__sdei_asm_exit_trampoline) .ltorg .popsection // .entry.tramp.text -#ifdef CONFIG_RANDOMIZE_BASE -.pushsection ".rodata", "a" -SYM_DATA_START(__sdei_asm_trampoline_next_handler) - .quad __sdei_asm_handler -SYM_DATA_END(__sdei_asm_trampoline_next_handler) -.popsection // .rodata -#endif /* CONFIG_RANDOMIZE_BASE */ #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ /* From 30180ef431768893926a5a8973c357b55ad228f5 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Tue, 15 Mar 2022 11:05:26 -0700 Subject: [PATCH 69/80] Revert "BACKPORT: FROMLIST: scsi: core: Reserve one tag for the UFS driver" Revert commit ff748ee29819 since it is no longer needed now that commit a8f9df1ffcf2 ("FROMGIT: scsi: ufs: Fix a deadlock in the error handler") has been applied. This patch increases the UFS queue depth from 30 to 31. For the original patch, see also https://android-review.googlesource.com/q/I6273114ae8cc6c2a74c72f7bc090eb0319ec5772 Bug: 220258338 Test: $ adb shell cat /sys/devices/platform/*/host0/scsi_host/host0/can_queue Test: 31 Test: $ fio --bs=4096 --group_reporting=1 --gtod_reduce=1 --invalidate=1 --ioengine=io_uring --ioscheduler=none --norandommap --runtime=30 --rw=randread --thread --time_based=1 --numjobs=1 --iodepth=64 --iodepth_batch_submit=32 --iodepth_batch_complete=32 --name=/dev/block/sda --filename=/dev/block/sda Test: IOPS=28.6K (with a debug kernel) Change-Id: Ie6afb17fde2f688967e84c50161b4526bb7845e7 Signed-off-by: Bart Van Assche --- drivers/scsi/hosts.c | 10 ++++------ drivers/scsi/scsi_lib.c | 4 ---- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c index 6d5a8c5835e1..d664c4650b2d 100644 --- a/drivers/scsi/hosts.c +++ b/drivers/scsi/hosts.c @@ -220,6 +220,10 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, goto fail; } + /* Use min_t(int, ...) in case shost->can_queue exceeds SHRT_MAX */ + shost->cmd_per_lun = min_t(int, shost->cmd_per_lun, + shost->can_queue); + error = scsi_init_sense_cache(shost); if (error) goto fail; @@ -228,12 +232,6 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, if (error) goto fail; - shost->can_queue = shost->tag_set.queue_depth; - - /* Use min_t(int, ...) in case shost->can_queue exceeds SHRT_MAX */ - shost->cmd_per_lun = min_t(int, shost->cmd_per_lun, - shost->can_queue); - if (!shost->shost_gendev.parent) shost->shost_gendev.parent = dev ? dev : &platform_bus; if (!dma_dev) diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 3dfdc7daa664..6f3d29d16d1f 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1907,10 +1907,6 @@ int scsi_mq_setup_tags(struct Scsi_Host *shost) tag_set->ops = &scsi_mq_ops_no_commit; tag_set->nr_hw_queues = shost->nr_hw_queues ? : 1; tag_set->queue_depth = shost->can_queue; - if (shost->hostt->name && strcmp(shost->hostt->name, "ufshcd") == 0) { - tag_set->queue_depth--; - tag_set->reserved_tags++; - } tag_set->cmd_size = cmd_size; tag_set->numa_node = NUMA_NO_NODE; tag_set->flags = BLK_MQ_F_SHOULD_MERGE; From d95b0b4e5dc29079065d785c425972799bfd30da Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 23 Nov 2021 18:29:25 +0000 Subject: [PATCH 70/80] UPSTREAM: arm64: Add percpu vectors for EL1 commit bd09128d16fac3c34b80bd6a29088ac632e8ce09 upstream. The Spectre-BHB workaround adds a firmware call to the vectors. This is needed on some CPUs, but not others. To avoid the unaffected CPU in a big/little pair from making the firmware call, create per cpu vectors. The per-cpu vectors only apply when returning from EL0. Systems using KPTI can use the canonical 'full-fat' vectors directly at EL1, the trampoline exit code will switch to this_cpu_vector on exit to EL0. Systems not using KPTI should always use this_cpu_vector. this_cpu_vector will point at a vector in tramp_vecs or __bp_harden_el1_vectors, depending on whether KPTI is in use. Bug: 215557547 Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ie96a6896e26d54d09143ec668a0bdfbf21fa2618 --- arch/arm64/include/asm/vectors.h | 27 +++++++++++++++++++++++++++ arch/arm64/kernel/cpufeature.c | 11 +++++++++++ arch/arm64/kernel/entry.S | 12 ++++++------ arch/arm64/kvm/hyp/vhe/switch.c | 9 +++++++-- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h index 16ca74260375..3f76dfd9e074 100644 --- a/arch/arm64/include/asm/vectors.h +++ b/arch/arm64/include/asm/vectors.h @@ -5,6 +5,15 @@ #ifndef __ASM_VECTORS_H #define __ASM_VECTORS_H +#include +#include + +#include + +extern char vectors[]; +extern char tramp_vectors[]; +extern char __bp_harden_el1_vectors[]; + /* * Note: the order of this enum corresponds to two arrays in entry.S: * tramp_vecs and __bp_harden_el1_vectors. By default the canonical @@ -31,4 +40,22 @@ enum arm64_bp_harden_el1_vectors { EL1_VECTOR_KPTI, }; +/* The vectors to use on return from EL0. e.g. to remap the kernel */ +DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector); + +#ifndef CONFIG_UNMAP_KERNEL_AT_EL0 +#define TRAMP_VALIAS 0 +#endif + +static inline const char * +arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot) +{ + if (arm64_kernel_unmapped_at_el0()) + return (char *)TRAMP_VALIAS + SZ_2K * slot; + + WARN_ON_ONCE(slot == EL1_VECTOR_KPTI); + + return __bp_harden_el1_vectors + SZ_2K * slot; +} + #endif /* __ASM_VECTORS_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 3aa0489262b1..21f0d84e74ac 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -65,6 +65,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,7 @@ #include #include #include + #include #include #include @@ -82,6 +84,7 @@ #include #include #include +#include #include /* Kernel representation of AT_HWCAP and AT_HWCAP2 */ @@ -107,6 +110,8 @@ DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE); bool arm64_use_ng_mappings = false; EXPORT_SYMBOL(arm64_use_ng_mappings); +DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors; + /* * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs * support it? @@ -1557,6 +1562,12 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) int cpu = smp_processor_id(); + if (__this_cpu_read(this_cpu_vector) == vectors) { + const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI); + + __this_cpu_write(this_cpu_vector, v); + } + /* * We don't need to rewrite the page-tables if either we've done * it already or we have KASLR enabled and therefore have not diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index b8212b6416a3..08a82f4c8150 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -63,7 +63,6 @@ .macro kernel_ventry, el, label, regsize = 64 .align 7 .Lventry_start\@: -#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 .if \el == 0 /* * This must be the first instruction of the EL0 vector entries. It is @@ -78,7 +77,6 @@ .endif .Lskip_tramp_vectors_cleanup\@: .endif -#endif sub sp, sp, #S_FRAME_SIZE #ifdef CONFIG_VMAP_STACK @@ -979,10 +977,10 @@ alternative_else_nop_endif .endm .macro tramp_exit, regsize = 64 - adr x30, tramp_vectors -#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY - add x30, x30, SZ_4K -#endif + tramp_data_read_var x30, this_cpu_vector + this_cpu_offset x29 + ldr x30, [x30, x29] + msr vbar_el1, x30 ldr lr, [sp, #S_LR] tramp_unmap_kernel x29 @@ -1042,6 +1040,8 @@ __entry_tramp_data_vectors: __entry_tramp_data___sdei_asm_handler: .quad __sdei_asm_handler #endif /* CONFIG_ARM_SDE_INTERFACE */ +__entry_tramp_data_this_cpu_vector: + .quad this_cpu_vector SYM_DATA_END(__entry_tramp_data_start) .popsection // .rodata #endif /* CONFIG_RANDOMIZE_BASE */ diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index af8e940d0f03..e57e5d35a017 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -26,6 +27,7 @@ #include #include #include +#include const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n"; @@ -71,7 +73,7 @@ NOKPROBE_SYMBOL(__activate_traps); static void __deactivate_traps(struct kvm_vcpu *vcpu) { - extern char vectors[]; /* kernel exception vectors */ + const char *host_vectors = vectors; ___deactivate_traps(vcpu); @@ -85,7 +87,10 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1); - write_sysreg(vectors, vbar_el1); + + if (!arm64_kernel_unmapped_at_el0()) + host_vectors = __this_cpu_read(this_cpu_vector); + write_sysreg(host_vectors, vbar_el1); } NOKPROBE_SYMBOL(__deactivate_traps); From fee1ae7c7cde31a2344f2a7afaab26aa55546c39 Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 8 Feb 2022 16:08:13 +0000 Subject: [PATCH 71/80] UPSTREAM: arm64: proton-pack: Report Spectre-BHB vulnerabilities as part of Spectre-v2 commit dee435be76f4117410bbd90573a881fd33488f37 upstream. Speculation attacks against some high-performance processors can make use of branch history to influence future speculation as part of a spectre-v2 attack. This is not mitigated by CSV2, meaning CPUs that previously reported 'Not affected' are now moderately mitigated by CSV2. Update the value in /sys/devices/system/cpu/vulnerabilities/spectre_v2 to also show the state of the BHB mitigation. Bug: 215557547 Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I3a6aeb077869dc310d420c27f6752dfeacf438a6 --- arch/arm64/include/asm/spectre.h | 1 + arch/arm64/kernel/proton-pack.c | 36 ++++++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/spectre.h b/arch/arm64/include/asm/spectre.h index f62ca39da6c5..84b174b57aae 100644 --- a/arch/arm64/include/asm/spectre.h +++ b/arch/arm64/include/asm/spectre.h @@ -92,6 +92,7 @@ void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused) void spectre_v4_enable_task_mitigation(struct task_struct *tsk); enum mitigation_state arm64_get_meltdown_state(void); +enum mitigation_state arm64_get_spectre_bhb_state(void); #endif /* __ASSEMBLY__ */ #endif /* __ASM_SPECTRE_H */ diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 9394f21d7566..b4bef7707e19 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -96,14 +96,39 @@ static bool spectre_v2_mitigations_off(void) return ret; } +static const char *get_bhb_affected_string(enum mitigation_state bhb_state) +{ + switch (bhb_state) { + case SPECTRE_UNAFFECTED: + return ""; + default: + case SPECTRE_VULNERABLE: + return ", but not BHB"; + case SPECTRE_MITIGATED: + return ", BHB"; + } +} + ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) { + enum mitigation_state bhb_state = arm64_get_spectre_bhb_state(); + const char *bhb_str = get_bhb_affected_string(bhb_state); + const char *v2_str = "Branch predictor hardening"; + switch (spectre_v2_state) { case SPECTRE_UNAFFECTED: - return sprintf(buf, "Not affected\n"); + if (bhb_state == SPECTRE_UNAFFECTED) + return sprintf(buf, "Not affected\n"); + + /* + * Platforms affected by Spectre-BHB can't report + * "Not affected" for Spectre-v2. + */ + v2_str = "CSV2"; + fallthrough; case SPECTRE_MITIGATED: - return sprintf(buf, "Mitigation: Branch predictor hardening\n"); + return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str); case SPECTRE_VULNERABLE: fallthrough; default: @@ -770,3 +795,10 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) return -ENODEV; } } + +static enum mitigation_state spectre_bhb_state; + +enum mitigation_state arm64_get_spectre_bhb_state(void) +{ + return spectre_bhb_state; +} From e9a39a642cb573cb664e2527ca20a4f3bb4e700f Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 16 Nov 2021 15:06:19 +0000 Subject: [PATCH 72/80] UPSTREAM: KVM: arm64: Allow indirect vectors to be used without SPECTRE_V3A CPUs vulnerable to Spectre-BHB either need to make an SMC-CC firmware call from the vectors, or run a sequence of branches. This gets added to the hyp vectors. If there is no support for arch-workaround-1 in firmware, the indirect vector will be used. kvm_init_vector_slots() only initialises the two indirect slots if the platform is vulnerable to Spectre-v3a. pKVM's hyp_map_vectors() only initialises __hyp_bp_vect_base if the platform is vulnerable to Spectre-v3a. As there are about to more users of the indirect vectors, ensure their entries in hyp_spectre_vector_selector[] are always initialised, and __hyp_bp_vect_base defaults to the regular VA mapping. The Spectre-v3a check is moved to a helper kvm_system_needs_idmapped_vectors(), and merged with the code that creates the hyp mappings. Bug: 215557547 Reviewed-by: Catalin Marinas Signed-off-by: James Morse (cherry picked from commit 5bdf3437603d4af87f9c7f424b0c8aeed2420745) Signed-off-by: Greg Kroah-Hartman Change-Id: I6ad747d012e2d9965a9450e62bebd5bfe1605316 --- arch/arm64/include/asm/kvm_host.h | 5 +++++ arch/arm64/kvm/arm.c | 5 +---- arch/arm64/kvm/hyp/nvhe/mm.c | 4 +++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 923ffa4cbc5f..4e9ee0676b53 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -705,6 +705,11 @@ static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt) ctxt_sys_reg(cpu_ctxt, MPIDR_EL1) = read_cpuid_mpidr(); } +static inline bool kvm_system_needs_idmapped_vectors(void) +{ + return cpus_have_const_cap(ARM64_SPECTRE_V3A); +} + void kvm_arm_vcpu_ptrauth_trap(struct kvm_vcpu *vcpu); static inline void kvm_arch_hardware_unsetup(void) {} diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index f3b67a40630e..67749fd3604d 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1388,10 +1388,7 @@ static int kvm_init_vector_slots(void) base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs)); kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT); - if (!cpus_have_const_cap(ARM64_SPECTRE_V3A)) - return 0; - - if (!has_vhe()) { + if (kvm_system_needs_idmapped_vectors() && !has_vhe()) { err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs), __BP_HARDEN_HYP_VECS_SZ, &base); if (err) diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index a8efdf0f9003..676a8871e30e 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -132,8 +132,10 @@ int hyp_map_vectors(void) phys_addr_t phys; void *bp_base; - if (!cpus_have_const_cap(ARM64_SPECTRE_V3A)) + if (!kvm_system_needs_idmapped_vectors()) { + __hyp_bp_vect_base = __bp_harden_hyp_vecs; return 0; + } phys = __hyp_pa(__bp_harden_hyp_vecs); bp_base = (void *)__pkvm_create_private_mapping(phys, From 5411474f65b8c531dde558ab0f16bc49e747a292 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 9 Mar 2022 12:16:34 -0700 Subject: [PATCH 73/80] UPSTREAM: arm64: Do not include __READ_ONCE() block in assembly files When building arm64 defconfig + CONFIG_LTO_CLANG_{FULL,THIN}=y after commit 558c303c9734 ("arm64: Mitigate spectre style branch history side channels"), the following error occurs: :4:2: error: invalid fixup for movz/movk instruction mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 ^ Marc figured out that moving "#include " in include/linux/arm-smccc.h into a !__ASSEMBLY__ block resolves it. The full include chain with CONFIG_LTO=y from include/linux/arm-smccc.h: include/linux/init.h include/linux/compiler.h arch/arm64/include/asm/rwonce.h arch/arm64/include/asm/alternative-macros.h arch/arm64/include/asm/assembler.h The asm/alternative-macros.h include in asm/rwonce.h only happens when CONFIG_LTO is set, which ultimately casues asm/assembler.h to be included before the definition of ARM_SMCCC_ARCH_WORKAROUND_3. As a result, the preprocessor does not expand ARM_SMCCC_ARCH_WORKAROUND_3 in __mitigate_spectre_bhb_fw, which results in the error above. Avoid this problem by just avoiding the CONFIG_LTO=y __READ_ONCE() block in asm/rwonce.h with assembly files, as nothing in that block is useful to assembly files, which allows ARM_SMCCC_ARCH_WORKAROUND_3 to be properly expanded with CONFIG_LTO=y builds. Bug: 215557547 Fixes: e35123d83ee3 ("arm64: lto: Strengthen READ_ONCE() to acquire when CONFIG_LTO=y") Cc: # 5.11.x Link: https://lore.kernel.org/r/20220309155716.3988480-1-maz@kernel.org/ Reported-by: Marc Zyngier Acked-by: James Morse Signed-off-by: Nathan Chancellor Link: https://lore.kernel.org/r/20220309191633.2307110-1-nathan@kernel.org Signed-off-by: Catalin Marinas (cherry picked from commit 52c9f93a9c482251cb0d224faa602ba26d462be8) Signed-off-by: Greg Kroah-Hartman Change-Id: I3beec784b16c9b094bc2e86466f2dadddc3ec6a7 --- arch/arm64/include/asm/rwonce.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/rwonce.h b/arch/arm64/include/asm/rwonce.h index 1bce62fa908a..56f7b1d4d54b 100644 --- a/arch/arm64/include/asm/rwonce.h +++ b/arch/arm64/include/asm/rwonce.h @@ -5,7 +5,7 @@ #ifndef __ASM_RWONCE_H #define __ASM_RWONCE_H -#ifdef CONFIG_LTO +#if defined(CONFIG_LTO) && !defined(__ASSEMBLY__) #include #include @@ -66,7 +66,7 @@ }) #endif /* !BUILD_VDSO */ -#endif /* CONFIG_LTO */ +#endif /* CONFIG_LTO && !__ASSEMBLY__ */ #include From 0f76dfc55d9ec6379221b3d699046bc3a434d71d Mon Sep 17 00:00:00 2001 From: James Morse Date: Wed, 10 Nov 2021 14:48:00 +0000 Subject: [PATCH 74/80] UPSTREAM: arm64: Mitigate spectre style branch history side channels commit 558c303c9734af5a813739cd284879227f7297d2 upstream. Speculation attacks against some high-performance processors can make use of branch history to influence future speculation. When taking an exception from user-space, a sequence of branches or a firmware call overwrites or invalidates the branch history. The sequence of branches is added to the vectors, and should appear before the first indirect branch. For systems using KPTI the sequence is added to the kpti trampoline where it has a free register as the exit from the trampoline is via a 'ret'. For systems not using KPTI, the same register tricks are used to free up a register in the vectors. For the firmware call, arch-workaround-3 clobbers 4 registers, so there is no choice but to save them to the EL1 stack. This only happens for entry from EL0, so if we take an exception due to the stack access, it will not become re-entrant. For KVM, the existing branch-predictor-hardening vectors are used. When a spectre version of these vectors is in use, the firmware call is sufficient to mitigate against Spectre-BHB. For the non-spectre versions, the sequence of branches is added to the indirect vector. Bug: 215557547 Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Marc Zyngier Change-Id: I7d1f5a9767d1dbc9e6ef363ca3bf7bffe91c402c Signed-off-by: Greg Kroah-Hartman --- arch/arm64/Kconfig | 9 + arch/arm64/include/asm/assembler.h | 14 +- arch/arm64/include/asm/cpucaps.h | 3 +- arch/arm64/include/asm/cpufeature.h | 16 ++ arch/arm64/include/asm/cputype.h | 8 + arch/arm64/include/asm/spectre.h | 4 +- arch/arm64/include/asm/sysreg.h | 1 + arch/arm64/include/asm/vectors.h | 5 + arch/arm64/kernel/cpu_errata.c | 7 + arch/arm64/kernel/image-vars.h | 3 + arch/arm64/kernel/proton-pack.c | 294 ++++++++++++++++++++++++++++ arch/arm64/kvm/hyp/hyp-entry.S | 8 + 12 files changed, 369 insertions(+), 3 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index c18cd7d63e2b..ecab62be633d 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1239,6 +1239,15 @@ config UNMAP_KERNEL_AT_EL0 If unsure, say Y. +config MITIGATE_SPECTRE_BRANCH_HISTORY + bool "Mitigate Spectre style attacks against branch history" if EXPERT + default y + help + Speculation attacks against some high-performance processors can + make use of branch history to influence future speculation. + When taking an exception from user-space, a sequence of branches + or a firmware call overwrites the branch history. + config RODATA_FULL_DEFAULT_ENABLED bool "Apply r/o permissions of VM areas also to their linear aliases" default y diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 107e0f4651a3..8a455d406810 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -803,7 +803,9 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU .macro __mitigate_spectre_bhb_loop tmp #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY - mov \tmp, #32 +alternative_cb spectre_bhb_patch_loop_iter + mov \tmp, #32 // Patched to correct the immediate +alternative_cb_end .Lspectre_bhb_loop\@: b . + 4 subs \tmp, \tmp, #1 @@ -812,6 +814,16 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ .endm + .macro mitigate_spectre_bhb_loop tmp +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY +alternative_cb spectre_bhb_patch_loop_mitigation_enable + b .L_spectre_bhb_loop_done\@ // Patched to NOP +alternative_cb_end + __mitigate_spectre_bhb_loop \tmp +.L_spectre_bhb_loop_done\@: +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + .endm + /* Save/restores x0-x3 to the stack */ .macro __mitigate_spectre_bhb_fw #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index c0ff1d4bccab..ed39d674de5f 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -70,8 +70,9 @@ #define ARM64_HAS_LDAPR 59 #define ARM64_KVM_PROTECTED_MODE 60 #define ARM64_WORKAROUND_TSB_FLUSH_FAILURE 61 +#define ARM64_SPECTRE_BHB 62 -/* kabi: reserve 62 - 76 for future cpu capabilities */ +/* kabi: reserve 63 - 76 for future cpu capabilities */ #define ARM64_NCAPS 76 #endif /* __ASM_CPUCAPS_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 87fc3b173677..28a2270e0f17 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -613,6 +613,22 @@ static inline bool cpu_supports_mixed_endian_el0(void) return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); } + +static inline bool supports_csv2p3(int scope) +{ + u64 pfr0; + u8 csv2_val; + + if (scope == SCOPE_LOCAL_CPU) + pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1); + else + pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); + + csv2_val = cpuid_feature_extract_unsigned_field(pfr0, + ID_AA64PFR0_CSV2_SHIFT); + return csv2_val == 3; +} + const struct cpumask *system_32bit_el0_cpumask(void); DECLARE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0); diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 999b9149f856..bfbf0c4c7c5e 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -73,10 +73,14 @@ #define ARM_CPU_PART_CORTEX_A76 0xD0B #define ARM_CPU_PART_NEOVERSE_N1 0xD0C #define ARM_CPU_PART_CORTEX_A77 0xD0D +#define ARM_CPU_PART_NEOVERSE_V1 0xD40 +#define ARM_CPU_PART_CORTEX_A78 0xD41 +#define ARM_CPU_PART_CORTEX_X1 0xD44 #define ARM_CPU_PART_CORTEX_A510 0xD46 #define ARM_CPU_PART_CORTEX_A710 0xD47 #define ARM_CPU_PART_CORTEX_X2 0xD48 #define ARM_CPU_PART_NEOVERSE_N2 0xD49 +#define ARM_CPU_PART_CORTEX_A78C 0xD4B #define APM_CPU_PART_POTENZA 0x000 @@ -117,10 +121,14 @@ #define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76) #define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1) #define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77) +#define MIDR_NEOVERSE_V1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V1) +#define MIDR_CORTEX_A78 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78) +#define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1) #define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510) #define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710) #define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2) #define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2) +#define MIDR_CORTEX_A78C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX) #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX) diff --git a/arch/arm64/include/asm/spectre.h b/arch/arm64/include/asm/spectre.h index 84b174b57aae..c04d01dd4457 100644 --- a/arch/arm64/include/asm/spectre.h +++ b/arch/arm64/include/asm/spectre.h @@ -93,6 +93,8 @@ void spectre_v4_enable_task_mitigation(struct task_struct *tsk); enum mitigation_state arm64_get_meltdown_state(void); enum mitigation_state arm64_get_spectre_bhb_state(void); - +bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope); +u8 spectre_bhb_loop_affected(int scope); +void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused); #endif /* __ASSEMBLY__ */ #endif /* __ASM_SPECTRE_H */ diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 253b69b7d027..8006920a00f8 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -867,6 +867,7 @@ #endif /* id_aa64mmfr1 */ +#define ID_AA64MMFR1_ECBHB_SHIFT 60 #define ID_AA64MMFR1_AFP_SHIFT 44 #define ID_AA64MMFR1_ETS_SHIFT 36 #define ID_AA64MMFR1_TWED_SHIFT 32 diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h index 3f76dfd9e074..1f65c37dc653 100644 --- a/arch/arm64/include/asm/vectors.h +++ b/arch/arm64/include/asm/vectors.h @@ -40,6 +40,11 @@ enum arm64_bp_harden_el1_vectors { EL1_VECTOR_KPTI, }; +#ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY +#define EL1_VECTOR_BHB_LOOP -1 +#define EL1_VECTOR_BHB_FW -1 +#endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + /* The vectors to use on return from EL0. e.g. to remap the kernel */ DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector); diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 7c6d59c4f099..e458bf3e9e8d 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -478,6 +478,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .matches = has_spectre_v4, .cpu_enable = spectre_v4_enable_mitigation, }, + { + .desc = "Spectre-BHB", + .capability = ARM64_SPECTRE_BHB, + .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, + .matches = is_spectre_bhb_affected, + .cpu_enable = spectre_bhb_enable_mitigation, + }, #ifdef CONFIG_ARM64_ERRATUM_1418040 { .desc = "ARM erratum 1418040", diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index fdd60cd1d7e8..ddf364cfa008 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -66,6 +66,9 @@ KVM_NVHE_ALIAS(kvm_patch_vector_branch); KVM_NVHE_ALIAS(kvm_update_va_mask); KVM_NVHE_ALIAS(kvm_get_kimage_voffset); KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0); +KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter); +KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable); +KVM_NVHE_ALIAS(spectre_bhb_patch_wa3); /* Global kernel state accessed by nVHE hyp code. */ KVM_NVHE_ALIAS(kvm_vgic_global_state); diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index b4bef7707e19..fbbcd1d2eb19 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -24,9 +24,11 @@ #include #include +#include #include #include #include +#include #include /* @@ -796,9 +798,301 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) } } +/* + * Spectre BHB. + * + * A CPU is either: + * - Mitigated by a branchy loop a CPU specific number of times, and listed + * in our "loop mitigated list". + * - Mitigated in software by the firmware Spectre v2 call. + * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no + * software mitigation in the vectors is needed. + * - Has CSV2.3, so is unaffected. + */ static enum mitigation_state spectre_bhb_state; enum mitigation_state arm64_get_spectre_bhb_state(void) { return spectre_bhb_state; } + +enum bhb_mitigation_bits { + BHB_LOOP, + BHB_FW, + BHB_HW, +}; +static unsigned long system_bhb_mitigations; + +/* + * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any + * SCOPE_SYSTEM call will give the right answer. + */ +u8 spectre_bhb_loop_affected(int scope) +{ + u8 k = 0; + static u8 max_bhb_k; + + if (scope == SCOPE_LOCAL_CPU) { + static const struct midr_range spectre_bhb_k32_list[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A710), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + {}, + }; + static const struct midr_range spectre_bhb_k24_list[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A76), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A77), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + {}, + }; + static const struct midr_range spectre_bhb_k8_list[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), + {}, + }; + + if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list)) + k = 32; + else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list)) + k = 24; + else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list)) + k = 8; + + max_bhb_k = max(max_bhb_k, k); + } else { + k = max_bhb_k; + } + + return k; +} + +static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void) +{ + int ret; + struct arm_smccc_res res; + + arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, + ARM_SMCCC_ARCH_WORKAROUND_3, &res); + + ret = res.a0; + switch (ret) { + case SMCCC_RET_SUCCESS: + return SPECTRE_MITIGATED; + case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: + return SPECTRE_UNAFFECTED; + default: + fallthrough; + case SMCCC_RET_NOT_SUPPORTED: + return SPECTRE_VULNERABLE; + } +} + +static bool is_spectre_bhb_fw_affected(int scope) +{ + static bool system_affected; + enum mitigation_state fw_state; + bool has_smccc = arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_NONE; + static const struct midr_range spectre_bhb_firmware_mitigated_list[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A73), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A75), + {}, + }; + bool cpu_in_list = is_midr_in_range_list(read_cpuid_id(), + spectre_bhb_firmware_mitigated_list); + + if (scope != SCOPE_LOCAL_CPU) + return system_affected; + + fw_state = spectre_bhb_get_cpu_fw_mitigation_state(); + if (cpu_in_list || (has_smccc && fw_state == SPECTRE_MITIGATED)) { + system_affected = true; + return true; + } + + return false; +} + +static bool supports_ecbhb(int scope) +{ + u64 mmfr1; + + if (scope == SCOPE_LOCAL_CPU) + mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1); + else + mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); + + return cpuid_feature_extract_unsigned_field(mmfr1, + ID_AA64MMFR1_ECBHB_SHIFT); +} + +bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, + int scope) +{ + WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible()); + + if (supports_csv2p3(scope)) + return false; + + if (spectre_bhb_loop_affected(scope)) + return true; + + if (is_spectre_bhb_fw_affected(scope)) + return true; + + return false; +} + +static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) +{ + const char *v = arm64_get_bp_hardening_vector(slot); + + if (slot < 0) + return; + + __this_cpu_write(this_cpu_vector, v); + + /* + * When KPTI is in use, the vectors are switched when exiting to + * user-space. + */ + if (arm64_kernel_unmapped_at_el0()) + return; + + write_sysreg(v, vbar_el1); + isb(); +} + +void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) +{ + bp_hardening_cb_t cpu_cb; + enum mitigation_state fw_state, state = SPECTRE_VULNERABLE; + struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data); + + if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU)) + return; + + if (arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) { + /* No point mitigating Spectre-BHB alone. */ + } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) { + pr_info_once("spectre-bhb mitigation disabled by compile time option\n"); + } else if (cpu_mitigations_off()) { + pr_info_once("spectre-bhb mitigation disabled by command line option\n"); + } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { + state = SPECTRE_MITIGATED; + set_bit(BHB_HW, &system_bhb_mitigations); + } else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) { + /* + * Ensure KVM uses the indirect vector which will have the + * branchy-loop added. A57/A72-r0 will already have selected + * the spectre-indirect vector, which is sufficient for BHB + * too. + */ + if (!data->slot) + data->slot = HYP_VECTOR_INDIRECT; + + this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP); + state = SPECTRE_MITIGATED; + set_bit(BHB_LOOP, &system_bhb_mitigations); + } else if (is_spectre_bhb_fw_affected(SCOPE_LOCAL_CPU)) { + fw_state = spectre_bhb_get_cpu_fw_mitigation_state(); + if (fw_state == SPECTRE_MITIGATED) { + /* + * Ensure KVM uses one of the spectre bp_hardening + * vectors. The indirect vector doesn't include the EL3 + * call, so needs upgrading to + * HYP_VECTOR_SPECTRE_INDIRECT. + */ + if (!data->slot || data->slot == HYP_VECTOR_INDIRECT) + data->slot += 1; + + this_cpu_set_vectors(EL1_VECTOR_BHB_FW); + + /* + * The WA3 call in the vectors supersedes the WA1 call + * made during context-switch. Uninstall any firmware + * bp_hardening callback. + */ + cpu_cb = spectre_v2_get_sw_mitigation_cb(); + if (__this_cpu_read(bp_hardening_data.fn) != cpu_cb) + __this_cpu_write(bp_hardening_data.fn, NULL); + + state = SPECTRE_MITIGATED; + set_bit(BHB_FW, &system_bhb_mitigations); + } + } + + update_mitigation_state(&spectre_bhb_state, state); +} + +/* Patched to NOP when enabled */ +void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt, + __le32 *origptr, + __le32 *updptr, int nr_inst) +{ + BUG_ON(nr_inst != 1); + + if (test_bit(BHB_LOOP, &system_bhb_mitigations)) + *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); +} + +/* Patched to NOP when enabled */ +void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt, + __le32 *origptr, + __le32 *updptr, int nr_inst) +{ + BUG_ON(nr_inst != 1); + + if (test_bit(BHB_FW, &system_bhb_mitigations)) + *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); +} + +/* Patched to correct the immediate */ +void noinstr spectre_bhb_patch_loop_iter(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst) +{ + u8 rd; + u32 insn; + u16 loop_count = spectre_bhb_loop_affected(SCOPE_SYSTEM); + + BUG_ON(nr_inst != 1); /* MOV -> MOV */ + + if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) + return; + + insn = le32_to_cpu(*origptr); + rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn); + insn = aarch64_insn_gen_movewide(rd, loop_count, 0, + AARCH64_INSN_VARIANT_64BIT, + AARCH64_INSN_MOVEWIDE_ZERO); + *updptr++ = cpu_to_le32(insn); +} + +/* Patched to mov WA3 when supported */ +void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst) +{ + u8 rd; + u32 insn; + + BUG_ON(nr_inst != 1); /* MOV -> MOV */ + + if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) || + !test_bit(BHB_FW, &system_bhb_mitigations)) + return; + + insn = le32_to_cpu(*origptr); + rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn); + + insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_ORR, + AARCH64_INSN_VARIANT_32BIT, + AARCH64_INSN_REG_ZR, rd, + ARM_SMCCC_ARCH_WORKAROUND_3); + if (WARN_ON_ONCE(insn == AARCH64_BREAK_FAULT)) + return; + + *updptr++ = cpu_to_le32(insn); +} diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index 6e53e517176c..7cc9d86b690a 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -62,6 +62,10 @@ el1_sync: // Guest trapped into EL2 /* ARM_SMCCC_ARCH_WORKAROUND_2 handling */ eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \ ARM_SMCCC_ARCH_WORKAROUND_2) + cbz w1, wa_epilogue + + eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_2 ^ \ + ARM_SMCCC_ARCH_WORKAROUND_3) cbnz w1, el1_trap wa_epilogue: @@ -192,7 +196,10 @@ SYM_CODE_END(__kvm_hyp_vector) sub sp, sp, #(8 * 4) stp x2, x3, [sp, #(8 * 0)] stp x0, x1, [sp, #(8 * 2)] + alternative_cb spectre_bhb_patch_wa3 + /* Patched to mov WA3 when supported */ mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1 + alternative_cb_end smc #0 ldp x2, x3, [sp, #(8 * 0)] add sp, sp, #(8 * 2) @@ -205,6 +212,7 @@ SYM_CODE_END(__kvm_hyp_vector) spectrev2_smccc_wa1_smc .else stp x0, x1, [sp, #-16]! + mitigate_spectre_bhb_loop x0 .endif .if \indirect != 0 alternative_cb kvm_patch_vector_branch From 98b16e808fa3f3d5d8b20b85e80dc3a6f7d5d4f3 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 10 Dec 2021 11:16:18 +0000 Subject: [PATCH 75/80] UPSTREAM: KVM: arm64: Allow SMCCC_ARCH_WORKAROUND_3 to be discovered and migrated commit a5905d6af492ee6a4a2205f0d550b3f931b03d03 upstream. KVM allows the guest to discover whether the ARCH_WORKAROUND SMCCC are implemented, and to preserve that state during migration through its firmware register interface. Add the necessary boiler plate for SMCCC_ARCH_WORKAROUND_3. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: I9e0964b54727930bbda4f089c91075447b046c91 --- arch/arm64/include/uapi/asm/kvm.h | 5 +++++ arch/arm64/kvm/hypercalls.c | 12 ++++++++++++ arch/arm64/kvm/psci.c | 18 +++++++++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 24223adae150..065a31ccfe8f 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -270,6 +270,11 @@ struct kvm_vcpu_events { #define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED 3 #define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED (1U << 4) +#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3 KVM_REG_ARM_FW_REG(3) +#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL 0 +#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL 1 +#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED 2 + /* SVE registers */ #define KVM_REG_ARM64_SVE (0x15 << KVM_REG_ARM_COPROC_SHIFT) diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index ead21b98b620..c11c90de8050 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -58,6 +58,18 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) break; } break; + case ARM_SMCCC_ARCH_WORKAROUND_3: + switch (arm64_get_spectre_bhb_state()) { + case SPECTRE_VULNERABLE: + break; + case SPECTRE_MITIGATED: + val = SMCCC_RET_SUCCESS; + break; + case SPECTRE_UNAFFECTED: + val = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; + break; + } + break; case ARM_SMCCC_HV_PV_TIME_FEATURES: val = SMCCC_RET_SUCCESS; break; diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index db4056ecccfd..20ba5136ac3d 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -397,7 +397,7 @@ int kvm_psci_call(struct kvm_vcpu *vcpu) int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu) { - return 3; /* PSCI version and two workaround registers */ + return 4; /* PSCI version and three workaround registers */ } int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) @@ -411,6 +411,9 @@ int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2, uindices++)) return -EFAULT; + if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3, uindices++)) + return -EFAULT; + return 0; } @@ -450,6 +453,17 @@ static int get_kernel_wa_level(u64 regid) case SPECTRE_VULNERABLE: return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL; } + break; + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: + switch (arm64_get_spectre_bhb_state()) { + case SPECTRE_VULNERABLE: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL; + case SPECTRE_MITIGATED: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL; + case SPECTRE_UNAFFECTED: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED; + } + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL; } return -EINVAL; @@ -466,6 +480,7 @@ int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) break; case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2: + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: val = get_kernel_wa_level(reg->id) & KVM_REG_FEATURE_LEVEL_MASK; break; default: @@ -511,6 +526,7 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) } case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: if (val & ~KVM_REG_FEATURE_LEVEL_MASK) return -EINVAL; From d236f7b4cb865291ee9eedb58879c9a88093e597 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 10 Dec 2021 14:32:56 +0000 Subject: [PATCH 76/80] UPSTREAM: arm64: Use the clearbhb instruction in mitigations Future CPUs may implement a clearbhb instruction that is sufficient to mitigate SpectreBHB. CPUs that implement this instruction, but not CSV2.3 must be affected by Spectre-BHB. Add support to use this instruction as the BHB mitigation on CPUs that support it. The instruction is in the hint space, so it will be treated by a NOP as older CPUs. Bug: 215557547 Reviewed-by: Russell King (Oracle) Reviewed-by: Catalin Marinas Signed-off-by: James Morse (cherry picked from commit 228a26b912287934789023b4132ba76065d9491c) Signed-off-by: Greg Kroah-Hartman Change-Id: Ie233d92acfaed7b44ac255472ddb06367660384d --- arch/arm64/include/asm/assembler.h | 17 +++++++++++++++++ arch/arm64/include/asm/cpufeature.h | 13 +++++++++++++ arch/arm64/include/asm/insn.h | 1 + arch/arm64/include/asm/sysreg.h | 1 + arch/arm64/include/asm/vectors.h | 7 +++++++ arch/arm64/kernel/cpufeature.c | 1 + arch/arm64/kernel/entry.S | 8 ++++++++ arch/arm64/kernel/image-vars.h | 1 + arch/arm64/kernel/proton-pack.c | 29 +++++++++++++++++++++++++++++ arch/arm64/kvm/hyp/hyp-entry.S | 1 + 10 files changed, 79 insertions(+) diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 8a455d406810..22bbe743dbae 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -107,6 +107,13 @@ hint #20 .endm +/* + * Clear Branch History instruction + */ + .macro clearbhb + hint #22 + .endm + /* * Speculation barrier */ @@ -835,6 +842,16 @@ alternative_cb smccc_patch_fw_mitigation_conduit alternative_cb_end ldp x2, x3, [sp], #16 ldp x0, x1, [sp], #16 +#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ + .endm + + .macro mitigate_spectre_bhb_clear_insn +#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY +alternative_cb spectre_bhb_patch_clearbhb + /* Patched to NOP when not supported */ + clearbhb + isb +alternative_cb_end #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ .endm #endif /* __ASM_ASSEMBLER_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 28a2270e0f17..717551d45fe8 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -629,6 +629,19 @@ static inline bool supports_csv2p3(int scope) return csv2_val == 3; } +static inline bool supports_clearbhb(int scope) +{ + u64 isar2; + + if (scope == SCOPE_LOCAL_CPU) + isar2 = read_sysreg_s(SYS_ID_AA64ISAR2_EL1); + else + isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1); + + return cpuid_feature_extract_unsigned_field(isar2, + ID_AA64ISAR2_CLEARBHB_SHIFT); +} + const struct cpumask *system_32bit_el0_cpumask(void); DECLARE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0); diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 4ebb9c054ccc..c0973345e6e1 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -64,6 +64,7 @@ enum aarch64_insn_hint_cr_op { AARCH64_INSN_HINT_PSB = 0x11 << 5, AARCH64_INSN_HINT_TSB = 0x12 << 5, AARCH64_INSN_HINT_CSDB = 0x14 << 5, + AARCH64_INSN_HINT_CLEARBHB = 0x16 << 5, AARCH64_INSN_HINT_BTI = 0x20 << 5, AARCH64_INSN_HINT_BTIC = 0x22 << 5, diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 8006920a00f8..df5a60ce1aa9 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -755,6 +755,7 @@ #define ID_AA64ISAR1_GPI_IMP_DEF 0x1 /* id_aa64isar2 */ +#define ID_AA64ISAR2_CLEARBHB_SHIFT 28 #define ID_AA64ISAR2_RPRES_SHIFT 4 #define ID_AA64ISAR2_WFXT_SHIFT 0 diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h index 1f65c37dc653..f64613a96d53 100644 --- a/arch/arm64/include/asm/vectors.h +++ b/arch/arm64/include/asm/vectors.h @@ -32,6 +32,12 @@ enum arm64_bp_harden_el1_vectors { * canonical vectors. */ EL1_VECTOR_BHB_FW, + + /* + * Use the ClearBHB instruction, before branching to the canonical + * vectors. + */ + EL1_VECTOR_BHB_CLEAR_INSN, #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ /* @@ -43,6 +49,7 @@ enum arm64_bp_harden_el1_vectors { #ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY #define EL1_VECTOR_BHB_LOOP -1 #define EL1_VECTOR_BHB_FW -1 +#define EL1_VECTOR_BHB_CLEAR_INSN -1 #endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ /* The vectors to use on return from EL0. e.g. to remap the kernel */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 21f0d84e74ac..40ac42cc3216 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -232,6 +232,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { }; static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_RPRES_SHIFT, 4, 0), ARM64_FTR_END, }; diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 08a82f4c8150..9f19e6bb48df 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -924,6 +924,7 @@ alternative_else_nop_endif #define BHB_MITIGATION_NONE 0 #define BHB_MITIGATION_LOOP 1 #define BHB_MITIGATION_FW 2 +#define BHB_MITIGATION_INSN 3 .macro tramp_ventry, vector_start, regsize, kpti, bhb .align 7 @@ -940,6 +941,11 @@ alternative_else_nop_endif __mitigate_spectre_bhb_loop x30 .endif // \bhb == BHB_MITIGATION_LOOP + .if \bhb == BHB_MITIGATION_INSN + clearbhb + isb + .endif // \bhb == BHB_MITIGATION_INSN + .if \kpti == 1 /* * Defend against branch aliasing attacks by pushing a dummy @@ -1016,6 +1022,7 @@ SYM_CODE_START_NOALIGN(tramp_vectors) #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW + generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_INSN #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE SYM_CODE_END(tramp_vectors) @@ -1078,6 +1085,7 @@ SYM_CODE_START(__bp_harden_el1_vectors) #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY generate_el1_vector bhb=BHB_MITIGATION_LOOP generate_el1_vector bhb=BHB_MITIGATION_FW + generate_el1_vector bhb=BHB_MITIGATION_INSN #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ SYM_CODE_END(__bp_harden_el1_vectors) .popsection diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index ddf364cfa008..d6c6480e95d1 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -69,6 +69,7 @@ KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0); KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter); KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable); KVM_NVHE_ALIAS(spectre_bhb_patch_wa3); +KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb); /* Global kernel state accessed by nVHE hyp code. */ KVM_NVHE_ALIAS(kvm_vgic_global_state); diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index fbbcd1d2eb19..d3fbff00993d 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -805,6 +805,7 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) * - Mitigated by a branchy loop a CPU specific number of times, and listed * in our "loop mitigated list". * - Mitigated in software by the firmware Spectre v2 call. + * - Has the ClearBHB instruction to perform the mitigation. * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no * software mitigation in the vectors is needed. * - Has CSV2.3, so is unaffected. @@ -820,6 +821,7 @@ enum bhb_mitigation_bits { BHB_LOOP, BHB_FW, BHB_HW, + BHB_INSN, }; static unsigned long system_bhb_mitigations; @@ -937,6 +939,9 @@ bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, if (supports_csv2p3(scope)) return false; + if (supports_clearbhb(scope)) + return true; + if (spectre_bhb_loop_affected(scope)) return true; @@ -984,6 +989,17 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { state = SPECTRE_MITIGATED; set_bit(BHB_HW, &system_bhb_mitigations); + } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) { + /* + * Ensure KVM uses the indirect vector which will have ClearBHB + * added. + */ + if (!data->slot) + data->slot = HYP_VECTOR_INDIRECT; + + this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN); + state = SPECTRE_MITIGATED; + set_bit(BHB_INSN, &system_bhb_mitigations); } else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) { /* * Ensure KVM uses the indirect vector which will have the @@ -1096,3 +1112,16 @@ void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt, *updptr++ = cpu_to_le32(insn); } + +/* Patched to NOP when not supported */ +void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst) +{ + BUG_ON(nr_inst != 2); + + if (test_bit(BHB_INSN, &system_bhb_mitigations)) + return; + + *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); + *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); +} diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index 7cc9d86b690a..3dc6e10873f3 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -213,6 +213,7 @@ SYM_CODE_END(__kvm_hyp_vector) .else stp x0, x1, [sp, #-16]! mitigate_spectre_bhb_loop x0 + mitigate_spectre_bhb_clear_insn .endif .if \indirect != 0 alternative_cb kvm_patch_vector_branch From de0334216bbbf9f1ab1a2703ebc58445fdace8a6 Mon Sep 17 00:00:00 2001 From: James Morse Date: Thu, 3 Mar 2022 16:53:56 +0000 Subject: [PATCH 77/80] UPSTREAM: arm64: proton-pack: Include unprivileged eBPF status in Spectre v2 mitigation reporting commit 58c9a5060cb7cd529d49c93954cdafe81c1d642a upstream. The mitigations for Spectre-BHB are only applied when an exception is taken from user-space. The mitigation status is reported via the spectre_v2 sysfs vulnerabilities file. When unprivileged eBPF is enabled the mitigation in the exception vectors can be avoided by an eBPF program. When unprivileged eBPF is enabled, print a warning and report vulnerable via the sysfs vulnerabilities file. Bug: 215557547 Acked-by: Catalin Marinas Signed-off-by: James Morse Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Change-Id: Ib4a7b967d820684dbb18eafacdbaa22794469040 --- arch/arm64/kernel/proton-pack.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index d3fbff00993d..6d45c63c6454 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -111,6 +112,15 @@ static const char *get_bhb_affected_string(enum mitigation_state bhb_state) } } +static bool _unprivileged_ebpf_enabled(void) +{ +#ifdef CONFIG_BPF_SYSCALL + return !sysctl_unprivileged_bpf_disabled; +#else + return false; +#endif +} + ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) { @@ -130,6 +140,9 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, v2_str = "CSV2"; fallthrough; case SPECTRE_MITIGATED: + if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled()) + return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n"); + return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str); case SPECTRE_VULNERABLE: fallthrough; @@ -1125,3 +1138,16 @@ void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt, *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); *updptr++ = cpu_to_le32(aarch64_insn_gen_nop()); } + +#ifdef CONFIG_BPF_SYSCALL +#define EBPF_WARN "Unprivileged eBPF is enabled, data leaks possible via Spectre v2 BHB attacks!\n" +void unpriv_ebpf_notify(int new_state) +{ + if (spectre_v2_state == SPECTRE_VULNERABLE || + spectre_bhb_state != SPECTRE_MITIGATED) + return; + + if (!new_state) + pr_err("WARNING: %s", EBPF_WARN); +} +#endif From 39aca1597913c5ed5548d00d2238470f918dddec Mon Sep 17 00:00:00 2001 From: "andrew.yang" Date: Tue, 15 Mar 2022 16:58:34 +1100 Subject: [PATCH 78/80] FROMGIT: mm/migrate: fix race between lock page and clear PG_Isolated When memory is tight, system may start to compact memory for large continuous memory demands. If one process tries to lock a memory page that is being locked and isolated for compaction, it may wait a long time or even forever. This is because compaction will perform non-atomic PG_Isolated clear while holding page lock, this may overwrite PG_waiters set by the process that can't obtain the page lock and add itself to the waiting queue to wait for the lock to be unlocked. CPU1 CPU2 lock_page(page); (successful) lock_page(); (failed) __ClearPageIsolated(page); SetPageWaiters(page) (may be overwritten) unlock_page(page); The solution is to not perform non-atomic operation on page flags while holding page lock. Link: https://lkml.kernel.org/r/20220315030515.20263-1-andrew.yang@mediatek.com Signed-off-by: andrew.yang Cc: Matthias Brugger Cc: Matthew Wilcox Cc: "Vlastimil Babka" Cc: David Howells Cc: "William Kucharski" Cc: David Hildenbrand Cc: Yang Shi Cc: Marc Zyngier Cc: Nicholas Tang Cc: Kuan-Ying Lee Signed-off-by: Andrew Morton Signed-off-by: Stephen Rothwell (cherry picked from commit 48911e41ddee4fe113bf1e4303dda1a413b169c9 https://github.com/hnaz/linux-mm.git) Bug: 225086204 Change-Id: I58547aeae29bcbc2071776330f9c31b9eb9f5aa8 --- include/linux/page-flags.h | 2 +- mm/migrate.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 9f67e4653af5..a2738944340c 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -795,7 +795,7 @@ PAGE_TYPE_OPS(Guard, guard) extern bool is_free_buddy_page(struct page *page); -__PAGEFLAG(Isolated, isolated, PF_ANY); +PAGEFLAG(Isolated, isolated, PF_ANY); /* * If network-based swap is enabled, sl*b must keep track of whether pages diff --git a/mm/migrate.c b/mm/migrate.c index 7137465caf40..26f8595de076 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -105,7 +105,7 @@ int isolate_movable_page(struct page *page, isolate_mode_t mode) /* Driver shouldn't use PG_isolated bit of page->flags */ WARN_ON_ONCE(PageIsolated(page)); - __SetPageIsolated(page); + SetPageIsolated(page); unlock_page(page); return 0; @@ -129,7 +129,7 @@ void putback_movable_page(struct page *page) mapping = page_mapping(page); mapping->a_ops->putback_page(page); - __ClearPageIsolated(page); + ClearPageIsolated(page); } /* @@ -162,7 +162,7 @@ void putback_movable_pages(struct list_head *l) if (PageMovable(page)) putback_movable_page(page); else - __ClearPageIsolated(page); + ClearPageIsolated(page); unlock_page(page); put_page(page); } else { @@ -952,7 +952,7 @@ static int move_to_new_page(struct page *newpage, struct page *page, VM_BUG_ON_PAGE(!PageIsolated(page), page); if (!PageMovable(page)) { rc = MIGRATEPAGE_SUCCESS; - __ClearPageIsolated(page); + ClearPageIsolated(page); goto out; } @@ -974,7 +974,7 @@ static int move_to_new_page(struct page *newpage, struct page *page, * We clear PG_movable under page_lock so any compactor * cannot try to migrate this page. */ - __ClearPageIsolated(page); + ClearPageIsolated(page); } /* @@ -1160,7 +1160,7 @@ static int unmap_and_move(new_page_t get_new_page, if (unlikely(__PageMovable(page))) { lock_page(page); if (!PageMovable(page)) - __ClearPageIsolated(page); + ClearPageIsolated(page); unlock_page(page); } goto out; @@ -1215,7 +1215,7 @@ out: if (PageMovable(page)) putback_movable_page(page); else - __ClearPageIsolated(page); + ClearPageIsolated(page); unlock_page(page); put_page(page); } From 821f3e53d95bad11003e8f1c1c31d12c730214c4 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 1 Mar 2022 11:04:24 +0300 Subject: [PATCH 79/80] UPSTREAM: usb: gadget: rndis: prevent integer overflow in rndis_set_response() If "BufOffset" is very large the "BufOffset + 8" operation can have an integer overflow. Bug: 213172319 Cc: stable@kernel.org Fixes: 38ea1eac7d88 ("usb: gadget: rndis: check size of RNDIS_MSG_SET command") Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/20220301080424.GA17208@kili Signed-off-by: Greg Kroah-Hartman (cherry picked from commit 65f3324f4b6fed78b8761c3b74615ecf0ffa81fa) Signed-off-by: Greg Kroah-Hartman Change-Id: I16c7c6c5a0a3a0102d1c6c9ddfca2a1ad1106880 --- drivers/usb/gadget/function/rndis.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/usb/gadget/function/rndis.c b/drivers/usb/gadget/function/rndis.c index d9ed651f06ac..ea77a18e927d 100644 --- a/drivers/usb/gadget/function/rndis.c +++ b/drivers/usb/gadget/function/rndis.c @@ -640,6 +640,7 @@ static int rndis_set_response(struct rndis_params *params, BufLength = le32_to_cpu(buf->InformationBufferLength); BufOffset = le32_to_cpu(buf->InformationBufferOffset); if ((BufLength > RNDIS_MAX_TOTAL_SIZE) || + (BufOffset > RNDIS_MAX_TOTAL_SIZE) || (BufOffset + 8 >= RNDIS_MAX_TOTAL_SIZE)) return -EINVAL; From 3eec4418225fbd8bd36336bd75bd5e93e6edc82d Mon Sep 17 00:00:00 2001 From: Alan Stern Date: Sat, 5 Mar 2022 21:47:22 -0500 Subject: [PATCH 80/80] UPSTREAM: usb: gadget: Fix use-after-free bug by not setting udc->dev.driver The syzbot fuzzer found a use-after-free bug: BUG: KASAN: use-after-free in dev_uevent+0x712/0x780 drivers/base/core.c:2320 Read of size 8 at addr ffff88802b934098 by task udevd/3689 CPU: 2 PID: 3689 Comm: udevd Not tainted 5.17.0-rc4-syzkaller-00229-g4f12b742eb2b #0 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.14.0-2 04/01/2014 Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0xcd/0x134 lib/dump_stack.c:106 print_address_description.constprop.0.cold+0x8d/0x303 mm/kasan/report.c:255 __kasan_report mm/kasan/report.c:442 [inline] kasan_report.cold+0x83/0xdf mm/kasan/report.c:459 dev_uevent+0x712/0x780 drivers/base/core.c:2320 uevent_show+0x1b8/0x380 drivers/base/core.c:2391 dev_attr_show+0x4b/0x90 drivers/base/core.c:2094 Although the bug manifested in the driver core, the real cause was a race with the gadget core. dev_uevent() does: if (dev->driver) add_uevent_var(env, "DRIVER=%s", dev->driver->name); and between the test and the dereference of dev->driver, the gadget core sets dev->driver to NULL. The race wouldn't occur if the gadget core registered its devices on a real bus, using the standard synchronization techniques of the driver core. However, it's not necessary to make such a large change in order to fix this bug; all we need to do is make sure that udc->dev.driver is always NULL. In fact, there is no reason for udc->dev.driver ever to be set to anything, let alone to the value it currently gets: the address of the gadget's driver. After all, a gadget driver only knows how to manage a gadget, not how to manage a UDC. This patch simply removes the statements in the gadget core that touch udc->dev.driver. Fixes: 2ccea03a8f7e ("usb: gadget: introduce UDC Class") CC: Reported-and-tested-by: syzbot+348b571beb5eeb70a582@syzkaller.appspotmail.com Signed-off-by: Alan Stern Link: https://lore.kernel.org/r/YiQgukfFFbBnwJ/9@rowland.harvard.edu Signed-off-by: Greg Kroah-Hartman (cherry picked from commit 16b1941eac2bd499f065a6739a40ce0011a3d740) Signed-off-by: Greg Kroah-Hartman Change-Id: I857485f80975151edada479529fb418b4d85eaaa --- drivers/usb/gadget/udc/core.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index 24d0ace3ed3c..e1e243e52a2d 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -1437,7 +1437,6 @@ static void usb_gadget_remove_driver(struct usb_udc *udc) usb_gadget_udc_stop(udc); udc->driver = NULL; - udc->dev.driver = NULL; udc->gadget->dev.driver = NULL; } @@ -1499,7 +1498,6 @@ static int udc_bind_to_driver(struct usb_udc *udc, struct usb_gadget_driver *dri driver->function); udc->driver = driver; - udc->dev.driver = &driver->driver; udc->gadget->dev.driver = &driver->driver; usb_gadget_udc_set_speed(udc, driver->max_speed); @@ -1522,7 +1520,6 @@ err1: dev_err(&udc->dev, "failed to start %s: %d\n", udc->driver->function, ret); udc->driver = NULL; - udc->dev.driver = NULL; udc->gadget->dev.driver = NULL; return ret; }