Merge keystone/android12-5.10-keystone-qcom-release.66+ (2037ed8) into msm-5.10

* refs/heads/tmp-2037ed8: ANDROID: Incremental fs: Fix dentry get/put imbalance on vfs_mkdir() failure ANDROID: GKI: Update symbols to symbol list ANDROID: Fix mmu_notifier imbalance ANDROID: ABI: Update symbol list for IMX ANDROID: Update symbol list for mtk ANDROID: GKI: update virtual device symbol list ANDROID: ABI: add new symbols required by fips140.ko ANDROID: fips140: zeroize temporary values from integrity check ANDROID: fips140: remove in-place updating of live algorithms ANDROID: fips140: block crypto operations until tests complete ANDROID: fips140: preserve RELA sections without relying on the module loader ANDROID: module: apply special LTO treatment to .text even if CFI is disabled ANDROID: fips140: use FIPS140_CFLAGS when compiling fips140-selftests.c ANDROID: fips140: take into account AES-GCM not being approvable ANDROID: fips140: add jitterentropy to fips140 module ANDROID: fips140: add AES-CMAC ANDROID: fips140: add AES-CBC-CTS ANDROID: fips140: remove non-prediction-resistant DRBG test ANDROID: fips140: use full 16-byte IV ANDROID: fips140: test all implementations ANDROID: Update symbol list for mtk ANDROID: ASoC: soc-pcm: Get all BEs along DAPM path Change-Id: Iad2d638ed56691001358f00c97464fbee623f1f7 Signed-off-by: Sivasri Kumar, Vanka <quic_svanka@quicinc.com>
2021-11-09 16:40:17 +05:30
parent 6f05a2bc28 2037ed82f5
commit 7e089a88e3
24 changed files with 6499 additions and 5984 deletions
--- a/android/abi_gki_aarch64.xml
+++ b/android/abi_gki_aarch64.xml
--- a/android/abi_gki_aarch64_fips140
+++ b/android/abi_gki_aarch64_fips140
@@ -6,12 +6,14 @@
 # required by fips140.ko
  add_random_ready_callback
  aead_register_instance
-  arm64_const_caps_ready
+  ahash_register_instance
+  arch_timer_read_counter
  bcmp
  cancel_work_sync
  __cfi_slowpath
+  complete_all
+  completion_done
  cpu_have_feature
-  cpu_hwcap_keys
  crypto_aead_decrypt
  crypto_aead_encrypt
  crypto_aead_setauthsize
@@ -19,7 +21,6 @@
  crypto_ahash_finup
  crypto_ahash_setkey
  crypto_alg_list
-  crypto_alg_mod_lookup
  crypto_alg_sem
  crypto_alloc_aead
  crypto_alloc_base
@@ -28,7 +29,6 @@
  crypto_alloc_skcipher
  crypto_attr_alg_name
  crypto_check_attr_type
-  crypto_cipher_decrypt_one
  crypto_cipher_encrypt_one
  crypto_cipher_setkey
  crypto_destroy_tfm
@@ -38,17 +38,23 @@
  crypto_grab_ahash
  crypto_grab_shash
  crypto_grab_skcipher
+  crypto_grab_spawn
  crypto_inst_setname
  crypto_put_default_null_skcipher
  crypto_register_aead
+  crypto_register_aeads
+  crypto_register_ahash
+  crypto_register_ahashes
  crypto_register_alg
+  crypto_register_algs
+  crypto_register_rng
  crypto_register_rngs
  crypto_register_shash
  crypto_register_shashes
+  crypto_register_skcipher
  crypto_register_skciphers
  crypto_register_template
  crypto_register_templates
-  crypto_remove_final
  crypto_remove_spawns
  crypto_req_done
  crypto_rng_reset
@@ -63,8 +69,10 @@
  crypto_skcipher_encrypt
  crypto_skcipher_setkey
  crypto_spawn_tfm2
+  crypto_spawn_tfm
  crypto_unregister_aead
  crypto_unregister_alg
+  crypto_unregister_rng
  crypto_unregister_rngs
  crypto_unregister_shash
  crypto_unregister_shashes
@@ -87,6 +95,7 @@
  kmalloc_order_trace
  kmem_cache_alloc_trace
  kmemdup
+  ktime_get
  __list_add_valid
  __list_del_entry_valid
  memcpy
@@ -99,6 +108,9 @@
  preempt_schedule_notrace
  printk
  queue_work_on
+  ___ratelimit
+  _raw_spin_lock
+  _raw_spin_unlock
  refcount_warn_saturate
  scatterwalk_ffwd
  scatterwalk_map_and_copy
@@ -121,7 +133,6 @@
  strlcpy
  strlen
  strncmp
-  synchronize_rcu_tasks
  system_wq
  __traceiter_android_vh_aes_decrypt
  __traceiter_android_vh_aes_encrypt
@@ -134,23 +145,29 @@
  tracepoint_probe_register
  up_write
  wait_for_completion
+  xa_load
+  xa_store

-# needed by fips140.ko but not identified by the tooling
-# TODO(b/189327973): [GKI: ABI] Build of fips140.ko module fails to identify some symbols
-  __crypto_memneq
-  __crypto_xor
+# preserved by --additions-only
  aes_decrypt
  aes_encrypt
  aes_expandkey
+  arm64_const_caps_ready
  ce_aes_expandkey
+  cpu_hwcap_keys
  crypto_aes_inv_sbox
  crypto_aes_sbox
  crypto_aes_set_key
+  crypto_alg_mod_lookup
+  crypto_cipher_decrypt_one
  crypto_ft_tab
  crypto_inc
  crypto_it_tab
+  __crypto_memneq
+  crypto_remove_final
  crypto_sha1_finup
  crypto_sha1_update
+  __crypto_xor
  gf128mul_lle
  sha1_transform
  sha224_final
@@ -158,3 +175,4 @@
  sha256_block_data_order
  sha256_final
  sha256_update
+  synchronize_rcu_tasks
--- a/android/abi_gki_aarch64_imx
+++ b/android/abi_gki_aarch64_imx
@@ -133,6 +133,7 @@
  devm_extcon_dev_allocate
  devm_extcon_dev_register
  devm_free_irq
+  devm_get_clk_from_child
  devm_gpiochip_add_data_with_key
  devm_gpiod_get
  devm_gpiod_get_index
@@ -148,7 +149,6 @@
  devm_phy_create
  devm_phy_get
  devm_pinctrl_get
-  devm_pinctrl_put
  devm_platform_ioremap_resource
  __devm_regmap_init
  __devm_regmap_init_i2c
@@ -307,6 +307,7 @@
  drm_mode_duplicate
  drm_mode_equal
  drm_mode_find_dmt
+  drm_mode_is_420_also
  drm_mode_is_420_only
  drm_mode_probed_add
  drm_mode_set_name
@@ -337,7 +338,6 @@
  eth_validate_addr
  event_triggers_call
  extcon_set_state_sync
-  filp_close
  find_next_bit
  finish_wait
  flush_work
@@ -433,7 +433,6 @@
  kobject_put
  kobject_uevent_env
  kthread_create_on_node
-  kthread_should_stop
  ktime_get
  ktime_get_mono_fast_ns
  ktime_get_real_ts64
@@ -500,6 +499,7 @@
  netif_device_attach
  netif_device_detach
  netif_napi_add
+  netif_receive_skb
  netif_rx
  netif_rx_ni
  netif_tx_stop_all_queues
@@ -652,6 +652,8 @@
  regcache_mark_dirty
  regcache_sync
  __register_chrdev
+  register_inet6addr_notifier
+  register_inetaddr_notifier
  register_netdev
  register_netdevice
  register_netdevice_notifier
@@ -818,10 +820,11 @@
  trace_raw_output_prep
  trace_seq_printf
  try_module_get
-  tty_flip_buffer_push
  __udelay
  __unregister_chrdev
  unregister_chrdev_region
+  unregister_inet6addr_notifier
+  unregister_inetaddr_notifier
  unregister_netdev
  unregister_netdevice_notifier
  unregister_netdevice_queue
@@ -877,6 +880,7 @@
  v4l2_m2m_job_finish
  v4l2_m2m_next_buf
  v4l2_m2m_release
+  v4l2_src_change_event_subscribe
  v4l2_subdev_call_wrappers
  v4l2_subdev_init
  v4l2_subdev_link_validate
@@ -1140,7 +1144,6 @@
  drm_atomic_get_new_connector_for_encoder
  drm_connector_attach_max_bpc_property
  drm_connector_init_with_ddc
-  drm_mode_is_420_also
  drm_scdc_read
  drm_scdc_set_high_tmds_clock_ratio
  drm_scdc_set_scrambling
@@ -1258,8 +1261,6 @@
  __put_page
  reset_control_reset
  schedule_hrtimeout
-  set_user_nice
-  si_meminfo
  sync_file_create
  sync_file_get_fence
  __task_pid_nr_ns
@@ -1268,6 +1269,7 @@
  up_write
  vm_mmap
  vm_munmap
+  vm_zone_stat

 # required by gmsl-max9286.ko
  i2c_smbus_read_byte_data
@@ -1402,6 +1404,7 @@
  rational_best_approximation
  _raw_spin_trylock
  sysrq_mask
+  tty_flip_buffer_push
  __tty_insert_flip_char
  tty_insert_flip_string_fixed_flag
  tty_termios_encode_baud_rate
@@ -1533,13 +1536,10 @@
  kvfree
  kvmalloc_node
  netdev_set_default_ethtool_ops
-  netif_receive_skb
  netif_receive_skb_list
  prandom_bytes
  ___pskb_trim
  rcu_barrier
-  register_inet6addr_notifier
-  register_inetaddr_notifier
  rhashtable_free_and_destroy
  rhashtable_insert_slow
  rhltable_init
@@ -1560,14 +1560,10 @@
  __skb_gso_segment
  skb_queue_head
  system_freezable_wq
-  unregister_inet6addr_notifier
-  unregister_inetaddr_notifier
  unregister_netdevice_many
  __usecs_to_jiffies

 # required by moal.ko
-  bitmap_parse
-  cpumask_next_and
  default_wake_function
  hex_dump_to_buffer
  in4_pton
@@ -1579,6 +1575,7 @@
  iw_handler_get_thrspy
  iw_handler_set_spy
  iw_handler_set_thrspy
+  kthread_should_stop
  ktime_get_raw_ts64
  mmc_hw_reset
  mmc_set_data_timeout
@@ -1609,7 +1606,7 @@
  proc_create_data
  proc_mkdir
  remove_proc_entry
-  rps_needed
+  request_firmware_direct
  sdio_claim_host
  sdio_claim_irq
  sdio_disable_func
@@ -1629,8 +1626,6 @@
  sdio_writesb
  skb_realloc_headroom
  sort
-  static_key_slow_dec
-  static_key_slow_inc
  strcat
  strchr
  strim
@@ -1680,7 +1675,6 @@
 # required by mxc-jpeg-encdec.ko
  v4l2_m2m_dqbuf
  v4l2_m2m_qbuf
-  v4l2_src_change_event_subscribe

 # required by mxs-dma.ko
  dmaenginem_async_device_register
@@ -1720,6 +1714,9 @@
  v4l2_event_subdev_unsubscribe
  __v4l2_find_nearest_size

+# required by ov5640_camera_mipi_v2.ko
+  devm_pinctrl_put
+
 # required by panel-raydium-rm67191.ko
  devm_backlight_device_register
  mipi_dsi_dcs_enter_sleep_mode
@@ -1836,7 +1833,6 @@
  backlight_device_register
  backlight_device_unregister
  devm_pwm_get
-  gpiod_get_direction
  pwm_apply_state
  pwm_free
  pwm_request
@@ -1995,7 +1991,6 @@
  snd_soc_dapm_put_enum_double

 # required by snd-soc-simple-card-utils.ko
-  devm_get_clk_from_child
  devm_kvasprintf
  of_property_read_string_helper
  snd_soc_dapm_get_pin_switch
@@ -2056,6 +2051,8 @@

 # required by trusty-ipc.ko
  _copy_from_iter
+  dma_buf_vmap
+  dma_buf_vunmap
  fget
  import_iovec
  strncpy_from_user
@@ -2117,6 +2114,7 @@
 # required by vsiv4l2.ko
  devm_device_add_group
  v4l2_event_dequeue
+  vb2_create_bufs
  vb2_prepare_buf

 # required by vvcam-video.ko
--- a/android/abi_gki_aarch64_mtk
+++ b/android/abi_gki_aarch64_mtk
@@ -1940,6 +1940,7 @@
  __traceiter_android_vh_rwsem_wake
  __traceiter_android_vh_rwsem_write_finished
  __traceiter_android_vh_scheduler_tick
+  __traceiter_android_vh_scmi_timeout_sync
  __traceiter_android_vh_selinux_avc_insert
  __traceiter_android_vh_selinux_avc_lookup
  __traceiter_android_vh_selinux_avc_node_delete
@@ -1952,6 +1953,7 @@
  __traceiter_android_vh_set_module_permit_after_init
  __traceiter_android_vh_set_module_permit_before_init
  __traceiter_android_vh_set_wake_flags
+  __traceiter_android_vh_snd_soc_card_get_comp_chain
  __traceiter_android_vh_syscall_prctl_finished
  __traceiter_android_vh_ufs_send_command
  __traceiter_android_vh_ufs_send_tm_command
@@ -2018,6 +2020,7 @@
  __tracepoint_android_vh_rwsem_wake
  __tracepoint_android_vh_rwsem_write_finished
  __tracepoint_android_vh_scheduler_tick
+  __tracepoint_android_vh_scmi_timeout_sync
  __tracepoint_android_vh_selinux_avc_insert
  __tracepoint_android_vh_selinux_avc_lookup
  __tracepoint_android_vh_selinux_avc_node_delete
@@ -2030,6 +2033,7 @@
  __tracepoint_android_vh_set_module_permit_after_init
  __tracepoint_android_vh_set_module_permit_before_init
  __tracepoint_android_vh_set_wake_flags
+  __tracepoint_android_vh_snd_soc_card_get_comp_chain
  __tracepoint_android_vh_syscall_prctl_finished
  __tracepoint_android_vh_ufs_send_command
  __tracepoint_android_vh_ufs_send_tm_command
--- a/android/abi_gki_aarch64_oplus
+++ b/android/abi_gki_aarch64_oplus
@@ -2594,6 +2594,7 @@
  __traceiter_android_vh_commit_creds
  __traceiter_android_vh_cpu_idle_enter
  __traceiter_android_vh_cpu_idle_exit
+  __traceiter_android_vh_cpu_up
  __traceiter_android_vh_do_send_sig_info
  __traceiter_android_vh_em_cpu_energy
  __traceiter_android_vh_exclude_reserved_zone
@@ -2776,6 +2777,7 @@
  __tracepoint_android_vh_commit_creds
  __tracepoint_android_vh_cpu_idle_enter
  __tracepoint_android_vh_cpu_idle_exit
+  __tracepoint_android_vh_cpu_up
  __tracepoint_android_vh_do_send_sig_info
  __tracepoint_android_vh_em_cpu_energy
  __tracepoint_android_vh_exclude_reserved_zone
--- a/android/abi_gki_aarch64_virtual_device
+++ b/android/abi_gki_aarch64_virtual_device
@@ -693,9 +693,7 @@
  led_set_brightness_nosleep
  led_trigger_event
  led_trigger_register
-  led_trigger_register_simple
  led_trigger_unregister
-  led_trigger_unregister_simple

 # required by lzo-rle.ko
  lzorle1x_1_compress
@@ -1349,6 +1347,7 @@
 # required by virtio_pci.ko
  irq_set_affinity_hint
  pci_alloc_irq_vectors_affinity
+  pci_device_is_present
  pci_disable_sriov
  pci_enable_sriov
  pci_find_capability
@@ -1383,8 +1382,6 @@
  wait_for_completion_interruptible_timeout

 # required by vmw_vsock_virtio_transport.ko
-  lock_sock_nested
-  release_sock
  virtio_transport_connect
  virtio_transport_deliver_tap_pkt
  virtio_transport_destruct
--- a/arch/arm64/Makefile.postlink
+++ b/arch/arm64/Makefile.postlink
@@ -3,8 +3,11 @@
 #
 # This file is included by the generic Kbuild makefile to permit the
 # architecture to perform postlink actions on vmlinux and any .ko module file.
-# In this case, we only need it for fips140.ko, which needs a HMAC digest to be
-# injected into it. All other targets are NOPs.
+# In this case, we only need it for fips140.ko, which needs some postprocessing
+# for the integrity check mandated by FIPS. This involves making copies of the
+# relocation sections so that the module will have access to them at
+# initialization time, and calculating and injecting a HMAC digest into the
+# module. All other targets are NOPs.
 #

 PHONY := __archpost
@@ -15,7 +18,14 @@ include scripts/Kbuild.include

 CMD_FIPS140_GEN_HMAC = crypto/fips140_gen_hmac
 quiet_cmd_gen_hmac = HMAC    $@
-      cmd_gen_hmac = $(CMD_FIPS140_GEN_HMAC) $@
+      cmd_gen_hmac = $(OBJCOPY) $@ \
+	--dump-section=$(shell $(READELF) -SW $@|grep -Eo '\.rela\.text\S*')=$@.rela.text \
+	--dump-section=$(shell $(READELF) -SW $@|grep -Eo '\.rela\.rodata\S*')=$@.rela.rodata \
+	--add-section=.init.rela.text=$@.rela.text \
+	--add-section=.init.rela.rodata=$@.rela.rodata \
+	--set-section-flags=.init.rela.text=alloc,readonly \
+	--set-section-flags=.init.rela.rodata=alloc,readonly && \
+	$(CMD_FIPS140_GEN_HMAC) $@

 # `@true` prevents complaints when there is nothing to be done

@@ -29,7 +39,7 @@ $(objtree)/crypto/fips140.ko: FORCE
 	@true

 clean:
-	@true
+	rm -f $(objtree)/crypto/fips140.ko.rela.*

 PHONY += FORCE clean

--- a/build.config.gki.aarch64.fips140
+++ b/build.config.gki.aarch64.fips140
@@ -15,3 +15,4 @@ DEFCONFIG=fips140_gki_defconfig
 KMI_SYMBOL_LIST=android/abi_gki_aarch64_fips140
 PRE_DEFCONFIG_CMDS="cat ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/gki_defconfig ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/fips140_gki.fragment > ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/${DEFCONFIG};"
 POST_DEFCONFIG_CMDS="rm ${ROOT_DIR}/${KERNEL_DIR}/arch/arm64/configs/${DEFCONFIG}"
+KMI_SYMBOL_LIST_ADD_ONLY=1
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -200,14 +200,19 @@ obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o

 ifneq ($(CONFIG_CRYPTO_FIPS140_MOD),)

-FIPS140_CFLAGS := -D__DISABLE_EXPORTS -DBUILD_FIPS140_KO
+FIPS140_CFLAGS := -D__DISABLE_EXPORTS -DBUILD_FIPS140_KO -include fips140-defs.h
+
+CFLAGS_jitterentropy-fips.o := -O0
+KASAN_SANITIZE_jitterentropy-fips.o = n
+UBSAN_SANITIZE_jitterentropy-fips.o = n

 #
 # Create a separate FIPS archive containing a duplicate of each builtin generic
 # module that is in scope for FIPS 140-2 certification
 #
-crypto-fips-objs := drbg.o ecb.o cbc.o ctr.o gcm.o xts.o hmac.o memneq.o \
-		    gf128mul.o aes_generic.o lib-crypto-aes.o \
+crypto-fips-objs := drbg.o ecb.o cbc.o ctr.o cts.o gcm.o xts.o hmac.o cmac.o \
+		    memneq.o gf128mul.o aes_generic.o lib-crypto-aes.o \
+		    jitterentropy.o jitterentropy-kcapi.o \
 		    sha1_generic.o sha256_generic.o sha512_generic.o \
 		    lib-sha1.o lib-crypto-sha256.o
 crypto-fips-objs := $(foreach o,$(crypto-fips-objs),$(o:.o=-fips.o))
@@ -228,10 +233,17 @@ $(obj)/lib-crypto-%-fips.o: $(srctree)/lib/crypto/%.c FORCE
 $(obj)/crypto-fips.a: $(addprefix $(obj)/,$(crypto-fips-objs)) FORCE
 	$(call if_changed,ar_and_symver)

-fips140-objs		:= fips140-module.o fips140-selftests.o crypto-fips.a
-obj-m			+= fips140.o
+fips140-objs := \
+	fips140-alg-registration.o \
+	fips140-module.o \
+	fips140-refs.o \
+	fips140-selftests.o \
+	crypto-fips.a
+obj-m += fips140.o

+CFLAGS_fips140-alg-registration.o += $(FIPS140_CFLAGS)
 CFLAGS_fips140-module.o += $(FIPS140_CFLAGS)
+CFLAGS_fips140-selftests.o += $(FIPS140_CFLAGS)

 hostprogs-always-y := fips140_gen_hmac
 HOSTLDLIBS_fips140_gen_hmac := -lcrypto -lelf
--- a/crypto/fips140-alg-registration.c
+++ b/crypto/fips140-alg-registration.c
@@ -0,0 +1,388 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Block crypto operations until tests complete
+ *
+ * Copyright 2021 Google LLC
+ *
+ * This file defines the fips140_crypto_register_*() functions, to which all
+ * calls to crypto_register_*() in the module are redirected.  These functions
+ * override the tfm initialization function of each algorithm to insert a wait
+ * for the module having completed its self-tests and integrity check.
+ *
+ * The exact field that we override depends on the algorithm type.  For
+ * algorithm types that have a strongly-typed initialization function pointer
+ * (e.g. skcipher), we must override that, since cra_init isn't guaranteed to be
+ * called for those despite the field being present in the base struct.  For the
+ * other algorithm types (e.g. "cipher") we must override cra_init.
+ *
+ * All of this applies to both normal algorithms and template instances.
+ *
+ * The purpose of all of this is to meet a FIPS requirement where the module
+ * must not produce any output from cryptographic algorithms until it completes
+ * its tests.  Technically this is impossible, but this solution meets the
+ * intent of the requirement, assuming the user makes a supported sequence of
+ * API calls.  Note that we can't simply run the tests before registering the
+ * algorithms, as the algorithms must be registered in order to run the tests.
+ *
+ * It would be much easier to handle this in the kernel's crypto API framework.
+ * Unfortunately, that was deemed insufficient because the module itself is
+ * required to do the enforcement.  What is *actually* required is still very
+ * vague, but the approach implemented here should meet the requirement.
+ */
+
+/*
+ * This file is the one place in fips140.ko that needs to call the kernel's real
+ * algorithm registration functions, so #undefine all the macros from
+ * fips140-defs.h so that the "fips140_" prefix doesn't automatically get added.
+ */
+#undef aead_register_instance
+#undef ahash_register_instance
+#undef crypto_register_aead
+#undef crypto_register_aeads
+#undef crypto_register_ahash
+#undef crypto_register_ahashes
+#undef crypto_register_alg
+#undef crypto_register_algs
+#undef crypto_register_rng
+#undef crypto_register_rngs
+#undef crypto_register_shash
+#undef crypto_register_shashes
+#undef crypto_register_skcipher
+#undef crypto_register_skciphers
+#undef shash_register_instance
+#undef skcipher_register_instance
+
+#include <crypto/algapi.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/hash.h>
+#include <crypto/internal/rng.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/xarray.h>
+
+#include "fips140-module.h"
+
+/* Indicates whether the self-tests and integrity check have completed */
+DECLARE_COMPLETION(fips140_tests_done);
+
+/* The thread running the self-tests and integrity check */
+struct task_struct *fips140_init_thread;
+
+/*
+ * Map from crypto_alg to original initialization function (possibly NULL)
+ *
+ * Note: unregistering an algorithm will leak its map entry, as we don't bother
+ * to remove it.  This should be fine since fips140.ko can't be unloaded.  The
+ * proper solution would be to store the original function pointer in a new
+ * field in 'struct crypto_alg', but that would require kernel support.
+ */
+static DEFINE_XARRAY(fips140_init_func_map);
+
+static bool fips140_ready(void)
+{
+	return completion_done(&fips140_tests_done);
+}
+
+/*
+ * Wait until crypto operations are allowed to proceed.  Return true if the
+ * tests are done, or false if the caller is the thread running the tests so it
+ * is allowed to proceed anyway.
+ */
+static bool fips140_wait_until_ready(struct crypto_alg *alg)
+{
+	if (fips140_ready())
+		return true;
+	/*
+	 * The thread running the tests must not wait.  Since tfms can only be
+	 * allocated in task context, we can reliably determine whether the
+	 * invocation is from that thread or not by checking 'current'.
+	 */
+	if (current == fips140_init_thread)
+		return false;
+
+	pr_info("blocking user of %s until tests complete\n",
+		alg->cra_driver_name);
+	wait_for_completion(&fips140_tests_done);
+	pr_info("tests done, allowing %s to proceed\n", alg->cra_driver_name);
+	return true;
+}
+
+static int fips140_store_init_function(struct crypto_alg *alg, void *func)
+{
+	void *ret;
+
+	/*
+	 * The XArray API requires 4-byte aligned values.  Although function
+	 * pointers in general aren't guaranteed to be 4-byte aligned, it should
+	 * be the case for the platforms this module is used on.
+	 */
+	if (WARN_ON((unsigned long)func & 3))
+		return -EINVAL;
+
+	ret = xa_store(&fips140_init_func_map, (unsigned long)alg, func,
+		       GFP_KERNEL);
+	return xa_err(ret);
+}
+
+/* Get the algorithm's original initialization function (possibly NULL) */
+static void *fips140_load_init_function(struct crypto_alg *alg)
+{
+	return xa_load(&fips140_init_func_map, (unsigned long)alg);
+}
+
+/* tfm initialization function overrides */
+
+static int fips140_alg_init_tfm(struct crypto_tfm *tfm)
+{
+	struct crypto_alg *alg = tfm->__crt_alg;
+	int (*cra_init)(struct crypto_tfm *tfm) =
+		fips140_load_init_function(alg);
+
+	if (fips140_wait_until_ready(alg))
+		WRITE_ONCE(alg->cra_init, cra_init);
+	return cra_init ? cra_init(tfm) : 0;
+}
+
+static int fips140_aead_init_tfm(struct crypto_aead *tfm)
+{
+	struct aead_alg *alg = crypto_aead_alg(tfm);
+	int (*init)(struct crypto_aead *tfm) =
+		fips140_load_init_function(&alg->base);
+
+	if (fips140_wait_until_ready(&alg->base))
+		WRITE_ONCE(alg->init, init);
+	return init ? init(tfm) : 0;
+}
+
+static int fips140_ahash_init_tfm(struct crypto_ahash *tfm)
+{
+	struct hash_alg_common *halg = crypto_hash_alg_common(tfm);
+	struct ahash_alg *alg = container_of(halg, struct ahash_alg, halg);
+	int (*init_tfm)(struct crypto_ahash *tfm) =
+		fips140_load_init_function(&halg->base);
+
+	if (fips140_wait_until_ready(&halg->base))
+		WRITE_ONCE(alg->init_tfm, init_tfm);
+	return init_tfm ? init_tfm(tfm) : 0;
+}
+
+static int fips140_shash_init_tfm(struct crypto_shash *tfm)
+{
+	struct shash_alg *alg = crypto_shash_alg(tfm);
+	int (*init_tfm)(struct crypto_shash *tfm) =
+		fips140_load_init_function(&alg->base);
+
+	if (fips140_wait_until_ready(&alg->base))
+		WRITE_ONCE(alg->init_tfm, init_tfm);
+	return init_tfm ? init_tfm(tfm) : 0;
+}
+
+static int fips140_skcipher_init_tfm(struct crypto_skcipher *tfm)
+{
+	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
+	int (*init)(struct crypto_skcipher *tfm) =
+		fips140_load_init_function(&alg->base);
+
+	if (fips140_wait_until_ready(&alg->base))
+		WRITE_ONCE(alg->init, init);
+	return init ? init(tfm) : 0;
+}
+
+/* Single algorithm registration */
+
+#define prepare_alg(alg, base_alg, field, wrapper_func)			\
+({									\
+	int err = 0;							\
+									\
+	if (!fips140_ready() && alg->field != wrapper_func) {		\
+		err = fips140_store_init_function(base_alg, alg->field);\
+		if (err == 0)						\
+			alg->field = wrapper_func;			\
+	}								\
+	err;								\
+})
+
+static int fips140_prepare_alg(struct crypto_alg *alg)
+{
+	/*
+	 * Override cra_init.  This is only for algorithm types like cipher and
+	 * rng that don't have a strongly-typed initialization function.
+	 */
+	return prepare_alg(alg, alg, cra_init, fips140_alg_init_tfm);
+}
+
+static int fips140_prepare_aead_alg(struct aead_alg *alg)
+{
+	return prepare_alg(alg, &alg->base, init, fips140_aead_init_tfm);
+}
+
+static int fips140_prepare_ahash_alg(struct ahash_alg *alg)
+{
+	return prepare_alg(alg, &alg->halg.base, init_tfm,
+			   fips140_ahash_init_tfm);
+}
+
+static int fips140_prepare_rng_alg(struct rng_alg *alg)
+{
+	/*
+	 * rng doesn't have a strongly-typed initialization function, so we must
+	 * treat rng algorithms as "generic" algorithms.
+	 */
+	return fips140_prepare_alg(&alg->base);
+}
+
+static int fips140_prepare_shash_alg(struct shash_alg *alg)
+{
+	return prepare_alg(alg, &alg->base, init_tfm, fips140_shash_init_tfm);
+}
+
+static int fips140_prepare_skcipher_alg(struct skcipher_alg *alg)
+{
+	return prepare_alg(alg, &alg->base, init, fips140_skcipher_init_tfm);
+}
+
+int fips140_crypto_register_alg(struct crypto_alg *alg)
+{
+	return fips140_prepare_alg(alg) ?: crypto_register_alg(alg);
+}
+
+int fips140_crypto_register_aead(struct aead_alg *alg)
+{
+	return fips140_prepare_aead_alg(alg) ?: crypto_register_aead(alg);
+}
+
+int fips140_crypto_register_ahash(struct ahash_alg *alg)
+{
+	return fips140_prepare_ahash_alg(alg) ?: crypto_register_ahash(alg);
+}
+
+int fips140_crypto_register_rng(struct rng_alg *alg)
+{
+	return fips140_prepare_rng_alg(alg) ?: crypto_register_rng(alg);
+}
+
+int fips140_crypto_register_shash(struct shash_alg *alg)
+{
+	return fips140_prepare_shash_alg(alg) ?: crypto_register_shash(alg);
+}
+
+int fips140_crypto_register_skcipher(struct skcipher_alg *alg)
+{
+	return fips140_prepare_skcipher_alg(alg) ?:
+		crypto_register_skcipher(alg);
+}
+
+/* Instance registration */
+
+int fips140_aead_register_instance(struct crypto_template *tmpl,
+				   struct aead_instance *inst)
+{
+	return fips140_prepare_aead_alg(&inst->alg) ?:
+		aead_register_instance(tmpl, inst);
+}
+
+int fips140_ahash_register_instance(struct crypto_template *tmpl,
+				    struct ahash_instance *inst)
+{
+	return fips140_prepare_ahash_alg(&inst->alg) ?:
+		ahash_register_instance(tmpl, inst);
+}
+
+int fips140_shash_register_instance(struct crypto_template *tmpl,
+				    struct shash_instance *inst)
+{
+	return fips140_prepare_shash_alg(&inst->alg) ?:
+		shash_register_instance(tmpl, inst);
+}
+
+int fips140_skcipher_register_instance(struct crypto_template *tmpl,
+				       struct skcipher_instance *inst)
+{
+	return fips140_prepare_skcipher_alg(&inst->alg) ?:
+		skcipher_register_instance(tmpl, inst);
+}
+
+/* Bulk algorithm registration */
+
+int fips140_crypto_register_algs(struct crypto_alg *algs, int count)
+{
+	int i;
+	int err;
+
+	for (i = 0; i < count; i++) {
+		err = fips140_prepare_alg(&algs[i]);
+		if (err)
+			return err;
+	}
+
+	return crypto_register_algs(algs, count);
+}
+
+int fips140_crypto_register_aeads(struct aead_alg *algs, int count)
+{
+	int i;
+	int err;
+
+	for (i = 0; i < count; i++) {
+		err = fips140_prepare_aead_alg(&algs[i]);
+		if (err)
+			return err;
+	}
+
+	return crypto_register_aeads(algs, count);
+}
+
+int fips140_crypto_register_ahashes(struct ahash_alg *algs, int count)
+{
+	int i;
+	int err;
+
+	for (i = 0; i < count; i++) {
+		err = fips140_prepare_ahash_alg(&algs[i]);
+		if (err)
+			return err;
+	}
+
+	return crypto_register_ahashes(algs, count);
+}
+
+int fips140_crypto_register_rngs(struct rng_alg *algs, int count)
+{
+	int i;
+	int err;
+
+	for (i = 0; i < count; i++) {
+		err = fips140_prepare_rng_alg(&algs[i]);
+		if (err)
+			return err;
+	}
+
+	return crypto_register_rngs(algs, count);
+}
+
+int fips140_crypto_register_shashes(struct shash_alg *algs, int count)
+{
+	int i;
+	int err;
+
+	for (i = 0; i < count; i++) {
+		err = fips140_prepare_shash_alg(&algs[i]);
+		if (err)
+			return err;
+	}
+
+	return crypto_register_shashes(algs, count);
+}
+
+int fips140_crypto_register_skciphers(struct skcipher_alg *algs, int count)
+{
+	int i;
+	int err;
+
+	for (i = 0; i < count; i++) {
+		err = fips140_prepare_skcipher_alg(&algs[i]);
+		if (err)
+			return err;
+	}
+
+	return crypto_register_skciphers(algs, count);
+}
--- a/crypto/fips140-defs.h
+++ b/crypto/fips140-defs.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This file is automatically included by all files built into fips140.ko, via
+ * the "-include" compiler flag.  It redirects all calls to algorithm
+ * registration functions to the wrapper functions defined within the module.
+ */
+
+#define aead_register_instance		fips140_aead_register_instance
+#define ahash_register_instance		fips140_ahash_register_instance
+#define crypto_register_aead		fips140_crypto_register_aead
+#define crypto_register_aeads		fips140_crypto_register_aeads
+#define crypto_register_ahash		fips140_crypto_register_ahash
+#define crypto_register_ahashes		fips140_crypto_register_ahashes
+#define crypto_register_alg		fips140_crypto_register_alg
+#define crypto_register_algs		fips140_crypto_register_algs
+#define crypto_register_rng		fips140_crypto_register_rng
+#define crypto_register_rngs		fips140_crypto_register_rngs
+#define crypto_register_shash		fips140_crypto_register_shash
+#define crypto_register_shashes		fips140_crypto_register_shashes
+#define crypto_register_skcipher	fips140_crypto_register_skcipher
+#define crypto_register_skciphers	fips140_crypto_register_skciphers
+#define shash_register_instance		fips140_shash_register_instance
+#define skcipher_register_instance	fips140_skcipher_register_instance
--- a/crypto/fips140-generated-testvecs.h
+++ b/crypto/fips140-generated-testvecs.h
@@ -11,19 +11,19 @@ static const u8 fips_message[32] __initconst =

 static const u8 fips_aes_key[16] __initconst = "128-bit AES key";

-static const u8 fips_aes_iv[16] __initconst = "ABCDEFGHIJKL";
+static const u8 fips_aes_iv[16] __initconst = "ABCDEFGHIJKLMNOP";

 static const u8 fips_aes_cbc_ciphertext[32] __initconst =
-	"\xc4\x6d\xad\xa4\x04\x52\x11\x5a\x7a\xb3\x7c\x68\x85\x8d\x90\xf0"
-	"\x55\xc3\xd3\x35\xc1\x75\x31\x90\xdf\x90\x4b\x5a\x56\xfd\xa7\x89";
+	"\x4c\x3e\xeb\x38\x8d\x1f\x28\xfd\xa2\x3b\xa9\xda\x36\xf2\x99\xe2"
+	"\x84\x84\x66\x37\x0a\x53\x68\x2f\x17\x95\x8d\x7f\xca\x5a\x68\x4e";

 static const u8 fips_aes_ecb_ciphertext[32] __initconst =
 	"\xc1\x9d\xe6\xb8\xb2\x90\xff\xfe\xf2\x77\x18\xb0\x55\xd3\xee\xa9"
 	"\xe2\x6f\x4a\x32\x67\xfd\xb7\xa5\x2f\x4b\x6e\x1a\x86\x2b\x6e\x3a";

 static const u8 fips_aes_ctr_ciphertext[32] __initconst =
-	"\x92\xbe\x23\xa1\x80\x88\x5d\x31\x27\xb3\x9c\x40\x58\x57\x1d\xde"
-	"\xc1\x8d\x5b\xe7\x42\x93\x09\xf8\xd4\xf7\x49\x42\xcf\x40\x62\x7e";
+	"\xed\x06\x2c\xd0\xbc\x48\xd1\x2e\x6a\x4e\x13\xe9\xaa\x17\x40\xca"
+	"\x00\xb4\xaf\x3b\x4f\xee\x73\xd6\x6c\x41\xf6\x4c\x8b\x0d\x6a\x0f";

 static const u8 fips_aes_gcm_assoc[22] __initconst = "associated data string";

@@ -36,8 +36,11 @@ static const u8 fips_aes_xts_key[32] __initconst =
 	"This is an AES-128-XTS key.";

 static const u8 fips_aes_xts_ciphertext[32] __initconst =
-	"\x5e\xb9\x98\xd6\x26\xb3\x55\xbf\x44\xab\x3e\xae\x73\xc0\x81\xc9"
-	"\xf4\x29\x0e\x17\x1e\xc5\xc8\x90\x79\x99\xf1\x43\x3a\x23\x08\x5a";
+	"\x4f\xf7\x9f\x6c\x00\xa8\x30\xdf\xff\xf3\x25\x9c\xf6\x0b\x1b\xfd"
+	"\x3b\x34\x5e\x67\x7c\xf8\x8b\x68\x9a\xb9\x5a\x89\x51\x51\xbd\x35";
+
+static const u8 fips_aes_cmac_digest[16] __initconst =
+	"\x0c\x05\xda\x64\x51\x0c\x8e\x6c\x86\x52\x46\xa8\x2d\xb1\xfe\x0f";

 static const u8 fips_hmac_key[16] __initconst = "128-bit HMAC key";

--- a/crypto/fips140-module.c
+++ b/crypto/fips140-module.c
@@ -66,18 +66,37 @@ const u8 *__rodata_start = &__fips140_rodata_start;
 /*
 * The list of the crypto API algorithms (by cra_name) that will be unregistered
 * by this module, in preparation for the module registering its own
- * implementation(s) of them.  When adding a new algorithm here, make sure to
- * consider whether it needs a self-test added to fips140_selftests[] as well.
+ * implementation(s) of them.
+ *
+ * All algorithms that will be declared as FIPS-approved in the module
+ * certification must be listed here, to ensure that the non-FIPS-approved
+ * implementations of these algorithms in the kernel image aren't used.
+ *
+ * For every algorithm in this list, the module should contain all the "same"
+ * implementations that the kernel image does, including the C implementation as
+ * well as any architecture-specific implementations.  This is needed to avoid
+ * performance regressions as well as the possibility of an algorithm being
+ * unavailable on some CPUs.  E.g., "xcbc(aes)" isn't in this list, as the
+ * module doesn't have a C implementation of it (and it won't be FIPS-approved).
+ *
+ * Due to a quirk in the FIPS requirements, "gcm(aes)" isn't actually able to be
+ * FIPS-approved.  However, we otherwise treat it the same as the algorithms
+ * that will be FIPS-approved, and therefore it's included in this list.
+ *
+ * When adding a new algorithm here, make sure to consider whether it needs a
+ * self-test added to fips140_selftests[] as well.
 */
 static const char * const fips140_algorithms[] __initconst = {
 	"aes",

-	"gcm(aes)",
-
+	"cmac(aes)",
 	"ecb(aes)",
+
 	"cbc(aes)",
+	"cts(cbc(aes))",
 	"ctr(aes)",
 	"xts(aes)",
+	"gcm(aes)",

 	"hmac(sha1)",
 	"hmac(sha224)",
@@ -91,6 +110,7 @@ static const char * const fips140_algorithms[] __initconst = {
 	"sha512",

 	"stdrng",
+	"jitterentropy_rng",
 };

 static bool __init is_fips140_algo(struct crypto_alg *alg)
@@ -110,7 +130,7 @@ static bool __init is_fips140_algo(struct crypto_alg *alg)
 	return false;
 }

-static LIST_HEAD(unchecked_fips140_algos);
+static LIST_HEAD(existing_live_algos);

 /*
 * Release a list of algorithms which have been removed from crypto_alg_list.
@@ -153,38 +173,53 @@ static void __init unregister_existing_fips140_algos(void)
 	down_write(&crypto_alg_sem);

 	/*
-	 * Find all registered algorithms that we care about, and move them to
-	 * a private list so that they are no longer exposed via the algo
-	 * lookup API. Subsequently, we will unregister them if they are not in
-	 * active use. If they are, we cannot simply remove them but we can
-	 * adapt them later to use our integrity checked backing code.
+	 * Find all registered algorithms that we care about, and move them to a
+	 * private list so that they are no longer exposed via the algo lookup
+	 * API. Subsequently, we will unregister them if they are not in active
+	 * use. If they are, we can't fully unregister them but we can ensure
+	 * that new users won't use them.
 	 */
 	list_for_each_entry_safe(alg, tmp, &crypto_alg_list, cra_list) {
-		if (is_fips140_algo(alg)) {
-			if (refcount_read(&alg->cra_refcnt) == 1) {
-				/*
-				 * This algorithm is not currently in use, but
-				 * there may be template instances holding
-				 * references to it via spawns. So let's tear
-				 * it down like crypto_unregister_alg() would,
-				 * but without releasing the lock, to prevent
-				 * races with concurrent TFM allocations.
-				 */
-				alg->cra_flags |= CRYPTO_ALG_DEAD;
-				list_move(&alg->cra_list, &remove_list);
-				crypto_remove_spawns(alg, &spawns, NULL);
-			} else {
-				/*
-				 * This algorithm is live, i.e., there are TFMs
-				 * allocated that rely on it for its crypto
-				 * transformations. We will swap these out
-				 * later with integrity checked versions.
-				 */
-				pr_info("found already-live algorithm '%s' ('%s')\n",
-					alg->cra_name, alg->cra_driver_name);
-				list_move(&alg->cra_list,
-					  &unchecked_fips140_algos);
-			}
+		if (!is_fips140_algo(alg))
+			continue;
+		if (refcount_read(&alg->cra_refcnt) == 1) {
+			/*
+			 * This algorithm is not currently in use, but there may
+			 * be template instances holding references to it via
+			 * spawns. So let's tear it down like
+			 * crypto_unregister_alg() would, but without releasing
+			 * the lock, to prevent races with concurrent TFM
+			 * allocations.
+			 */
+			alg->cra_flags |= CRYPTO_ALG_DEAD;
+			list_move(&alg->cra_list, &remove_list);
+			crypto_remove_spawns(alg, &spawns, NULL);
+		} else {
+			/*
+			 * This algorithm is live, i.e. it has TFMs allocated,
+			 * so we can't fully unregister it.  It's not necessary
+			 * to dynamically redirect existing users to the FIPS
+			 * code, given that they can't be relying on FIPS
+			 * certified crypto in the first place.  However, we do
+			 * need to ensure that new users will get the FIPS code.
+			 *
+			 * In most cases, setting alg->cra_priority to 0
+			 * achieves this.  However, that isn't enough for
+			 * algorithms like "hmac(sha256)" that need to be
+			 * instantiated from a template, since existing
+			 * algorithms always take priority over a template being
+			 * instantiated.  Therefore, we move the algorithm to
+			 * a private list so that algorithm lookups won't find
+			 * it anymore.  To further distinguish it from the FIPS
+			 * algorithms, we also append "+orig" to its name.
+			 */
+			pr_info("found already-live algorithm '%s' ('%s')\n",
+				alg->cra_name, alg->cra_driver_name);
+			alg->cra_priority = 0;
+			strlcat(alg->cra_name, "+orig", CRYPTO_MAX_ALG_NAME);
+			strlcat(alg->cra_driver_name, "+orig",
+				CRYPTO_MAX_ALG_NAME);
+			list_move(&alg->cra_list, &existing_live_algos);
 		}
 	}
 	up_write(&crypto_alg_sem);
@@ -259,12 +294,19 @@ static void __init unapply_rodata_relocations(void *section, int section_size,
 	}
 }

+extern struct {
+	u32	offset;
+	u32	count;
+} fips140_rela_text, fips140_rela_rodata;
+
 static bool __init check_fips140_module_hmac(void)
 {
+	struct crypto_shash *tfm = NULL;
 	SHASH_DESC_ON_STACK(desc, dontcare);
 	u8 digest[SHA256_DIGEST_SIZE];
 	void *textcopy, *rodatacopy;
 	int textsize, rodatasize;
+	bool ok = false;
 	int err;

 	textsize	= &__fips140_text_end - &__fips140_text_start;
@@ -276,7 +318,7 @@ static bool __init check_fips140_module_hmac(void)
 	textcopy = kmalloc(textsize + rodatasize, GFP_KERNEL);
 	if (!textcopy) {
 		pr_err("Failed to allocate memory for copy of .text\n");
-		return false;
+		goto out;
 	}

 	rodatacopy = textcopy + textsize;
@@ -286,38 +328,36 @@ static bool __init check_fips140_module_hmac(void)

 	// apply the relocations in reverse on the copies of .text  and .rodata
 	unapply_text_relocations(textcopy, textsize,
-				 __this_module.arch.text_relocations,
-				 __this_module.arch.num_text_relocations);
+				 offset_to_ptr(&fips140_rela_text.offset),
+				 fips140_rela_text.count);

 	unapply_rodata_relocations(rodatacopy, rodatasize,
-				   __this_module.arch.rodata_relocations,
-				   __this_module.arch.num_rodata_relocations);
+				  offset_to_ptr(&fips140_rela_rodata.offset),
+				  fips140_rela_rodata.count);

-	kfree(__this_module.arch.text_relocations);
-	kfree(__this_module.arch.rodata_relocations);
-
-	desc->tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);
-	if (IS_ERR(desc->tfm)) {
-		pr_err("failed to allocate hmac tfm (%ld)\n", PTR_ERR(desc->tfm));
-		kfree(textcopy);
-		return false;
+	tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);
+	if (IS_ERR(tfm)) {
+		pr_err("failed to allocate hmac tfm (%ld)\n", PTR_ERR(tfm));
+		tfm = NULL;
+		goto out;
 	}
+	desc->tfm = tfm;

 	pr_info("using '%s' for integrity check\n",
-		crypto_shash_driver_name(desc->tfm));
+		crypto_shash_driver_name(tfm));

-	err = crypto_shash_setkey(desc->tfm, fips140_integ_hmac_key,
+	err = crypto_shash_setkey(tfm, fips140_integ_hmac_key,
 				  strlen(fips140_integ_hmac_key)) ?:
 	      crypto_shash_init(desc) ?:
 	      crypto_shash_update(desc, textcopy, textsize) ?:
 	      crypto_shash_finup(desc, rodatacopy, rodatasize, digest);

-	crypto_free_shash(desc->tfm);
-	kfree(textcopy);
+	/* Zeroizing this is important; see the comment below. */
+	shash_desc_zero(desc);

 	if (err) {
 		pr_err("failed to calculate hmac shash (%d)\n", err);
-		return false;
+		goto out;
 	}

 	if (memcmp(digest, fips140_integ_hmac_digest, sizeof(digest))) {
@@ -326,171 +366,20 @@ static bool __init check_fips140_module_hmac(void)

 		pr_err("calculated digest: %*phN\n", (int)sizeof(digest),
 		       digest);
-
-		return false;
+		goto out;
 	}
-
-	return true;
-}
-
-static bool __init update_live_fips140_algos(void)
-{
-	struct crypto_alg *alg, *new_alg, *tmp;
-
+	ok = true;
+out:
 	/*
-	 * Find all algorithms that we could not unregister the last time
-	 * around, due to the fact that they were already in use.
+	 * FIPS 140-3 requires that all "temporary value(s) generated during the
+	 * integrity test" be zeroized (ref: FIPS 140-3 IG 9.7.B).  There is no
+	 * technical reason to do this given that these values are public
+	 * information, but this is the requirement so we follow it.
 	 */
-	down_write(&crypto_alg_sem);
-	list_for_each_entry_safe(alg, tmp, &unchecked_fips140_algos, cra_list) {
-
-		/*
-		 * Take this algo off the list before releasing the lock. This
-		 * ensures that a concurrent invocation of
-		 * crypto_unregister_alg() observes a consistent state, i.e.,
-		 * the algo is still on the list, and crypto_unregister_alg()
-		 * will release it, or it is not, and crypto_unregister_alg()
-		 * will issue a warning but ignore this condition otherwise.
-		 */
-		list_del_init(&alg->cra_list);
-		up_write(&crypto_alg_sem);
-
-		/*
-		 * Grab the algo that will replace the live one.
-		 * Note that this will instantiate template based instances as
-		 * well, as long as their driver name uses the conventional
-		 * pattern of "template(algo)". In this case, we are relying on
-		 * the fact that the templates carried by this module will
-		 * supersede the builtin ones, due to the fact that they were
-		 * registered later, and therefore appear first in the linked
-		 * list. For example, "hmac(sha1-ce)" constructed using the
-		 * builtin hmac template and the builtin SHA1 driver will be
-		 * superseded by the integrity checked versions of HMAC and
-		 * SHA1-ce carried in this module.
-		 *
-		 * Note that this takes a reference to the new algorithm which
-		 * will never get released. This is intentional: once we copy
-		 * the function pointers from the new algo into the old one, we
-		 * cannot drop the new algo unless we are sure that the old one
-		 * has been released, and this is someting we don't keep track
-		 * of at the moment.
-		 */
-		new_alg = crypto_alg_mod_lookup(alg->cra_driver_name,
-						alg->cra_flags & CRYPTO_ALG_TYPE_MASK,
-						CRYPTO_ALG_TYPE_MASK | CRYPTO_NOLOAD);
-
-		if (IS_ERR(new_alg)) {
-			pr_crit("Failed to allocate '%s' for updating live algo (%ld)\n",
-				alg->cra_driver_name, PTR_ERR(new_alg));
-			return false;
-		}
-
-		/*
-		 * The FIPS module's algorithms are expected to be built from
-		 * the same source code as the in-kernel ones so that they are
-		 * fully compatible. In general, there's no way to verify full
-		 * compatibility at runtime, but we can at least verify that
-		 * the algorithm properties match.
-		 */
-		if (alg->cra_ctxsize != new_alg->cra_ctxsize ||
-		    alg->cra_alignmask != new_alg->cra_alignmask) {
-			pr_crit("Failed to update live algo '%s' due to mismatch:\n"
-				"cra_ctxsize   : %u vs %u\n"
-				"cra_alignmask : 0x%x vs 0x%x\n",
-				alg->cra_driver_name,
-				alg->cra_ctxsize, new_alg->cra_ctxsize,
-				alg->cra_alignmask, new_alg->cra_alignmask);
-			return false;
-		}
-
-		/*
-		 * Update the name and priority so the algorithm stands out as
-		 * one that was updated in order to comply with FIPS140, and
-		 * that it is not the preferred version for further use.
-		 */
-		strlcat(alg->cra_name, "+orig", CRYPTO_MAX_ALG_NAME);
-		alg->cra_priority = 0;
-
-		switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
-			struct aead_alg *old_aead, *new_aead;
-			struct skcipher_alg *old_skcipher, *new_skcipher;
-			struct shash_alg *old_shash, *new_shash;
-			struct rng_alg *old_rng, *new_rng;
-
-		case CRYPTO_ALG_TYPE_CIPHER:
-			alg->cra_u.cipher = new_alg->cra_u.cipher;
-			break;
-
-		case CRYPTO_ALG_TYPE_AEAD:
-			old_aead = container_of(alg, struct aead_alg, base);
-			new_aead = container_of(new_alg, struct aead_alg, base);
-
-			old_aead->setkey	= new_aead->setkey;
-			old_aead->setauthsize	= new_aead->setauthsize;
-			old_aead->encrypt	= new_aead->encrypt;
-			old_aead->decrypt	= new_aead->decrypt;
-			old_aead->init		= new_aead->init;
-			old_aead->exit		= new_aead->exit;
-			break;
-
-		case CRYPTO_ALG_TYPE_SKCIPHER:
-			old_skcipher = container_of(alg, struct skcipher_alg, base);
-			new_skcipher = container_of(new_alg, struct skcipher_alg, base);
-
-			old_skcipher->setkey	= new_skcipher->setkey;
-			old_skcipher->encrypt	= new_skcipher->encrypt;
-			old_skcipher->decrypt	= new_skcipher->decrypt;
-			old_skcipher->init	= new_skcipher->init;
-			old_skcipher->exit	= new_skcipher->exit;
-			break;
-
-		case CRYPTO_ALG_TYPE_SHASH:
-			old_shash = container_of(alg, struct shash_alg, base);
-			new_shash = container_of(new_alg, struct shash_alg, base);
-
-			old_shash->init		= new_shash->init;
-			old_shash->update	= new_shash->update;
-			old_shash->final	= new_shash->final;
-			old_shash->finup	= new_shash->finup;
-			old_shash->digest	= new_shash->digest;
-			old_shash->export	= new_shash->export;
-			old_shash->import	= new_shash->import;
-			old_shash->setkey	= new_shash->setkey;
-			old_shash->init_tfm	= new_shash->init_tfm;
-			old_shash->exit_tfm	= new_shash->exit_tfm;
-			break;
-
-		case CRYPTO_ALG_TYPE_RNG:
-			old_rng = container_of(alg, struct rng_alg, base);
-			new_rng = container_of(new_alg, struct rng_alg, base);
-
-			old_rng->generate	= new_rng->generate;
-			old_rng->seed		= new_rng->seed;
-			old_rng->set_ent	= new_rng->set_ent;
-			break;
-		default:
-			/*
-			 * This should never happen: every item on the
-			 * fips140_algorithms list should match one of the
-			 * cases above, so if we end up here, something is
-			 * definitely wrong.
-			 */
-			pr_crit("Unexpected type %u for algo %s, giving up ...\n",
-				alg->cra_flags & CRYPTO_ALG_TYPE_MASK,
-				alg->cra_driver_name);
-			return false;
-		}
-
-		/*
-		 * Move the algorithm back to the algorithm list, so it is
-		 * visible in /proc/crypto et al.
-		 */
-		down_write(&crypto_alg_sem);
-		list_add_tail(&alg->cra_list, &crypto_alg_list);
-	}
-	up_write(&crypto_alg_sem);
-
-	return true;
+	crypto_free_shash(tfm);
+	memzero_explicit(digest, sizeof(digest));
+	kfree_sensitive(textcopy);
+	return ok;
 }

 static void fips140_sha256(void *p, const u8 *data, unsigned int len, u8 *out,
@@ -549,6 +438,7 @@ fips140_init(void)
 	const u32 *initcall;

 	pr_info("loading module\n");
+	fips140_init_thread = current;

 	unregister_existing_fips140_algos();

@@ -570,19 +460,6 @@ fips140_init(void)
 		}
 	}

-	if (!update_live_fips140_algos())
-		goto panic;
-
-	if (!update_fips140_library_routines())
-		goto panic;
-
-	/*
-	 * Wait until all tasks have at least been scheduled once and preempted
-	 * voluntarily. This ensures that none of the superseded algorithms that
-	 * were already in use will still be live.
-	 */
-	synchronize_rcu_tasks();
-
 	if (!fips140_run_selftests())
 		goto panic;

@@ -601,6 +478,11 @@ fips140_init(void)
 	}
 	pr_info("integrity check passed\n");

+	complete_all(&fips140_tests_done);
+
+	if (!update_fips140_library_routines())
+		goto panic;
+
 	pr_info("module successfully loaded\n");
 	return 0;

--- a/crypto/fips140-module.h
+++ b/crypto/fips140-module.h
@@ -6,6 +6,7 @@
 #ifndef _CRYPTO_FIPS140_MODULE_H
 #define _CRYPTO_FIPS140_MODULE_H

+#include <linux/completion.h>
 #include <linux/module.h>

 #undef pr_fmt
@@ -15,6 +16,9 @@
 extern char *fips140_broken_alg;
 #endif

+extern struct completion fips140_tests_done;
+extern struct task_struct *fips140_init_thread;
+
 bool __init __must_check fips140_run_selftests(void);

 #endif /* _CRYPTO_FIPS140_MODULE_H */
--- a/crypto/fips140-refs.S
+++ b/crypto/fips140-refs.S
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright 2021 Google LLC
+ * Author: Ard Biesheuvel <ardb@google.com>
+ *
+ * This file contains the variable definitions that will be used by the FIPS140
+ * s/w module to access the RELA sections in the ELF image. These are used to
+ * apply the relocations applied by the module loader in reverse, so that we
+ * can reconstruct the image that was used to derive the HMAC used by the
+ * integrity check.
+ *
+ * The first .long of each entry will be populated by the module loader based
+ * on the actual placement of the respective RELA section in memory. The second
+ * .long carries the RELA entry count, and is populated by the host tool that
+ * also generates the HMAC of the contents of .text and .rodata.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.section	".init.rodata", "a"
+
+	.align	2
+	.globl	fips140_rela_text
+fips140_rela_text:
+	.weak	__sec_rela_text
+	.long	__sec_rela_text - .
+	.long	0
+
+	.globl	fips140_rela_rodata
+fips140_rela_rodata:
+	.weak	__sec_rela_rodata
+	.long	__sec_rela_rodata - .
+	.long	0
--- a/crypto/fips140-selftests.c
+++ b/crypto/fips140-selftests.c
@@ -14,10 +14,6 @@
 * is somewhat helpful.  Basically, all implementations of all FIPS approved
 * algorithms (including modes of operation) must be tested.  However:
 *
- *   - If an implementation won't be used, it doesn't have to be tested.  So
- *     when multiple implementations of the same algorithm are registered with
- *     the crypto API, we only have to test the default (highest-priority) one.
- *
 *   - There are provisions for skipping tests that are already sufficiently
 *     covered by other tests.  E.g., HMAC-SHA256 may cover SHA-256.
 *
@@ -28,12 +24,16 @@
 *
 *   - Only one key size per algorithm needs to be tested.
 *
+ * There is some ambiguity about whether all implementations of each algorithm
+ * must be tested, or whether it is sufficient to test just the highest priority
+ * implementation.  To be safe we test all implementations, except ones that can
+ * be excluded by one of the rules above.
+ *
 * See fips140_selftests[] for the list of tests we've selected.  Currently, all
- * our test vectors except the DRBG ones were generated by the script
- * tools/crypto/gen_fips140_testvecs.py, using the known-good implementations in
- * the Python packages hashlib, pycryptodome, and cryptography.  The DRBG test
- * vectors were manually extracted from
- * https://csrc.nist.gov/CSRC/media/Projects/Cryptographic-Algorithm-Validation-Program/documents/drbg/drbgtestvectors.zip.
+ * our test vectors except the AES-CBC-CTS and DRBG ones were generated by the
+ * script tools/crypto/gen_fips140_testvecs.py, using the known-good
+ * implementations in the Python packages hashlib, pycryptodome, and
+ * cryptography.
 *
 * Note that we don't reuse the upstream crypto API's self-tests
 * (crypto/testmgr.{c,h}), for several reasons:
@@ -54,22 +54,12 @@
 #include <crypto/aes.h>
 #include <crypto/drbg.h>
 #include <crypto/hash.h>
-#include <crypto/internal/cipher.h>
 #include <crypto/rng.h>
 #include <crypto/sha.h>
 #include <crypto/skcipher.h>

 #include "fips140-module.h"

-/* Test vector for a block cipher algorithm */
-struct blockcipher_testvec {
-	const u8 *key;
-	size_t key_size;
-	const u8 *plaintext;
-	const u8 *ciphertext;
-	size_t block_size;
-};
-
 /* Test vector for an AEAD algorithm */
 struct aead_testvec {
 	const u8 *key;
@@ -121,15 +111,27 @@ struct drbg_testvec {
 	size_t out_size;
 };

-/*
- * A struct which specifies an algorithm name (using crypto API syntax), a test
- * function for that algorithm, and a test vector used by that test function.
- */
 struct fips_test {
+	/* The name of the algorithm, in crypto API syntax */
 	const char *alg;
-	int __must_check (*func)(const struct fips_test *test);
+
+	/*
+	 * The optional list of implementations to test.  @func will be called
+	 * once per implementation, or once with @alg if this list is empty.
+	 * The implementation names must be given in crypto API syntax, or in
+	 * the case of a library implementation should have "-lib" appended.
+	 */
+	const char *impls[8];
+
+	/*
+	 * The test function.  It should execute a known-answer test on an
+	 * algorithm implementation, using the below test vector.
+	 */
+	int __must_check (*func)(const struct fips_test *test,
+				 const char *impl);
+
+	/* The test vector, with a format specific to the type of algorithm */
 	union {
-		struct blockcipher_testvec blockcipher;
 		struct aead_testvec aead;
 		struct skcipher_testvec skcipher;
 		struct hash_testvec hash;
@@ -141,17 +143,16 @@ struct fips_test {
 #define MAX_IV_SIZE	16

 static int __init __must_check
-fips_check_result(const struct fips_test *test, u8 *result,
-		  const u8 *expected_result, size_t result_size,
-		  const char *operation)
+fips_check_result(u8 *result, const u8 *expected_result, size_t result_size,
+		  const char *impl, const char *operation)
 {
 #ifdef CONFIG_CRYPTO_FIPS140_MOD_ERROR_INJECTION
 	/* Inject a failure (via corrupting the result) if requested. */
-	if (fips140_broken_alg && strcmp(test->alg, fips140_broken_alg) == 0)
+	if (fips140_broken_alg && strcmp(impl, fips140_broken_alg) == 0)
 		result[0] ^= 0xff;
 #endif
 	if (memcmp(result, expected_result, result_size) != 0) {
-		pr_err("wrong result from %s %s\n", test->alg, operation);
+		pr_err("wrong result from %s %s\n", impl, operation);
 		return -EBADMSG;
 	}
 	return 0;
@@ -176,96 +177,56 @@ fips_validate_alg(const struct crypto_alg *alg)
 	return 0;
 }

-/* Test a block cipher using the crypto_cipher API. */
 static int __init __must_check
-fips_test_blockcipher(const struct fips_test *test)
+fips_handle_alloc_tfm_error(const char *impl, int err)
 {
-	const struct blockcipher_testvec *vec = &test->blockcipher;
-	struct crypto_cipher *tfm;
-	u8 block[MAX_CIPHER_BLOCKSIZE];
-	int err;
-
-	if (WARN_ON(vec->block_size > MAX_CIPHER_BLOCKSIZE))
-		return -EINVAL;
-
-	tfm = crypto_alloc_cipher(test->alg, 0, 0);
-	if (IS_ERR(tfm)) {
-		err = PTR_ERR(tfm);
-		pr_err("failed to allocate %s tfm: %d\n", test->alg, err);
-		return err;
+	if (err == -ENOENT) {
+		/*
+		 * The requested implementation of the algorithm wasn't found.
+		 * This is expected if the CPU lacks a feature the
+		 * implementation needs, such as the ARMv8 Crypto Extensions.
+		 *
+		 * When this happens, the implementation isn't available for
+		 * use, so we can't test it, nor do we need to.  So we just skip
+		 * the test.
+		 */
+		pr_info("%s is unavailable (no CPU support?), skipping testing it\n",
+			impl);
+		return 0;
 	}
-	err = fips_validate_alg(tfm->base.__crt_alg);
-	if (err)
-		goto out;
-	if (crypto_cipher_blocksize(tfm) != vec->block_size) {
-		pr_err("%s has wrong block size\n", test->alg);
-		err = -EINVAL;
-		goto out;
-	}
-
-	err = crypto_cipher_setkey(tfm, vec->key, vec->key_size);
-	if (err) {
-		pr_err("failed to set %s key: %d\n", test->alg, err);
-		goto out;
-	}
-
-	/* Encrypt the plaintext, then verify the resulting ciphertext. */
-	memcpy(block, vec->plaintext, vec->block_size);
-	crypto_cipher_encrypt_one(tfm, block, block);
-	err = fips_check_result(test, block, vec->ciphertext, vec->block_size,
-				"encryption");
-	if (err)
-		goto out;
-
-	/* Decrypt the ciphertext, then verify the resulting plaintext. */
-	crypto_cipher_decrypt_one(tfm, block, block);
-	err = fips_check_result(test, block, vec->plaintext, vec->block_size,
-				"decryption");
-out:
-	crypto_free_cipher(tfm);
+	pr_err("failed to allocate %s tfm: %d\n", impl, err);
 	return err;
 }

-/*
- * Test for plain AES (no mode of operation).  We test this separately from the
- * AES modes because the implementation of AES which is used by the "aes"
- * crypto_cipher isn't necessarily the same as that used by the AES modes such
- * as "ecb(aes)".  Similarly, the aes_{encrypt,decrypt}() library functions may
- * use a different implementation as well, so we test them separately too.
- */
 static int __init __must_check
-fips_test_aes(const struct fips_test *test)
+fips_test_aes_library(const struct fips_test *test, const char *impl)
 {
-	const struct blockcipher_testvec *vec = &test->blockcipher;
+	const struct skcipher_testvec *vec = &test->skcipher;
 	struct crypto_aes_ctx ctx;
 	u8 block[AES_BLOCK_SIZE];
 	int err;

-	if (WARN_ON(vec->block_size != AES_BLOCK_SIZE))
+	if (WARN_ON(vec->message_size != AES_BLOCK_SIZE))
 		return -EINVAL;

-	err = fips_test_blockcipher(test);
-	if (err)
-		return err;
-
 	err = aes_expandkey(&ctx, vec->key, vec->key_size);
 	if (err) {
 		pr_err("aes_expandkey() failed: %d\n", err);
 		return err;
 	}
 	aes_encrypt(&ctx, block, vec->plaintext);
-	err = fips_check_result(test, block, vec->ciphertext, AES_BLOCK_SIZE,
-				"encryption (library API)");
+	err = fips_check_result(block, vec->ciphertext, AES_BLOCK_SIZE,
+				impl, "encryption");
 	if (err)
 		return err;
 	aes_decrypt(&ctx, block, block);
-	return fips_check_result(test, block, vec->plaintext, AES_BLOCK_SIZE,
-				 "decryption (library API)");
+	return fips_check_result(block, vec->plaintext, AES_BLOCK_SIZE,
+				 impl, "decryption");
 }

 /* Test a length-preserving symmetric cipher using the crypto_skcipher API. */
 static int __init __must_check
-fips_test_skcipher(const struct fips_test *test)
+fips_test_skcipher(const struct fips_test *test, const char *impl)
 {
 	const struct skcipher_testvec *vec = &test->skcipher;
 	struct crypto_skcipher *tfm;
@@ -277,18 +238,17 @@ fips_test_skcipher(const struct fips_test *test)

 	if (WARN_ON(vec->iv_size > MAX_IV_SIZE))
 		return -EINVAL;
+	if (WARN_ON(vec->message_size <= 0))
+		return -EINVAL;

-	tfm = crypto_alloc_skcipher(test->alg, 0, 0);
-	if (IS_ERR(tfm)) {
-		err = PTR_ERR(tfm);
-		pr_err("failed to allocate %s tfm: %d\n", test->alg, err);
-		return err;
-	}
+	tfm = crypto_alloc_skcipher(impl, 0, 0);
+	if (IS_ERR(tfm))
+		return fips_handle_alloc_tfm_error(impl, PTR_ERR(tfm));
 	err = fips_validate_alg(&crypto_skcipher_alg(tfm)->base);
 	if (err)
 		goto out;
 	if (crypto_skcipher_ivsize(tfm) != vec->iv_size) {
-		pr_err("%s has wrong IV size\n", test->alg);
+		pr_err("%s has wrong IV size\n", impl);
 		err = -EINVAL;
 		goto out;
 	}
@@ -307,7 +267,7 @@ fips_test_skcipher(const struct fips_test *test)

 	err = crypto_skcipher_setkey(tfm, vec->key, vec->key_size);
 	if (err) {
-		pr_err("failed to set %s key: %d\n", test->alg, err);
+		pr_err("failed to set %s key: %d\n", impl, err);
 		goto out;
 	}

@@ -315,11 +275,11 @@ fips_test_skcipher(const struct fips_test *test)
 	memcpy(iv, vec->iv, vec->iv_size);
 	err = crypto_skcipher_encrypt(req);
 	if (err) {
-		pr_err("%s encryption failed: %d\n", test->alg, err);
+		pr_err("%s encryption failed: %d\n", impl, err);
 		goto out;
 	}
-	err = fips_check_result(test, message, vec->ciphertext,
-				vec->message_size, "encryption");
+	err = fips_check_result(message, vec->ciphertext, vec->message_size,
+				impl, "encryption");
 	if (err)
 		goto out;

@@ -327,11 +287,11 @@ fips_test_skcipher(const struct fips_test *test)
 	memcpy(iv, vec->iv, vec->iv_size);
 	err = crypto_skcipher_decrypt(req);
 	if (err) {
-		pr_err("%s decryption failed: %d\n", test->alg, err);
+		pr_err("%s decryption failed: %d\n", impl, err);
 		goto out;
 	}
-	err = fips_check_result(test, message, vec->plaintext,
-				vec->message_size, "decryption");
+	err = fips_check_result(message, vec->plaintext, vec->message_size,
+				impl, "decryption");
 out:
 	kfree(message);
 	skcipher_request_free(req);
@@ -341,7 +301,7 @@ out:

 /* Test an AEAD using the crypto_aead API. */
 static int __init __must_check
-fips_test_aead(const struct fips_test *test)
+fips_test_aead(const struct fips_test *test, const char *impl)
 {
 	const struct aead_testvec *vec = &test->aead;
 	const int tag_size = vec->ciphertext_size - vec->plaintext_size;
@@ -359,17 +319,14 @@ fips_test_aead(const struct fips_test *test)
 	if (WARN_ON(vec->ciphertext_size <= vec->plaintext_size))
 		return -EINVAL;

-	tfm = crypto_alloc_aead(test->alg, 0, 0);
-	if (IS_ERR(tfm)) {
-		err = PTR_ERR(tfm);
-		pr_err("failed to allocate %s tfm: %d\n", test->alg, err);
-		return err;
-	}
+	tfm = crypto_alloc_aead(impl, 0, 0);
+	if (IS_ERR(tfm))
+		return fips_handle_alloc_tfm_error(impl, PTR_ERR(tfm));
 	err = fips_validate_alg(&crypto_aead_alg(tfm)->base);
 	if (err)
 		goto out;
 	if (crypto_aead_ivsize(tfm) != vec->iv_size) {
-		pr_err("%s has wrong IV size\n", test->alg);
+		pr_err("%s has wrong IV size\n", impl);
 		err = -EINVAL;
 		goto out;
 	}
@@ -393,14 +350,14 @@ fips_test_aead(const struct fips_test *test)

 	err = crypto_aead_setkey(tfm, vec->key, vec->key_size);
 	if (err) {
-		pr_err("failed to set %s key: %d\n", test->alg, err);
+		pr_err("failed to set %s key: %d\n", impl, err);
 		goto out;
 	}

 	err = crypto_aead_setauthsize(tfm, tag_size);
 	if (err) {
 		pr_err("failed to set %s authentication tag size: %d\n",
-		       test->alg, err);
+		       impl, err);
 		goto out;
 	}

@@ -412,11 +369,11 @@ fips_test_aead(const struct fips_test *test)
 	aead_request_set_crypt(req, sg, sg, vec->plaintext_size, iv);
 	err = crypto_aead_encrypt(req);
 	if (err) {
-		pr_err("%s encryption failed: %d\n", test->alg, err);
+		pr_err("%s encryption failed: %d\n", impl, err);
 		goto out;
 	}
-	err = fips_check_result(test, message, vec->ciphertext,
-				vec->ciphertext_size, "encryption");
+	err = fips_check_result(message, vec->ciphertext, vec->ciphertext_size,
+				impl, "encryption");
 	if (err)
 		goto out;

@@ -428,11 +385,11 @@ fips_test_aead(const struct fips_test *test)
 	aead_request_set_crypt(req, sg, sg, vec->ciphertext_size, iv);
 	err = crypto_aead_decrypt(req);
 	if (err) {
-		pr_err("%s decryption failed: %d\n", test->alg, err);
+		pr_err("%s decryption failed: %d\n", impl, err);
 		goto out;
 	}
-	err = fips_check_result(test, message, vec->plaintext,
-				vec->plaintext_size, "decryption");
+	err = fips_check_result(message, vec->plaintext, vec->plaintext_size,
+				impl, "decryption");
 out:
 	kfree(message);
 	kfree(assoc);
@@ -449,7 +406,7 @@ out:
 * be no hash algorithms that can be accessed only through crypto_ahash.
 */
 static int __init __must_check
-fips_test_hash(const struct fips_test *test)
+fips_test_hash(const struct fips_test *test, const char *impl)
 {
 	const struct hash_testvec *vec = &test->hash;
 	struct crypto_shash *tfm;
@@ -459,17 +416,14 @@ fips_test_hash(const struct fips_test *test)
 	if (WARN_ON(vec->digest_size > HASH_MAX_DIGESTSIZE))
 		return -EINVAL;

-	tfm = crypto_alloc_shash(test->alg, 0, 0);
-	if (IS_ERR(tfm)) {
-		err = PTR_ERR(tfm);
-		pr_err("failed to allocate %s tfm: %d\n", test->alg, err);
-		return err;
-	}
+	tfm = crypto_alloc_shash(impl, 0, 0);
+	if (IS_ERR(tfm))
+		return fips_handle_alloc_tfm_error(impl, PTR_ERR(tfm));
 	err = fips_validate_alg(&crypto_shash_alg(tfm)->base);
 	if (err)
 		goto out;
 	if (crypto_shash_digestsize(tfm) != vec->digest_size) {
-		pr_err("%s has wrong digest size\n", test->alg);
+		pr_err("%s has wrong digest size\n", impl);
 		err = -EINVAL;
 		goto out;
 	}
@@ -477,7 +431,7 @@ fips_test_hash(const struct fips_test *test)
 	if (vec->key) {
 		err = crypto_shash_setkey(tfm, vec->key, vec->key_size);
 		if (err) {
-			pr_err("failed to set %s key: %d\n", test->alg, err);
+			pr_err("failed to set %s key: %d\n", impl, err);
 			goto out;
 		}
 	}
@@ -485,22 +439,18 @@ fips_test_hash(const struct fips_test *test)
 	err = crypto_shash_tfm_digest(tfm, vec->message, vec->message_size,
 				      digest);
 	if (err) {
-		pr_err("%s digest computation failed: %d\n", test->alg, err);
+		pr_err("%s digest computation failed: %d\n", impl, err);
 		goto out;
 	}
-	err = fips_check_result(test, digest, vec->digest, vec->digest_size,
-				"digest");
+	err = fips_check_result(digest, vec->digest, vec->digest_size,
+				impl, "digest");
 out:
 	crypto_free_shash(tfm);
 	return err;
 }

-/*
- * Test the sha256() library function, as it may not be covered by the "sha256"
- * crypto_shash, and thus may not be covered by the "hmac(sha256)" test we do.
- */
 static int __init __must_check
-fips_test_sha256_library(const struct fips_test *test)
+fips_test_sha256_library(const struct fips_test *test, const char *impl)
 {
 	const struct hash_testvec *vec = &test->hash;
 	u8 digest[SHA256_DIGEST_SIZE];
@@ -509,13 +459,13 @@ fips_test_sha256_library(const struct fips_test *test)
 		return -EINVAL;

 	sha256(vec->message, vec->message_size, digest);
-	return fips_check_result(test, digest, vec->digest, vec->digest_size,
-				 "digest (library API)");
+	return fips_check_result(digest, vec->digest, vec->digest_size,
+				 impl, "digest");
 }

 /* Test a DRBG using the crypto_rng API. */
 static int __init __must_check
-fips_test_drbg(const struct fips_test *test)
+fips_test_drbg(const struct fips_test *test, const char *impl)
 {
 	const struct drbg_testvec *vec = &test->drbg;
 	struct crypto_rng *rng;
@@ -524,12 +474,9 @@ fips_test_drbg(const struct fips_test *test)
 	struct drbg_string addtl, pers, testentropy;
 	int err;

-	rng = crypto_alloc_rng(test->alg, 0, 0);
-	if (IS_ERR(rng)) {
-		err = PTR_ERR(rng);
-		pr_err("failed to allocate %s tfm: %d\n", test->alg, err);
-		return PTR_ERR(rng);
-	}
+	rng = crypto_alloc_rng(impl, 0, 0);
+	if (IS_ERR(rng))
+		return fips_handle_alloc_tfm_error(impl, PTR_ERR(rng));
 	err = fips_validate_alg(&crypto_rng_alg(rng)->base);
 	if (err)
 		goto out;
@@ -549,7 +496,7 @@ fips_test_drbg(const struct fips_test *test)
 	drbg_string_fill(&pers, vec->pers, vec->pers_size);
 	err = crypto_drbg_reset_test(rng, &pers, &test_data);
 	if (err) {
-		pr_err("failed to reset %s\n", test->alg);
+		pr_err("failed to reset %s\n", impl);
 		goto out;
 	}

@@ -570,7 +517,7 @@ fips_test_drbg(const struct fips_test *test)
 	}
 	if (err) {
 		pr_err("failed to get bytes from %s (try 1): %d\n",
-		       test->alg, err);
+		       impl, err);
 		goto out;
 	}

@@ -590,13 +537,13 @@ fips_test_drbg(const struct fips_test *test)
 	}
 	if (err) {
 		pr_err("failed to get bytes from %s (try 2): %d\n",
-		       test->alg, err);
+		       impl, err);
 		goto out;
 	}

 	/* Check that the DRBG generated the expected output. */
-	err = fips_check_result(test, output, vec->output, vec->out_size,
-				"get_bytes");
+	err = fips_check_result(output, vec->output, vec->out_size,
+				impl, "get_bytes");
 out:
 	kfree(output);
 	crypto_free_rng(rng);
@@ -606,33 +553,144 @@ out:
 /* Include the test vectors generated by the Python script. */
 #include "fips140-generated-testvecs.h"

-/* List of all self-tests.  Keep this in sync with fips140_algorithms[]. */
+/*
+ * List of all self-tests.  Keep this in sync with fips140_algorithms[].
+ *
+ * When possible, we have followed the FIPS 140-2 Implementation Guidance (IG)
+ * document when creating this list of tests.  The result is intended to be a
+ * list of tests that is near-minimal (and thus minimizes runtime overhead)
+ * while complying with all requirements.  For additional details, see the
+ * comment at the beginning of this file.
+ */
 static const struct fips_test fips140_selftests[] __initconst = {
 	/*
-	 * Tests for AES and AES modes.
+	 * Test for the AES library API.
 	 *
-	 * The full list of AES algorithms we potentially need to test are AES
-	 * by itself, AES-CBC, AES-CTR, AES-ECB, AES-GCM, and AES-XTS.  We can
-	 * follow the FIPS 140-2 Implementation Guidance (IG) document to try to
-	 * reduce this list, but we run into the issue that the architecture-
-	 * specific implementations of these algorithms in Linux often don't
-	 * share the "same" underlying AES implementation.  E.g., the ARMv8 CE
-	 * optimized implementations issue ARMv8 CE instructions directly rather
-	 * than going through a separate AES implementation.  In this case,
-	 * separate tests are needed according to section 9.2 of the IG.
+	 * Since the AES library API may use its own AES implementation and the
+	 * module provides no support for composing it with a mode of operation
+	 * (it's just plain AES), we must test it directly.
+	 *
+	 * In contrast, we don't need to directly test the "aes" ciphers that
+	 * are accessible through the crypto_cipher API (e.g. "aes-ce"), as they
+	 * are covered indirectly by AES-CMAC and AES-ECB tests.
 	 */
 	{
 		.alg		= "aes",
-		.func		= fips_test_aes,
-		.blockcipher	= {
+		.impls		= {"aes-lib"},
+		.func		= fips_test_aes_library,
+		.skcipher	= {
 			.key		= fips_aes_key,
 			.key_size	= sizeof(fips_aes_key),
 			.plaintext	= fips_message,
 			.ciphertext	= fips_aes_ecb_ciphertext,
-			.block_size	= 16,
+			.message_size	= 16,
 		}
-	}, {
+	},
+	/*
+	 * Tests for AES-CMAC, a.k.a. "cmac(aes)" in crypto API syntax.
+	 *
+	 * The IG requires that each underlying AES implementation be tested in
+	 * an authenticated mode, if implemented.  Of such modes, this module
+	 * implements AES-GCM and AES-CMAC.  However, AES-GCM doesn't "count"
+	 * because this module's implementations of AES-GCM won't actually be
+	 * FIPS-approved, due to a quirk in the FIPS requirements.
+	 *
+	 * Therefore, for us this requirement applies to AES-CMAC, so we must
+	 * test the "cmac" template composed with each "aes" implementation.
+	 *
+	 * Separately from the above, we also must test all standalone
+	 * implementations of "cmac(aes)" such as "cmac-aes-ce", as they don't
+	 * reuse another full AES implementation and thus can't be covered by
+	 * another test.
+	 */
+	{
+		.alg		= "cmac(aes)",
+		.impls		= {
+			/* "cmac" template with all "aes" implementations */
+			"cmac(aes-generic)",
+			"cmac(aes-arm64)",
+			"cmac(aes-ce)",
+			/* All standalone implementations of "cmac(aes)" */
+			"cmac-aes-neon",
+			"cmac-aes-ce",
+		},
+		.func		= fips_test_hash,
+		.hash		= {
+			.key		= fips_aes_key,
+			.key_size	= sizeof(fips_aes_key),
+			.message	= fips_message,
+			.message_size	= sizeof(fips_message),
+			.digest		= fips_aes_cmac_digest,
+			.digest_size	= sizeof(fips_aes_cmac_digest),
+		}
+	},
+	/*
+	 * Tests for AES-ECB, a.k.a. "ecb(aes)" in crypto API syntax.
+	 *
+	 * The IG requires that each underlying AES implementation be tested in
+	 * a mode that exercises the encryption direction of AES and in a mode
+	 * that exercises the decryption direction of AES.  CMAC only covers the
+	 * encryption direction, so we choose ECB to test decryption.  Thus, we
+	 * test the "ecb" template composed with each "aes" implementation.
+	 *
+	 * Separately from the above, we also must test all standalone
+	 * implementations of "ecb(aes)" such as "ecb-aes-ce", as they don't
+	 * reuse another full AES implementation and thus can't be covered by
+	 * another test.
+	 */
+	{
+		.alg		= "ecb(aes)",
+		.impls		= {
+			/* "ecb" template with all "aes" implementations */
+			"ecb(aes-generic)",
+			"ecb(aes-arm64)",
+			"ecb(aes-ce)",
+			/* All standalone implementations of "ecb(aes)" */
+			"ecb-aes-neon",
+			"ecb-aes-neonbs",
+			"ecb-aes-ce",
+		},
+		.func		= fips_test_skcipher,
+		.skcipher	= {
+			.key		= fips_aes_key,
+			.key_size	= sizeof(fips_aes_key),
+			.plaintext	= fips_message,
+			.ciphertext	= fips_aes_ecb_ciphertext,
+			.message_size	= sizeof(fips_message)
+		}
+	},
+	/*
+	 * Tests for AES-CBC, AES-CBC-CTS, AES-CTR, AES-XTS, and AES-GCM.
+	 *
+	 * According to the IG, an AES mode of operation doesn't need to have
+	 * its own test, provided that (a) both the encryption and decryption
+	 * directions of the underlying AES implementation are already tested
+	 * via other mode(s), and (b) in the case of an authenticated mode, at
+	 * least one other authenticated mode is already tested.  The tests of
+	 * the "cmac" and "ecb" templates fulfill these conditions; therefore,
+	 * we don't need to test any other AES mode templates.
+	 *
+	 * This does *not* apply to standalone implementations of these modes
+	 * such as "cbc-aes-ce", as such implementations don't reuse another
+	 * full AES implementation and thus can't be covered by another test.
+	 * We must test all such standalone implementations.
+	 *
+	 * The AES-GCM test isn't actually required, as it's expected that this
+	 * module's AES-GCM implementation won't actually be able to be
+	 * FIPS-approved.  This is unfortunate; it's caused by the FIPS
+	 * requirements for GCM being incompatible with GCM implementations that
+	 * don't generate their own IVs.  We choose to still include the AES-GCM
+	 * test to keep it on par with the other FIPS-approved algorithms, in
+	 * case it turns out that AES-GCM can be approved after all.
+	 */
+	{
 		.alg		= "cbc(aes)",
+		.impls		= {
+			/* All standalone implementations of "cbc(aes)" */
+			"cbc-aes-neon",
+			"cbc-aes-neonbs",
+			"cbc-aes-ce",
+		},
 		.func		= fips_test_skcipher,
 		.skcipher	= {
 			.key		= fips_aes_key,
@@ -643,8 +701,40 @@ static const struct fips_test fips140_selftests[] __initconst = {
 			.ciphertext	= fips_aes_cbc_ciphertext,
 			.message_size	= sizeof(fips_message),
 		}
+	}, {
+		.alg		= "cts(cbc(aes))",
+		.impls		= {
+			/* All standalone implementations of "cts(cbc(aes))" */
+			"cts-cbc-aes-neon",
+			"cts-cbc-aes-ce",
+		},
+		.func		= fips_test_skcipher,
+		/* Test vector taken from RFC 3962 */
+		.skcipher	= {
+			.key    = "\x63\x68\x69\x63\x6b\x65\x6e\x20"
+				  "\x74\x65\x72\x69\x79\x61\x6b\x69",
+			.key_size = 16,
+			.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+				  "\x00\x00\x00\x00\x00\x00\x00\x00",
+			.iv_size = 16,
+			.plaintext = "\x49\x20\x77\x6f\x75\x6c\x64\x20"
+				     "\x6c\x69\x6b\x65\x20\x74\x68\x65"
+				     "\x20\x47\x65\x6e\x65\x72\x61\x6c"
+				     "\x20\x47\x61\x75\x27\x73\x20",
+			.ciphertext = "\xfc\x00\x78\x3e\x0e\xfd\xb2\xc1"
+				      "\xd4\x45\xd4\xc8\xef\xf7\xed\x22"
+				      "\x97\x68\x72\x68\xd6\xec\xcc\xc0"
+				      "\xc0\x7b\x25\xe2\x5e\xcf\xe5",
+			.message_size = 31,
+		}
 	}, {
 		.alg		= "ctr(aes)",
+		.impls		= {
+			/* All standalone implementations of "ctr(aes)" */
+			"ctr-aes-neon",
+			"ctr-aes-neonbs",
+			"ctr-aes-ce",
+		},
 		.func		= fips_test_skcipher,
 		.skcipher	= {
 			.key		= fips_aes_key,
@@ -655,34 +745,14 @@ static const struct fips_test fips140_selftests[] __initconst = {
 			.ciphertext	= fips_aes_ctr_ciphertext,
 			.message_size	= sizeof(fips_message),
 		}
-	}, {
-		.alg		= "ecb(aes)",
-		.func		= fips_test_skcipher,
-		.skcipher	= {
-			.key		= fips_aes_key,
-			.key_size	= sizeof(fips_aes_key),
-			.plaintext	= fips_message,
-			.ciphertext	= fips_aes_ecb_ciphertext,
-			.message_size	= sizeof(fips_message)
-		}
-	}, {
-		.alg		= "gcm(aes)",
-		.func		= fips_test_aead,
-		.aead		= {
-			.key		= fips_aes_key,
-			.key_size	= sizeof(fips_aes_key),
-			.iv		= fips_aes_iv,
-			/* The GCM implementation assumes an IV size of 12. */
-			.iv_size	= 12,
-			.assoc		= fips_aes_gcm_assoc,
-			.assoc_size	= sizeof(fips_aes_gcm_assoc),
-			.plaintext	= fips_message,
-			.plaintext_size	= sizeof(fips_message),
-			.ciphertext	= fips_aes_gcm_ciphertext,
-			.ciphertext_size = sizeof(fips_aes_gcm_ciphertext),
-		}
 	}, {
 		.alg		= "xts(aes)",
+		.impls		= {
+			/* All standalone implementations of "xts(aes)" */
+			"xts-aes-neon",
+			"xts-aes-neonbs",
+			"xts-aes-ce",
+		},
 		.func		= fips_test_skcipher,
 		.skcipher	= {
 			.key		= fips_aes_xts_key,
@@ -693,27 +763,36 @@ static const struct fips_test fips140_selftests[] __initconst = {
 			.ciphertext	= fips_aes_xts_ciphertext,
 			.message_size	= sizeof(fips_message),
 		}
-	/*
-	 * Tests for SHA-1, SHA-256, HMAC-SHA256, and SHA-512.
-	 *
-	 * The selection of these specific tests follows the guidance from
-	 * section 9 of the FIPS 140-2 Implementation Guidance (IG) document to
-	 * achieve a minimal list of tests, rather than testing all of
-	 * SHA-{1,224,256,384,512} and HMAC-SHA{1,224,256,384,512}.  As per the
-	 * IG, testing SHA-224 is only required if SHA-256 isn't implemented,
-	 * and testing SHA-384 is only required if SHA-512 isn't implemented.
-	 * Also, HMAC only has to be tested with one underlying SHA, and the
-	 * HMAC test also fulfills the test for its underlying SHA.  That would
-	 * result in a test list of e.g. SHA-1, HMAC-SHA256, and SHA-512.
-	 *
-	 * However we also need to take into account cases where implementations
-	 * aren't shared in the "natural" way assumed by the IG.  Currently the
-	 * only known exception w.r.t. SHA-* and HMAC-* is the sha256() library
-	 * function which may not be covered by the test of the "hmac(sha256)"
-	 * crypto_shash.  So, we test sha256() separately.
-	 */
 	}, {
+		.alg		= "gcm(aes)",
+		.impls		= {
+			/* All standalone implementations of "gcm(aes)" */
+			"gcm-aes-ce",
+		},
+		.func		= fips_test_aead,
+		.aead		= {
+			.key		= fips_aes_key,
+			.key_size	= sizeof(fips_aes_key),
+			.iv		= fips_aes_iv,
+			/* The GCM implementations assume an IV size of 12. */
+			.iv_size	= 12,
+			.assoc		= fips_aes_gcm_assoc,
+			.assoc_size	= sizeof(fips_aes_gcm_assoc),
+			.plaintext	= fips_message,
+			.plaintext_size	= sizeof(fips_message),
+			.ciphertext	= fips_aes_gcm_ciphertext,
+			.ciphertext_size = sizeof(fips_aes_gcm_ciphertext),
+		}
+	},
+
+	/* Tests for SHA-1 */
+	{
 		.alg		= "sha1",
+		.impls		= {
+			/* All implementations of "sha1" */
+			"sha1-generic",
+			"sha1-ce"
+		},
 		.func		= fips_test_hash,
 		.hash		= {
 			.message	= fips_message,
@@ -721,8 +800,35 @@ static const struct fips_test fips140_selftests[] __initconst = {
 			.digest		= fips_sha1_digest,
 			.digest_size	= sizeof(fips_sha1_digest)
 		}
-	}, {
+	},
+	/*
+	 * Tests for all SHA-256 implementations other than the sha256() library
+	 * function.  As per the IG, these tests also fulfill the tests for the
+	 * corresponding SHA-224 implementations.
+	 */
+	{
 		.alg		= "sha256",
+		.impls		= {
+			/* All implementations of "sha256" */
+			"sha256-generic",
+			"sha256-arm64",
+			"sha256-ce",
+		},
+		.func		= fips_test_hash,
+		.hash		= {
+			.message	= fips_message,
+			.message_size	= sizeof(fips_message),
+			.digest		= fips_sha256_digest,
+			.digest_size	= sizeof(fips_sha256_digest)
+		}
+	},
+	/*
+	 * Test for the sha256() library function.  This must be tested
+	 * separately because it may use its own SHA-256 implementation.
+	 */
+	{
+		.alg		= "sha256",
+		.impls		= {"sha256-lib"},
 		.func		= fips_test_sha256_library,
 		.hash		= {
 			.message	= fips_message,
@@ -730,7 +836,36 @@ static const struct fips_test fips140_selftests[] __initconst = {
 			.digest		= fips_sha256_digest,
 			.digest_size	= sizeof(fips_sha256_digest)
 		}
-	}, {
+	},
+	/*
+	 * Tests for all SHA-512 implementations.  As per the IG, these tests
+	 * also fulfill the tests for the corresponding SHA-384 implementations.
+	 */
+	{
+		.alg		= "sha512",
+		.impls		= {
+			/* All implementations of "sha512" */
+			"sha512-generic",
+			"sha512-arm64",
+			"sha512-ce",
+		},
+		.func		= fips_test_hash,
+		.hash		= {
+			.message	= fips_message,
+			.message_size	= sizeof(fips_message),
+			.digest		= fips_sha512_digest,
+			.digest_size	= sizeof(fips_sha512_digest)
+		}
+	},
+	/*
+	 * Test for HMAC.  As per the IG, only one HMAC test is required,
+	 * provided that the same HMAC code is shared by all HMAC-SHA*.  This is
+	 * true in our case.  We choose HMAC-SHA256 for the test.
+	 *
+	 * Note that as per the IG, this can fulfill the test for the underlying
+	 * SHA.  However, we don't currently rely on this.
+	 */
+	{
 		.alg		= "hmac(sha256)",
 		.func		= fips_test_hash,
 		.hash		= {
@@ -741,61 +876,36 @@ static const struct fips_test fips140_selftests[] __initconst = {
 			.digest		= fips_hmac_sha256_digest,
 			.digest_size	= sizeof(fips_hmac_sha256_digest)
 		}
-	}, {
-		.alg		= "sha512",
-		.func		= fips_test_hash,
-		.hash		= {
-			.message	= fips_message,
-			.message_size	= sizeof(fips_message),
-			.digest		= fips_sha512_digest,
-			.digest_size	= sizeof(fips_sha512_digest)
-		}
+	},
 	/*
-	 * Tests for DRBG algorithms.
+	 * Known-answer tests for the SP800-90A DRBG algorithms.
 	 *
-	 * Only the default variant (the one that users get when they request
-	 * "stdrng") is required to be tested, as we don't consider the other
-	 * variants to be used / usable in the FIPS security policy.  This is
-	 * similar to how e.g. we don't test both "xts(aes-generic)" and
-	 * "xts-aes-ce" but rather just "xts(aes)".
+	 * These test vectors were manually extracted from
+	 * https://csrc.nist.gov/CSRC/media/Projects/Cryptographic-Algorithm-Validation-Program/documents/drbg/drbgtestvectors.zip.
 	 *
-	 * Currently the default one is "drbg_nopr_hmac_sha256"; however, just
-	 * in case we also test the prediction-resistant enabled variant too.
+	 * The selection of these tests follows the FIPS 140-2 IG as well as
+	 * Section 11 of SP800-90A:
+	 *
+	 * - We must test all DRBG types (HMAC, Hash, and CTR) that the module
+	 *   implements.  However, currently the module only implements
+	 *   HMAC_DRBG (since CONFIG_CRYPTO_DRBG_CTR and CONFIG_CRYPTO_DRBG_HASH
+	 *   aren't enabled).  Therefore, we only need to test HMAC_DRBG.
+	 *
+	 * - We only need to test one HMAC variant.
+	 *
+	 * - We must test all DRBG operations: Instantiate(), Reseed(), and
+	 *   Generate().  However, a single test sequence with a single output
+	 *   comparison may cover all three operations, and this is what we do.
+	 *   Note that Reseed() happens implicitly via the use of the additional
+	 *   input and also via the use of prediction resistance when enabled.
+	 *
+	 * - The personalization string, additional input, and prediction
+	 *   resistance support must be tested.  Therefore we have chosen test
+	 *   vectors that have a nonempty personalization string and nonempty
+	 *   additional input, and we test the prediction-resistant variant.
+	 *   Testing the non-prediction-resistant variant is not required.
 	 */
-	}, {
-		.alg	= "drbg_nopr_hmac_sha256",
-		.func	= fips_test_drbg,
-		.drbg	= {
-			.entropy =
-				"\xf9\x7a\x3c\xfd\x91\xfa\xa0\x46\xb9\xe6\x1b\x94"
-				"\x93\xd4\x36\xc4\x93\x1f\x60\x4b\x22\xf1\x08\x15"
-				"\x21\xb3\x41\x91\x51\xe8\xff\x06\x11\xf3\xa7\xd4"
-				"\x35\x95\x35\x7d\x58\x12\x0b\xd1\xe2\xdd\x8a\xed",
-			.entropy_size = 48,
-			.output =
-				"\xc6\x87\x1c\xff\x08\x24\xfe\x55\xea\x76\x89\xa5"
-				"\x22\x29\x88\x67\x30\x45\x0e\x5d\x36\x2d\xa5\xbf"
-				"\x59\x0d\xcf\x9a\xcd\x67\xfe\xd4\xcb\x32\x10\x7d"
-				"\xf5\xd0\x39\x69\xa6\x6b\x1f\x64\x94\xfd\xf5\xd6"
-				"\x3d\x5b\x4d\x0d\x34\xea\x73\x99\xa0\x7d\x01\x16"
-				"\x12\x6d\x0d\x51\x8c\x7c\x55\xba\x46\xe1\x2f\x62"
-				"\xef\xc8\xfe\x28\xa5\x1c\x9d\x42\x8e\x6d\x37\x1d"
-				"\x73\x97\xab\x31\x9f\xc7\x3d\xed\x47\x22\xe5\xb4"
-				"\xf3\x00\x04\x03\x2a\x61\x28\xdf\x5e\x74\x97\xec"
-				"\xf8\x2c\xa7\xb0\xa5\x0e\x86\x7e\xf6\x72\x8a\x4f"
-				"\x50\x9a\x8c\x85\x90\x87\x03\x9c",
-			.out_size = 128,
-			.add_a =
-				"\x51\x72\x89\xaf\xe4\x44\xa0\xfe\x5e\xd1\xa4\x1d"
-				"\xbb\xb5\xeb\x17\x15\x00\x79\xbd\xd3\x1e\x29\xcf"
-				"\x2f\xf3\x00\x34\xd8\x26\x8e\x3b",
-			.add_b =
-				"\x88\x02\x8d\x29\xef\x80\xb4\xe6\xf0\xfe\x12\xf9"
-				"\x1d\x74\x49\xfe\x75\x06\x26\x82\xe8\x9c\x57\x14"
-				"\x40\xc0\xc9\xb5\x2c\x42\xa6\xe0",
-			.add_size = 32,
-		}
-	}, {
+	{
 		.alg	= "drbg_pr_hmac_sha256",
 		.func	= fips_test_drbg,
 		.drbg	= {
@@ -845,19 +955,44 @@ static const struct fips_test fips140_selftests[] __initconst = {
 	}
 };

+static int __init __must_check
+fips_run_test(const struct fips_test *test)
+{
+	int i;
+	int err;
+
+	/*
+	 * If no implementations were specified, then just test the default one.
+	 * Otherwise, test the specified list of implementations.
+	 */
+
+	if (test->impls[0] == NULL) {
+		err = test->func(test, test->alg);
+		if (err)
+			pr_emerg("self-tests failed for algorithm %s: %d\n",
+				 test->alg, err);
+		return err;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(test->impls) && test->impls[i] != NULL;
+	     i++) {
+		err = test->func(test, test->impls[i]);
+		if (err) {
+			pr_emerg("self-tests failed for algorithm %s, implementation %s: %d\n",
+				 test->alg, test->impls[i], err);
+			return err;
+		}
+	}
+	return 0;
+}
+
 bool __init fips140_run_selftests(void)
 {
 	int i;

 	pr_info("running self-tests\n");
 	for (i = 0; i < ARRAY_SIZE(fips140_selftests); i++) {
-		const struct fips_test *test = &fips140_selftests[i];
-		int err;
-
-		err = test->func(test);
-		if (err) {
-			pr_emerg("self-tests failed for algorithm %s: %d\n",
-				 test->alg, err);
+		if (fips_run_test(&fips140_selftests[i]) != 0) {
 			/* The caller is responsible for calling panic(). */
 			return false;
 		}
--- a/crypto/fips140_gen_hmac.c
+++ b/crypto/fips140_gen_hmac.c
@@ -28,7 +28,7 @@
 static Elf64_Ehdr *ehdr;
 static Elf64_Shdr *shdr;
 static int num_shdr;
-static const char *strtab;
+static const char *strtab, *shstrtab;
 static Elf64_Sym *syms;
 static int num_syms;

@@ -42,17 +42,78 @@ static Elf64_Shdr *find_symtab_section(void)
 	return NULL;
 }

-static void *get_sym_addr(const char *sym_name)
+static int get_section_idx(const char *name)
+{
+	int i;
+
+	for (i = 0; i < num_shdr; i++)
+		if (!strcmp(shstrtab + shdr[i].sh_name, name))
+			return i;
+	return -1;
+}
+
+static int get_sym_idx(const char *sym_name)
 {
 	int i;

 	for (i = 0; i < num_syms; i++)
 		if (!strcmp(strtab + syms[i].st_name, sym_name))
-			return (void *)ehdr + shdr[syms[i].st_shndx].sh_offset +
-			       syms[i].st_value;
+			return i;
+	return -1;
+}
+
+static void *get_sym_addr(const char *sym_name)
+{
+	int i = get_sym_idx(sym_name);
+
+	if (i >= 0)
+		return (void *)ehdr + shdr[syms[i].st_shndx].sh_offset +
+		       syms[i].st_value;
 	return NULL;
 }

+static int update_rela_ref(const char *name)
+{
+	/*
+	 * We need to do a couple of things to ensure that the copied RELA data
+	 * is accessible to the module itself at module init time:
+	 * - the associated entry in the symbol table needs to refer to the
+	 *   correct section index, and have SECTION type and GLOBAL linkage.
+	 * - the 'count' global variable in the module need to be set to the
+	 *   right value based on the size of the RELA section.
+	 */
+	unsigned int *size_var;
+	int sec_idx, sym_idx;
+	char str[32];
+
+	sprintf(str, "fips140_rela_%s", name);
+	size_var = get_sym_addr(str);
+	if (!size_var) {
+		printf("variable '%s' not found, disregarding .%s section\n",
+		       str, name);
+		return 1;
+	}
+
+	sprintf(str, "__sec_rela_%s", name);
+	sym_idx = get_sym_idx(str);
+
+	sprintf(str, ".init.rela.%s", name);
+	sec_idx = get_section_idx(str);
+
+	if (sec_idx < 0 || sym_idx < 0) {
+		fprintf(stderr, "failed to locate metadata for .%s section in binary\n",
+			name);
+		return 0;
+	}
+
+	syms[sym_idx].st_shndx = sec_idx;
+	syms[sym_idx].st_info = (STB_GLOBAL << 4) | STT_SECTION;
+
+	size_var[1] = shdr[sec_idx].sh_size / sizeof(Elf64_Rela);
+
+	return 1;
+}
+
 static void hmac_section(HMAC_CTX *hmac, const char *start, const char *end)
 {
 	void *start_addr = get_sym_addr(start);
@@ -103,6 +164,10 @@ int main(int argc, char **argv)
 	num_syms = symtab_shdr->sh_size / sizeof(Elf64_Sym);

 	strtab = (void *)ehdr + shdr[symtab_shdr->sh_link].sh_offset;
+	shstrtab = (void *)ehdr + shdr[ehdr->e_shstrndx].sh_offset;
+
+	if (!update_rela_ref("text") || !update_rela_ref("rodata"))
+		exit(EXIT_FAILURE);

 	hmac_key = get_sym_addr("fips140_integ_hmac_key");
 	if (!hmac_key) {
--- a/drivers/android/vendor_hooks.c
+++ b/drivers/android/vendor_hooks.c
@@ -374,6 +374,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_usb_dev_resume);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ipv6_gen_linklocal_addr);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sound_usb_support_cpu_suspend);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_snd_compr_use_pause_in_drain);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_snd_soc_card_get_comp_chain);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_show_max_freq);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_tcp_sendmsg_locked);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_tcp_recvmsg);
--- a/fs/incfs/vfs.c
+++ b/fs/incfs/vfs.c
@@ -458,8 +458,10 @@ static struct dentry *open_or_create_special_dir(struct dentry *backing_dir,
 	err = vfs_mkdir(backing_inode, index_dentry, 0777);
 	inode_unlock(backing_inode);

-	if (err)
+	if (err) {
+		dput(index_dentry);
 		return ERR_PTR(err);
+	}

 	if (!d_really_is_positive(index_dentry) ||
 		unlikely(d_unhashed(index_dentry))) {
--- a/include/trace/hooks/sound.h
+++ b/include/trace/hooks/sound.h
@@ -15,6 +15,10 @@ DECLARE_HOOK(android_vh_sound_usb_support_cpu_suspend,
 		bool *is_support),
 	TP_ARGS(udev, direction, is_support));

+DECLARE_HOOK(android_vh_snd_soc_card_get_comp_chain,
+	TP_PROTO(bool *component_chaining),
+	TP_ARGS(component_chaining));
+
 #endif /* _TRACE_HOOK_SOUND_H */
 /* This part must be outside protection */
 #include <trace/define_trace.h>
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -3159,7 +3159,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
 	 */
 	if (!pte_map_lock(vmf)) {
 		ret = VM_FAULT_RETRY;
-		goto out_free_new;
+		goto out_invalidate_end;
 	}
 	if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
 		if (old_page) {
@@ -3247,6 +3247,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
 		put_page(old_page);
 	}
 	return page_copied ? VM_FAULT_WRITE : 0;
+out_invalidate_end:
+	mmu_notifier_invalidate_range_only_end(&range);
 out_free_new:
 	put_page(new_page);
 out:
--- a/scripts/module.lds.S
+++ b/scripts/module.lds.S
@@ -56,14 +56,16 @@ SECTIONS {
 		*(.rodata.._end)
 	}

-#ifdef CONFIG_CFI_CLANG
-	/*
-	 * With CFI_CLANG, ensure __cfi_check is at the beginning of the
-	 * .text section, and that the section is aligned to page size.
-	 */
 	.text : ALIGN(PAGE_SIZE) {
 		*(.text.._start)
+#ifdef CONFIG_CFI_CLANG
+		/*
+		 * With CFI_CLANG, ensure __cfi_check is at the beginning of
+		 * the .text section, and that the section is aligned to page
+		 * size.
+		 */
 		*(.text.__cfi_check)
+#endif
 		*(.text .text.[0-9a-zA-Z_]*)
 		__cfi_jt_start = .;
 		*(.text..L.cfi.jumptable .text..L.cfi.jumptable.*)
@@ -71,7 +73,6 @@ SECTIONS {
 		*(.text.._end)
 	}
 #endif
-#endif
 }

 /* bring in arch-specific sections */
--- a/sound/soc/soc-pcm.c
+++ b/sound/soc/soc-pcm.c
@@ -26,6 +26,7 @@
 #include <sound/soc-dpcm.h>
 #include <sound/soc-link.h>
 #include <sound/initval.h>
+#include <trace/hooks/sound.h>

 #define DPCM_MAX_BE_USERS	8

@@ -1275,6 +1276,7 @@ int dpcm_path_get(struct snd_soc_pcm_runtime *fe,
 {
 	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(fe, 0);
 	int paths;
+	bool chaining = false;

 	if (fe->num_cpus > 1) {
 		dev_err(fe->dev,
@@ -1282,9 +1284,11 @@ int dpcm_path_get(struct snd_soc_pcm_runtime *fe,
 		return -EINVAL;
 	}

+	trace_android_vh_snd_soc_card_get_comp_chain(&chaining);
+
 	/* get number of valid DAI paths and their widgets */
 	paths = snd_soc_dapm_dai_get_connected_widgets(cpu_dai, stream, list,
-			dpcm_end_walk_at_be);
+			chaining ? NULL : dpcm_end_walk_at_be);

 	dev_dbg(fe->dev, "ASoC: found %d audio %s paths\n", paths,
 			stream ? "capture" : "playback");
--- a/tools/crypto/gen_fips140_testvecs.py
+++ b/tools/crypto/gen_fips140_testvecs.py
@@ -28,7 +28,7 @@ scriptname = os.path.basename(__file__)
 message     = bytes('This is a 32-byte test message.\0', 'ascii')
 aes_key     = bytes('128-bit AES key\0', 'ascii')
 aes_xts_key = bytes('This is an AES-128-XTS key.\0\0\0\0\0', 'ascii')
-aes_iv      = bytes('ABCDEFGHIJKL\0\0\0\0', 'ascii')
+aes_iv      = bytes('ABCDEFGHIJKLMNOP', 'ascii')
 assoc       = bytes('associated data string', 'ascii')
 hmac_key    = bytes('128-bit HMAC key', 'ascii')

@@ -82,7 +82,7 @@ def generate_aes_testvecs():
    print_value('aes_ecb_ciphertext', ecb.encrypt(message))

    ctr = Cryptodome.Cipher.AES.new(aes_key, Cryptodome.Cipher.AES.MODE_CTR,
-                                    nonce=aes_iv[:12])
+                                    nonce=bytes(), initial_value=aes_iv)
    print_value('aes_ctr_ciphertext', ctr.encrypt(message))

    print_value('aes_gcm_assoc', assoc)
@@ -101,6 +101,10 @@ def generate_aes_testvecs():
    ciphertext = xts.update(message) + xts.finalize()
    print_value('aes_xts_ciphertext', ciphertext)

+    cmac = Cryptodome.Hash.CMAC.new(aes_key, ciphermod=Cryptodome.Cipher.AES)
+    cmac.update(message)
+    print_value('aes_cmac_digest', cmac.digest())
+
 def generate_sha_testvecs():
    print_value('hmac_key', hmac_key)
    for alg in ['sha1', 'sha256', 'hmac_sha256', 'sha512']: