diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index cbeac1bebe2f..ddd4bd6116fc 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -409,3 +409,27 @@ Description: Give a way to change checkpoint merge daemon's io priority. I/O priority "3". We can select the class between "rt" and "be", and set the I/O priority within valid range of it. "," delimiter is necessary in between I/O class and priority number. + +What: /sys/fs/f2fs//compr_written_block +Date: March 2021 +Contact: "Daeho Jeong" +Description: Show the block count written after compression since mount. Note + that when the compressed blocks are deleted, this count doesn't + decrease. If you write "0" here, you can initialize + compr_written_block and compr_saved_block to "0". + +What: /sys/fs/f2fs//compr_saved_block +Date: March 2021 +Contact: "Daeho Jeong" +Description: Show the saved block count with compression since mount. Note + that when the compressed blocks are deleted, this count doesn't + decrease. If you write "0" here, you can initialize + compr_written_block and compr_saved_block to "0". + +What: /sys/fs/f2fs//compr_new_inode +Date: March 2021 +Contact: "Daeho Jeong" +Description: Show the count of inode newly enabled for compression since mount. + Note that when the compression is disabled for the files, this count + doesn't decrease. If you write "0" here, you can initialize + compr_new_inode to "0". diff --git a/MAINTAINERS b/MAINTAINERS index 0b41639836c5..0b53ad960990 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9757,6 +9757,18 @@ F: include/linux/keyctl.h F: include/uapi/linux/keyctl.h F: security/keys/ +KFENCE +M: Alexander Potapenko +M: Marco Elver +R: Dmitry Vyukov +L: kasan-dev@googlegroups.com +S: Maintained +F: Documentation/dev-tools/kfence.rst +F: arch/*/include/asm/kfence.h +F: include/linux/kfence.h +F: lib/Kconfig.kfence +F: mm/kfence/ + KFIFO M: Stefani Seibold S: Maintained diff --git a/Makefile b/Makefile index 7129824bd948..0564cc54d3e2 100644 --- a/Makefile +++ b/Makefile @@ -906,7 +906,13 @@ KBUILD_LDFLAGS += --thinlto-cache-dir=$(extmod-prefix).thinlto-cache else CC_FLAGS_LTO += -flto endif + +ifeq ($(SRCARCH),x86) +# TODO(b/182572011): Revert workaround for compiler / linker bug. CC_FLAGS_LTO += -fvisibility=hidden +else +CC_FLAGS_LTO += -fvisibility=default +endif # Limit inlining across translation units to reduce binary size KBUILD_LDFLAGS += -mllvm -import-instr-limit=5 diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index a06d92d09c84..e239c5ae9505 100644 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -1,14 +1,15 @@ - - - + + + + - - + + - + @@ -18,53 +19,53 @@ - - - - - - - + + + + + + + - - - - + + + + - - + + - - + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + + + + + @@ -73,132 +74,139 @@ - + + + - - + - - - - - - - + + + + + + + + - - + - + - + - - + + - - - - + + + + - - + + - - - - + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - + + + - - - - + + + + - + - - - - - - + + + + + + - - - - - - - - + + + + + + + + + + - - + + + + - - + + - + - + - - + + @@ -206,14 +214,14 @@ - - - - - - - - + + + + + + + + @@ -227,6 +235,7 @@ + @@ -239,34 +248,38 @@ - - - - - + + + + + + - - + + - - + + + + + - - - - - + + + + + - - - - - - + + + + + + @@ -275,281 +288,287 @@ - - - - - - - - - - + + + + + + + + + + - - - - - - - - + + + + + + + + + - - - - - - - - - - + + + + + + + + + + + + - - - - + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - - - + + + + - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + + - + - - - + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - + - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - + - + - + - - + + - - - - - - - + + + + + + + - - - - - - - - - + + + + + + + + + @@ -557,335 +576,355 @@ - - - - - - + + + + + + - + - + - + + - - + + - + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + @@ -900,48 +939,48 @@ - - - - - - - + + + + + + + - + - - - - - + + + + + - + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + @@ -951,415 +990,406 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - - - - - - + + + + + + + + + + + + - + - - - - + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - + + + + + - - - + + + - - + + - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - + - + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - + + + - - - - + + + + - - - - - - - - - - - + + + + + + + + + + + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - + + + + - - - - + + + + - - + + - + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + @@ -1375,84 +1405,95 @@ - - - + + + - - - - - - - - - - - - + + + + + + + + + + + - + - - + + + + - - - + + + - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - + + + + + + @@ -1461,59 +1502,62 @@ - - - - - - - + + + + + + + + - - + + - - - - - - - - - - + + + + + + + + + + - - - + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + @@ -1527,11 +1571,11 @@ - - + + - - + + @@ -1542,170 +1586,176 @@ - - - + + + - - - - + + + + - - - - + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - + + + + + + - - + + - - - - + + + + - - - - - - - - + + + + + + + + - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - - + + - + @@ -1713,53 +1763,53 @@ - - - - - + + + + + - + - + - - - + + + - + - - - - + + + + - - - - - - + + + + + + - - - - - - - - - - + + + + + + + + + + - + - + @@ -1776,27 +1826,28 @@ - - - - - - - - - - - - + + + + + + + + + + + + + - - + + - - - - + + + + @@ -1811,36 +1862,36 @@ - - - - - - + + + + + + - - + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + @@ -1868,87 +1919,87 @@ - - - - - - - - - - + + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1957,366 +2008,378 @@ - - - - - - + + + + + + - - + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - + - + + - - - - - - - + + + + + + + + + - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - - + + - + @@ -2325,31 +2388,44 @@ - - - + + + - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - + + @@ -2365,6 +2441,7 @@ + @@ -2383,37 +2460,37 @@ - + - - + + - - - - - + + + + + - - + + - + - - + + - + @@ -2421,86 +2498,87 @@ - - + + - - - - + + + + - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - + + - + - - - - + + + + - + - - + + @@ -2511,380 +2589,392 @@ - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - + - + - - - - - - + + + + + + - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + - + - - - + + + - + - - + + + + - - - - - + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - + + + + + - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + + @@ -2895,7 +2985,7 @@ - + @@ -2906,6 +2996,7 @@ + @@ -2914,11 +3005,11 @@ - - - - - + + + + + @@ -2926,191 +3017,202 @@ - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - + + - + - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - + + + + + + + - + + - + - - + + - - + + - - + + - + - + - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + - - + + - + - + - + - - + + - - - - - - - - + + + + + + + + + - + - - - - - + + + + + - - - + + + - - - + + + - + - + @@ -3122,224 +3224,229 @@ - - - - - + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - - + + + + - + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + - + + - + - - + + - - + + - + - - - + + + @@ -3350,39 +3457,41 @@ - - - - - - - - - - - - - + + + + + + + + + + + + + - - - + + + - + - - - - - - + + + + + + + + @@ -3396,7 +3505,9 @@ + + @@ -3406,10 +3517,12 @@ + + @@ -3420,6 +3533,7 @@ + @@ -3432,102 +3546,112 @@ + + + + + + + + + - + - + - - - + + + - + - + - - - + + + - + - + - + - - - - - - - - - - + + + + + + + + + + - + - - - + + + - - + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + @@ -3535,18 +3659,18 @@ - - + + - - + + - - + + @@ -3563,12 +3687,14 @@ - - + + - + + + @@ -3803,73 +3929,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -4054,20 +4180,20 @@ - + - + - + - + - + @@ -4201,42 +4327,54 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -4278,26 +4416,26 @@ - + - + - + - + - + - + - + - + @@ -4400,7 +4538,7 @@ - + @@ -4509,6 +4647,7 @@ + @@ -4562,17 +4701,17 @@ - + - + - + @@ -4596,90 +4735,114 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -4739,6 +4902,7 @@ + @@ -4766,6 +4930,7 @@ + @@ -4853,22 +5018,22 @@ - + - + - + - + - + - + @@ -4903,18 +5068,24 @@ - + - + - + - + - + + + + + + + @@ -4965,16 +5136,15 @@ - - + - + - + - + @@ -5004,34 +5174,34 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -5203,7 +5373,6 @@ - @@ -5237,7 +5406,6 @@ - @@ -5251,15 +5419,15 @@ - + - + - + - + @@ -5327,30 +5495,30 @@ - + - + - + - + - + - + - + - + - + @@ -5453,7 +5621,7 @@ - + @@ -5514,135 +5682,135 @@ - + - - - - - - - - - - - - - + - + - + - + - + - + + + + + + + + + + - + - - + + - - + + - - - - - - - - - - - + + - + - + - + - - - - + - + - + + + + - + - + - - - - + - - - - - - - + - + - + + + + + + + - + - + + + + - + - + - - + + + + + + + + + + + + + + - + - + - + - + - + @@ -5656,42 +5824,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -5791,21 +5965,21 @@ - + - + - + - + - + - + @@ -5872,22 +6046,22 @@ - + - + - + - + - + - + @@ -5905,7 +6079,7 @@ - + @@ -5923,61 +6097,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + - + @@ -6050,112 +6236,112 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -6191,355 +6377,355 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + @@ -6551,7 +6737,7 @@ - + @@ -6585,37 +6771,37 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -6659,7 +6845,7 @@ - + @@ -6892,23 +7078,7 @@ - - - - - - - - - - - - - - - - - + @@ -6996,7 +7166,7 @@ - + @@ -7027,70 +7197,76 @@ - + - + - + - + - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -7098,92 +7274,104 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + + + + - + + + + + + + + + + - + + + + @@ -7246,7 +7434,7 @@ - + @@ -7289,10 +7477,10 @@ - + - + @@ -7524,12 +7712,12 @@ - - - + + + @@ -7582,7 +7770,7 @@ - + @@ -7615,8 +7803,6 @@ - - @@ -7626,7 +7812,7 @@ - + @@ -7663,25 +7849,25 @@ - + - + - + - + - + - + - + @@ -7997,75 +8183,87 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -8328,7 +8526,7 @@ - + @@ -8383,145 +8581,145 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -8687,7 +8885,7 @@ - + @@ -8742,130 +8940,130 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -9042,7 +9240,7 @@ - + @@ -9061,40 +9259,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -9272,7 +9470,7 @@ - + @@ -9342,10 +9540,10 @@ - + - + @@ -9621,30 +9819,30 @@ - + - + - + - + - + - + - + - + - + @@ -9825,37 +10023,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -10025,81 +10192,81 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -10534,165 +10701,131 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -10755,7 +10888,7 @@ - + @@ -10795,79 +10928,79 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -11040,7 +11173,7 @@ - + @@ -11050,13 +11183,13 @@ - + - + @@ -11071,60 +11204,60 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -11157,36 +11290,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -11694,7 +11827,7 @@ - + @@ -11749,10 +11882,10 @@ - + - + @@ -12035,20 +12168,9 @@ - + - - - - - - - - - - - - + @@ -12155,7 +12277,7 @@ - + @@ -12376,57 +12498,57 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -12435,7 +12557,7 @@ - + @@ -12496,22 +12618,22 @@ - + - + - + - + - + - + @@ -12971,7 +13093,7 @@ - + @@ -12990,46 +13112,46 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -13041,7 +13163,7 @@ - + @@ -13210,38 +13332,41 @@ - - + + - - - - - - - - + + - - - - + - + + + + - + - - + + - - + + - - + + + + + + + + + + + @@ -13263,60 +13388,66 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -13365,7 +13496,11 @@ - + + + + + @@ -13477,7 +13612,7 @@ - + @@ -13566,7 +13701,7 @@ - + @@ -13615,13 +13750,10 @@ - + - - - @@ -13679,8 +13811,6 @@ - - @@ -13782,7 +13912,23 @@ - + + + + + + + + + + + + + + + + + @@ -13831,7 +13977,7 @@ - + @@ -13844,22 +13990,22 @@ - + - + - + - + - + - + @@ -14045,104 +14191,128 @@ - + - + - + - + - + - + - + - + - + - - + + - - - - - - - - - - - - - - + + - + - + - + - + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + @@ -14183,7 +14353,7 @@ - + @@ -14310,28 +14480,28 @@ - + - + - + - + - + - + - + - + @@ -14533,15 +14703,15 @@ - + - + - + - + @@ -15071,29 +15241,29 @@ - + - + - + - + - + - + - + - + - + @@ -15139,79 +15309,82 @@ - + - + - + - + - + - + - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -15339,7 +15512,7 @@ - + @@ -15423,6 +15596,7 @@ + @@ -15455,6 +15629,7 @@ + @@ -15544,7 +15719,7 @@ - + @@ -15558,36 +15733,39 @@ + + + - + - + - + - + - + - + - + - + - + - + @@ -16029,7 +16207,7 @@ - + @@ -16048,25 +16226,25 @@ - + - + - + - + - + - + - + @@ -16098,21 +16276,21 @@ - + - + - + - + - + - + @@ -16361,45 +16539,58 @@ - + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -16646,12 +16837,12 @@ - + - + - + @@ -16753,63 +16944,107 @@ - + - - - - - - - - - - + - + + + + + + + + + + - + - - - - - - - - - - - - - - - - + - - - - - - - - - - + - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -16834,7 +17069,7 @@ - + @@ -16863,18 +17098,18 @@ - + - + - + - + - + @@ -17130,7 +17365,26 @@ - + + + + + + + + + + + + + + + + + + + + @@ -17173,12 +17427,12 @@ - + - + - + @@ -17192,87 +17446,90 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + + + + - + - + - + @@ -17316,14 +17573,7 @@ - - - - - - - - + @@ -17376,7 +17626,7 @@ - + @@ -17481,12 +17731,12 @@ - + - + - + @@ -17558,6 +17808,9 @@ + + + @@ -17669,6 +17922,17 @@ + + + + + + + + + + + @@ -17741,23 +18005,23 @@ - + - + - + - + - + - + @@ -17812,39 +18076,56 @@ - + - + - - - - + - + + + + + + + + + + + + + + + - + - + - + - + - + - + - + + + + + + + @@ -18314,39 +18595,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -18371,24 +18658,24 @@ - + - + - + - + - + - + - + @@ -18479,15 +18766,15 @@ - + - + - + - + @@ -18520,6 +18807,7 @@ + @@ -18552,6 +18840,14 @@ + + + + + + + + @@ -18581,7 +18877,10 @@ + + + @@ -18901,9 +19200,14 @@ - + - + + + + + + @@ -18978,6 +19282,9 @@ + + + @@ -19025,7 +19332,7 @@ - + @@ -19050,45 +19357,51 @@ - - - - + + - - - - - - - - + + - + - + - + - + - + + + + + + + + + + + + + + + - + - + - + - + @@ -19116,8 +19429,12 @@ + + + + @@ -19154,12 +19471,12 @@ - + - + - + @@ -19570,7 +19887,7 @@ - + @@ -19624,7 +19941,7 @@ - + @@ -19633,7 +19950,7 @@ - + @@ -19645,7 +19962,7 @@ - + @@ -19656,12 +19973,10 @@ - - @@ -19805,7 +20120,7 @@ - + @@ -19874,7 +20189,7 @@ - + @@ -19884,7 +20199,7 @@ - + @@ -19927,6 +20242,10 @@ + + + + @@ -19935,6 +20254,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -20074,16 +20465,16 @@ - + - + - + - + @@ -20592,6 +20983,8 @@ + + @@ -20719,7 +21112,142 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -20769,7 +21297,7 @@ - + @@ -20845,58 +21373,58 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -21080,6 +21608,7 @@ + @@ -21092,6 +21621,10 @@ + + + + @@ -21401,183 +21934,11 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -21601,7 +21962,7 @@ - + @@ -21621,11 +21982,19 @@ + + + + + + + + - + @@ -21788,7 +22157,7 @@ - + @@ -21853,7 +22222,7 @@ - + @@ -21861,7 +22230,7 @@ - + @@ -21871,7 +22240,7 @@ - + @@ -21974,13 +22343,13 @@ - + - + @@ -22010,12 +22379,12 @@ - + - + @@ -22040,6 +22409,11 @@ + + + + + @@ -22053,21 +22427,21 @@ - + - + - + - + - + - + @@ -22106,7 +22480,7 @@ - + @@ -22130,24 +22504,24 @@ - + - + - + - + @@ -22182,13 +22556,6 @@ - - - - - - - @@ -22254,7 +22621,7 @@ - + @@ -22304,6 +22671,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -22947,7 +23352,7 @@ - + @@ -23270,9 +23675,9 @@ - - - + + + @@ -23290,7 +23695,7 @@ - + @@ -23441,7 +23846,7 @@ - + @@ -23782,483 +24187,519 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - + - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -24284,38 +24725,38 @@ - + - + - + - + - + - + - + - - - + + + - + - + - - + + @@ -24347,7 +24788,7 @@ - + @@ -24431,30 +24872,42 @@ - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -24697,7 +25150,7 @@ - + @@ -24716,22 +25169,22 @@ - + - + - + - + - + - + @@ -24895,119 +25348,119 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -25114,11 +25567,11 @@ - + - + @@ -25210,20 +25663,20 @@ - - - + + + - - - - + + + + - - - + + + @@ -25300,119 +25753,119 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -25550,11 +26003,11 @@ - + - + @@ -25937,6 +26390,12 @@ + + + + + + @@ -26219,30 +26678,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - @@ -26447,13 +26882,13 @@ - - - + + + - - + + @@ -26467,32 +26902,32 @@ - + - + - + - + - + - + - + - + - + @@ -26596,6 +27031,7 @@ + @@ -26608,35 +27044,41 @@ - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + @@ -26644,6 +27086,8 @@ + + @@ -26829,110 +27273,61 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -26942,99 +27337,119 @@ - - + + - - - + + + + + + + + + + + + + + + + - - - - - - - - + + + - - - + + + - - - - - - - - + + + - - - + + + + + + + + + + + + + + + + + + - - - + + + - - - + + + + - + + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + @@ -27096,8 +27511,8 @@ - - + + @@ -27292,7 +27707,7 @@ - + @@ -27305,28 +27720,28 @@ - + - + - + - + - + - + - + - + @@ -27641,128 +28056,128 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -27912,11 +28327,20 @@ + + + + + + + + + @@ -27937,9 +28361,9 @@ - - - + + + @@ -27958,13 +28382,13 @@ - - - + + + - - + + @@ -28053,7 +28477,6 @@ - @@ -28290,20 +28713,20 @@ - + - + - + - + - + @@ -28458,6 +28881,11 @@ + + + + + @@ -29014,7 +29442,7 @@ - + @@ -29030,19 +29458,19 @@ - + - + - + - + @@ -29073,31 +29501,31 @@ - + - + - + - + - + - + - + - + - + @@ -29289,6 +29717,8 @@ + + @@ -29369,6 +29799,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -29662,259 +30123,259 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -31062,14 +31523,14 @@ - + - + - + @@ -31194,282 +31655,282 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -31505,124 +31966,124 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + @@ -31729,13 +32190,10 @@ - - - - + - + @@ -32332,6 +32790,8 @@ + + @@ -32552,6 +33012,1518 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -32595,59 +34567,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -32662,6 +34634,13 @@ + + + + + + + @@ -32890,8 +34869,8 @@ - - + + @@ -33304,6 +35283,12 @@ + + + + + + @@ -33396,29 +35381,29 @@ - - + + - - - - + + + + - - - - + + + + - - + + - - - + + + @@ -33567,7 +35552,7 @@ - + @@ -33595,7 +35580,7 @@ - + @@ -34020,37 +36005,37 @@ - - + + - - + + - - - - + + + + - + - - - + + + - + - - + + - - + + @@ -34101,24 +36086,201 @@ - - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -34126,6 +36288,14 @@ + + + + + + + + @@ -34155,7 +36325,27 @@ - + + + + + + + + + + + + + + + + + + + + + @@ -34165,64 +36355,64 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -34293,71 +36483,51 @@ - + - + - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - + + + + + - - - + + + - - - - - + + + + + - - - + + + - + @@ -34437,17 +36607,17 @@ - + - + - + - + @@ -34738,7 +36908,7 @@ - + @@ -34757,10 +36927,10 @@ - + - + @@ -35384,14 +37554,14 @@ - + - + @@ -35818,7 +37988,7 @@ - + @@ -35855,43 +38025,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35906,7 +38076,7 @@ - + @@ -35934,7 +38104,7 @@ - + @@ -36049,8 +38219,13 @@ + + + + + - + @@ -36063,37 +38238,37 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -36176,7 +38351,7 @@ - + @@ -36189,37 +38364,37 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -36774,47 +38949,47 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36927,6 +39102,18 @@ + + + + + + + + + + + + @@ -36938,178 +39125,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -37232,23 +39248,6 @@ - - - - - - - - - - - - - - - - - @@ -37347,53 +39346,238 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - - - @@ -37462,13 +39646,13 @@ - - + + - - - + + + @@ -37484,97 +39668,105 @@ - - - - + + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + + + + + + + + + @@ -37683,7 +39875,7 @@ - + @@ -37705,10 +39897,10 @@ - + - + @@ -37941,7 +40133,7 @@ - + @@ -38050,22 +40242,22 @@ - + - + - + - + - + - + @@ -38125,9 +40317,6 @@ - - - @@ -38305,7 +40494,7 @@ - + @@ -38405,220 +40594,220 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39215,7 +41404,7 @@ - + @@ -39225,49 +41414,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -43087,7 +45276,7 @@ - + @@ -43196,22 +45385,22 @@ - + - + - + - + - + - + @@ -43237,7 +45426,7 @@ - + @@ -43337,220 +45526,220 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -45030,7 +47219,7 @@ - + @@ -45061,31 +47250,31 @@ - + - + - + - + - + - + - + - + - + @@ -46102,6 +48291,10 @@ + + + + @@ -46628,6 +48821,8 @@ + + @@ -46705,7 +48900,6 @@ - @@ -46760,7 +48954,6 @@ - @@ -47146,7 +49339,7 @@ - + @@ -47156,49 +49349,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -47759,41 +49952,41 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -47814,7 +50007,7 @@ - + @@ -48068,7 +50261,7 @@ - + @@ -48114,25 +50307,25 @@ - + - + - + - + - + - + - + @@ -48279,13 +50472,13 @@ - - - - - - - + + + + + + + @@ -48302,36 +50495,36 @@ - - - - + + + + - - + + - - + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -48339,10 +50532,10 @@ - - - - + + + + @@ -48353,17 +50546,17 @@ - - - + + + - - - - - - + + + + + + @@ -48371,37 +50564,37 @@ - - - - + + + + - - - - - + + + + + - - - + + + - - - + + + - - + + - - - - + + + + @@ -48421,8 +50614,8 @@ - - + + @@ -48698,7 +50891,7 @@ - + @@ -48714,13 +50907,13 @@ - + - + - + @@ -49003,7 +51196,7 @@ - + @@ -49093,7 +51286,7 @@ - + @@ -49133,7 +51326,7 @@ - + @@ -49176,11 +51369,11 @@ - + - + @@ -49313,7 +51506,7 @@ - + @@ -49474,11 +51667,11 @@ - + - + @@ -49536,17 +51729,17 @@ - + - + - + - + @@ -49834,7 +52027,7 @@ - + @@ -49905,32 +52098,32 @@ - + - + - + - + - + - + - + - + @@ -50202,6 +52395,11 @@ + + + + + @@ -50298,6 +52496,10 @@ + + + + @@ -50367,7 +52569,7 @@ - + @@ -50382,7 +52584,7 @@ - + @@ -50399,7 +52601,7 @@ - + @@ -50455,23 +52657,23 @@ - + - + - + - + @@ -50884,7 +53086,7 @@ - + @@ -50924,76 +53126,76 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -51116,7 +53318,9 @@ + + @@ -51136,6 +53340,10 @@ + + + + @@ -51172,6 +53380,13 @@ + + + + + + + @@ -51191,6 +53406,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -51221,7 +53479,7 @@ - + @@ -51237,79 +53495,79 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -51934,7 +54192,7 @@ - + @@ -52007,76 +54265,76 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -52517,7 +54775,7 @@ - + @@ -52572,40 +54830,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -52650,6 +54908,16 @@ + + + + + + + + + + @@ -52662,11 +54930,32 @@ + + + + + + + + + + + + + + + + + + + + + @@ -52736,8 +55025,83 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -52795,10 +55159,134 @@ - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -52860,6 +55348,10 @@ + + + + @@ -52873,6 +55365,10 @@ + + + + @@ -53168,7 +55664,7 @@ - + @@ -53217,10 +55713,10 @@ - + - + @@ -53288,6 +55784,11 @@ + + + + + @@ -53657,67 +56158,82 @@ - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - + + + + @@ -53939,7 +56455,7 @@ - + @@ -54024,6 +56540,10 @@ + + + + @@ -54045,7 +56565,7 @@ - + @@ -54062,20 +56582,20 @@ - + - + - + - + - + @@ -54196,7 +56716,7 @@ - + @@ -54224,7 +56744,7 @@ - + @@ -54393,7 +56913,7 @@ - + @@ -54403,112 +56923,112 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -54570,31 +57090,31 @@ - + - + - + - + - + - + - + - + @@ -54613,16 +57133,16 @@ - + - + - + - + @@ -54669,188 +57189,188 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -55084,7 +57604,7 @@ - + @@ -55094,320 +57614,320 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -55487,7 +58007,7 @@ - + @@ -55506,16 +58026,16 @@ - + - + - + - + @@ -56462,7 +58982,7 @@ - + @@ -56472,19 +58992,19 @@ - + - + - + - + - + @@ -56581,7 +59101,7 @@ - + @@ -56770,7 +59290,7 @@ - + @@ -56780,25 +59300,25 @@ - + - + - + - + - + - + @@ -57127,7 +59647,7 @@ - + @@ -57182,15 +59702,15 @@ - + - + - + - + @@ -57283,45 +59803,45 @@ - - - - - + + + + + - - + + - - - + + + - - - - + + + + - - - - - - - - + + + + + + + + - - - - - - - + + + + + + + @@ -58225,9 +60745,6 @@ - - - @@ -58673,7 +61190,7 @@ - + @@ -58692,67 +61209,67 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -64446,10 +66963,10 @@ - - - - + + + + @@ -65211,7 +67728,7 @@ - + @@ -65254,193 +67771,193 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -65564,7 +68081,7 @@ - + @@ -65607,193 +68124,193 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -66079,9 +68596,9 @@ - + - + @@ -66092,42 +68609,41 @@ - - + + - + - - + + - - + + - + - + - - + + - - + + - - + @@ -66212,620 +68728,426 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - - - - + + - + - - - - - - - - + + - + - - + + - - + + - + + + + - + - + - - + + - - - - - - - - - - - - - - + + - + - - + + - - + + - - + + - - + + - - + + - - - - - + + - + - + - + - + - - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -66834,12 +69156,6 @@ - - - - - - @@ -66854,7 +69170,7 @@ - + @@ -66863,7 +69179,7 @@ - + @@ -66885,179 +69201,109 @@ - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - + - + - - - - - - + - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -67074,7 +69320,6 @@ - @@ -67229,12 +69474,12 @@ - + - + - + @@ -67254,12 +69499,12 @@ - + - + - + @@ -67289,87 +69534,111 @@ - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - + + - - - - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - - + + - + - + + + + + + + + + + + + + - + - + - + @@ -67396,39 +69665,17 @@ - - - - - - - - - - - - - - - + - + - - - - - - - - - + @@ -67436,27 +69683,7 @@ - - - - - - - - - - - - - - - - - - - - - + @@ -67604,189 +69831,201 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -67814,24 +70053,30 @@ - + - + - + - + - + - + - + + + + + + + @@ -67877,245 +70122,269 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + @@ -68132,21 +70401,6 @@ - - - - - - - - - - - - - - - @@ -68169,7 +70423,7 @@ - + @@ -68183,9 +70437,6 @@ - - - @@ -68215,16 +70466,7 @@ - - - - - - - - - @@ -68312,7 +70554,6 @@ - @@ -68505,15 +70746,15 @@ - + - + - + - + @@ -68628,7 +70869,7 @@ - + @@ -68666,25 +70907,25 @@ - + - + - + - + - + - + - + @@ -68726,26 +70967,26 @@ - + - + - + - + - + - + - + - + @@ -68880,7 +71121,7 @@ - + @@ -69181,7 +71422,6 @@ - @@ -69223,7 +71463,7 @@ - + @@ -69233,46 +71473,46 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -69725,13 +71965,13 @@ - + - + - + @@ -70062,18 +72302,18 @@ - + - + - + - + - + @@ -70251,7 +72491,7 @@ - + @@ -70568,18 +72808,18 @@ - + - + - + - + - + @@ -70673,40 +72913,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -70721,70 +72961,70 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -70793,49 +73033,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -71382,162 +73622,174 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -71610,9 +73862,9 @@ - + - + @@ -71790,15 +74042,15 @@ - + - + - + - + @@ -72151,7 +74403,7 @@ - + @@ -72242,11 +74494,6 @@ - - - - - @@ -72279,12 +74526,12 @@ - + - + - + @@ -72516,27 +74763,27 @@ - + - + - + - + - + - + - + - + @@ -72816,26 +75063,26 @@ - + - + - + - + - + - + - + @@ -72857,64 +75104,64 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -72990,17 +75237,6 @@ - - - - - - - - - - - @@ -73024,12 +75260,6 @@ - - - - - - @@ -73403,14 +75633,14 @@ - + - + @@ -73460,7 +75690,7 @@ - + @@ -73563,46 +75793,46 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -73658,93 +75888,105 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -73823,10 +76065,10 @@ - + - + @@ -73904,7 +76146,7 @@ - + @@ -73957,7 +76199,7 @@ - + @@ -74157,7 +76399,7 @@ - + @@ -74205,7 +76447,7 @@ - + @@ -74592,9 +76834,9 @@ - + - + @@ -74635,58 +76877,58 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -74822,7 +77064,7 @@ - + @@ -74850,22 +77092,22 @@ - + - + - + - + - + - + @@ -74874,7 +77116,7 @@ - + @@ -74949,7 +77191,7 @@ - + @@ -75013,34 +77255,34 @@ - + - + - + - + - + - + - + - + - + - + @@ -75114,7 +77356,7 @@ - + @@ -75127,7 +77369,7 @@ - + @@ -75158,66 +77400,66 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + @@ -75241,15 +77483,15 @@ - - + + - + - + @@ -75268,14 +77510,14 @@ - + - + @@ -75301,7 +77543,7 @@ - + @@ -75410,7 +77652,7 @@ - + @@ -75426,19 +77668,19 @@ - + - + - + - + @@ -75491,15 +77733,15 @@ - + - + - + - + @@ -75513,7 +77755,7 @@ - + @@ -75545,46 +77787,46 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -75629,41 +77871,41 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -75703,7 +77945,7 @@ - + @@ -75735,39 +77977,39 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -75788,48 +78030,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -75899,17 +78141,17 @@ - + - + - + - + @@ -75954,100 +78196,100 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + @@ -76222,8 +78464,8 @@ - - + + @@ -76253,7 +78495,7 @@ - + @@ -76262,7 +78504,7 @@ - + @@ -76322,7 +78564,7 @@ - + @@ -76336,17 +78578,17 @@ - + - + - + @@ -76817,48 +79059,48 @@ - - - - - + + + + + - - + + - - + + - + - + - + - + - + - + - + @@ -76885,7 +79127,7 @@ - + @@ -76907,7 +79149,7 @@ - + @@ -76928,30 +79170,30 @@ - + - + - + - + - + @@ -76961,17 +79203,17 @@ - - + + - + - + @@ -76981,13 +79223,13 @@ - - + + - - + + @@ -76995,13 +79237,13 @@ - - + + - - + + @@ -77112,7 +79354,7 @@ - + @@ -77158,10 +79400,10 @@ - + - + @@ -77390,7 +79632,7 @@ - + @@ -77433,37 +79675,37 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -77505,34 +79747,34 @@ - + - + - + - + - + - + - + - + - + - + @@ -77544,28 +79786,28 @@ - + - + - + - + - + - + - + - + @@ -78100,14 +80342,14 @@ - + - - + + - - + + @@ -78131,11 +80373,11 @@ - - + + - + @@ -78149,7 +80391,7 @@ - + @@ -78191,7 +80433,7 @@ - + @@ -78246,64 +80488,64 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -78398,7 +80640,7 @@ - + @@ -78411,62 +80653,62 @@ - + - + - + - + - + - + - - + + - + - + - + - + - + - + @@ -78585,8 +80827,8 @@ - - + + @@ -78746,9 +80988,9 @@ - + - + @@ -78801,16 +81043,16 @@ - + - + - + - + @@ -79079,7 +81321,7 @@ - + @@ -79471,10 +81713,10 @@ - + - + @@ -79607,71 +81849,83 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + + + + + + + + + + + + + - + - + - + - - + + - + @@ -79682,7 +81936,7 @@ - + @@ -79691,28 +81945,28 @@ - + - + - + - + - + @@ -79724,7 +81978,7 @@ - + @@ -79737,22 +81991,22 @@ - + - + - + - + - + - + @@ -79792,7 +82046,7 @@ - + @@ -79805,28 +82059,28 @@ - + - + - + - + - + - + - + - + @@ -80048,83 +82302,83 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -80173,7 +82427,7 @@ - + @@ -80195,7 +82449,7 @@ - + @@ -80281,20 +82535,26 @@ - - - + + + - - - + + + - - - - + + + + + + + + + + @@ -80374,27 +82634,42 @@ - - - - - - - + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + @@ -80549,6 +82824,16 @@ + + + + + + + + + + @@ -80650,23 +82935,23 @@ - + - + - + - + - + - + @@ -80841,7 +83126,7 @@ - + @@ -80860,7 +83145,7 @@ - + @@ -80880,18 +83165,10 @@ - - - - - - - - @@ -80919,18 +83196,18 @@ - + - + - + - + - + @@ -80950,8 +83227,8 @@ - - + + @@ -81006,38 +83283,38 @@ - - + + - - - + + + - - + + - - + + - - + + - - - + + + - - - + + + - + @@ -81060,7 +83337,7 @@ - + @@ -81068,11 +83345,11 @@ - + - + @@ -81102,81 +83379,81 @@ - + - - - - - - - - - - - - - - - - + - + + + + - + + + + - + - - - - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -81232,22 +83509,22 @@ - + - + - + - + @@ -81310,13 +83587,13 @@ - + - + @@ -81332,29 +83609,29 @@ - + - + - + - + - + - + - + - + @@ -81538,39 +83815,39 @@ - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + @@ -81678,7 +83955,7 @@ - + @@ -82033,7 +84310,7 @@ - + @@ -82048,47 +84325,47 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -82097,7 +84374,7 @@ - + @@ -82316,7 +84593,7 @@ - + @@ -82587,6 +84864,24 @@ + + + + + + + + + + + + + + + + + + @@ -82595,8 +84890,8 @@ - - + + @@ -82674,7 +84969,7 @@ - + @@ -82684,88 +84979,88 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -82818,7 +85113,7 @@ - + @@ -82828,28 +85123,28 @@ - + - + - + - + - + - + - + - + @@ -82999,26 +85294,12 @@ - - - - - - - - - - - - - - @@ -83052,7 +85333,6 @@ - @@ -83218,8 +85498,7 @@ - - + @@ -83244,61 +85523,61 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -83614,7 +85893,7 @@ - + @@ -83675,271 +85954,274 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - + + - + - + - + - + - + - + - - - - + - + - - + + - + + + + - - - - + - + + + + - - - - + - + - + - + - + - + + + + - + - + - + - + - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + + + + - + - + - + - + - + - + - + @@ -84206,18 +86488,18 @@ - + - + - + - + - + @@ -84571,7 +86853,7 @@ - + @@ -84593,94 +86875,94 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -84952,6 +87234,10 @@ + + + + @@ -85094,29 +87380,29 @@ - + - + - + - + - + - + - + - + @@ -85344,92 +87630,92 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -85650,59 +87936,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -85825,78 +88111,13 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + @@ -86188,44 +88409,44 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -86400,16 +88621,16 @@ - + - + - + - + @@ -86589,7 +88810,7 @@ - + @@ -86602,79 +88823,79 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -86744,7 +88965,7 @@ - + @@ -86757,13 +88978,13 @@ - + - + - + @@ -86781,28 +89002,28 @@ - + - + - + - + - + - + - + - + @@ -86991,43 +89212,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -87098,18 +89282,42 @@ - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -87141,6 +89349,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -87148,21 +89388,14 @@ + + - - + + - - - - - - - - - @@ -87201,6 +89434,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -87263,353 +89526,338 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - + - + - - + + - - - - - + + - + - - + + - - + + + + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + @@ -87907,18 +90155,18 @@ - + - + - + - + - + @@ -88006,68 +90254,68 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88222,46 +90470,46 @@ - + - + - - - - - - - - + + - + - + - - + + - - + + - - + + - - + + + + + + + + - + - + @@ -88271,64 +90519,64 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88346,50 +90594,50 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88509,10 +90757,8 @@ - - @@ -88532,13 +90778,12 @@ - - + @@ -88588,12 +90833,6 @@ - - - - - - @@ -88605,12 +90844,6 @@ - - - - - - @@ -88643,11 +90876,6 @@ - - - - - @@ -88661,7 +90889,7 @@ - + @@ -88669,7 +90897,7 @@ - + @@ -88835,69 +91063,69 @@ - - - - - + + + + + - - + + - - - + + + - - + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - + + - - + + - - - + + + - - - + + + @@ -88908,6 +91136,10 @@ + + + + @@ -89286,31 +91518,31 @@ - + - + - + - + - + - + - + - + @@ -89429,195 +91661,15 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + @@ -89661,6 +91713,11 @@ + + + + + @@ -89725,6 +91782,12 @@ + + + + + + @@ -90283,7 +92346,7 @@ - + @@ -90381,12 +92444,12 @@ - + - + - + @@ -90543,43 +92606,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -90594,7 +92657,7 @@ - + @@ -91334,37 +93397,37 @@ - + - + - + - + - + - + - + - + - + - + @@ -91671,7 +93734,7 @@ - + @@ -91795,7 +93858,7 @@ - + @@ -91868,76 +93931,76 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92423,11 +94486,11 @@ - - + + - + @@ -92486,35 +94549,35 @@ - + - + - + - + - + - + - + - + - + @@ -92541,11 +94604,11 @@ - + - + @@ -92566,11 +94629,11 @@ - + - + @@ -92623,17 +94686,17 @@ - - - + + + - - + + - - + + @@ -92641,20 +94704,20 @@ - - + + - - - - - - - - + + + + + + + + @@ -92940,7 +95003,7 @@ - + @@ -92959,28 +95022,28 @@ - + - + - + - + - + - + - + - + @@ -93019,49 +95082,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -93422,7 +95485,7 @@ - + @@ -93441,28 +95504,28 @@ - + - + - + - + - + - + - + - + @@ -93501,49 +95564,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -93999,6 +96062,10 @@ + + + + @@ -94206,8 +96273,479 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -94337,7 +96875,7 @@ - + @@ -94347,375 +96885,387 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -94793,48 +97343,54 @@ - + - + - + - + - + - + - + - + + + + + + + - + @@ -94956,37 +97512,37 @@ - + - + - + - + - + - + - + @@ -95197,12 +97753,12 @@ - + - + - + @@ -95215,12 +97771,12 @@ - + - + - + @@ -95600,20 +98156,20 @@ - + - + - + - + - + - + @@ -96029,7 +98585,7 @@ - + @@ -96038,7 +98594,7 @@ - + @@ -96066,29 +98622,29 @@ - + - + - + - + - + - + - + - + @@ -96916,7 +99472,7 @@ - + @@ -96956,76 +99512,76 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -97158,7 +99714,7 @@ - + @@ -97207,7 +99763,7 @@ - + @@ -97241,8 +99797,8 @@ - - + + @@ -97362,11 +99918,22 @@ + + + + + + + + + + + @@ -97399,6 +99966,12 @@ + + + + + + @@ -97406,26 +99979,26 @@ - + - + - + - + - + - + @@ -97466,50 +100039,50 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -97542,20 +100115,20 @@ - + - + - + - + @@ -97571,26 +100144,26 @@ - + - + - + - + - + - + @@ -97684,29 +100257,29 @@ - + - + - + - + - + - + - + @@ -97907,24 +100480,24 @@ - + - + - + - + - + - + - + @@ -98047,36 +100620,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -98107,17 +100680,17 @@ - - + + - - - + + + - - + + @@ -98239,384 +100812,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -98639,32 +100834,10 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + @@ -98684,43 +100857,10 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -98739,6 +100879,9 @@ + + + @@ -99056,83 +101199,83 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -99465,7 +101608,7 @@ - + @@ -99497,7 +101640,7 @@ - + @@ -99519,10 +101662,10 @@ - + - + @@ -100281,6 +102424,15 @@ + + + + + + + + + @@ -101176,33 +103328,33 @@ - - + + - - - + + + - - - + + + - - - + + + - + - - - + + + @@ -101276,27 +103428,20 @@ - - + + - - - - + + + + - - - - - - - @@ -101311,6 +103456,7 @@ + @@ -101332,7 +103478,7 @@ - + @@ -101363,29 +103509,8 @@ - - - - - - - - - - - - - - - - - - - - - @@ -101544,6 +103669,24 @@ + + + + + + + + + + + + + + + + + + @@ -101842,6 +103985,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -101853,6 +104037,7 @@ + @@ -101867,13 +104052,13 @@ - + - + @@ -101917,6 +104102,14 @@ + + + + + + + + @@ -101942,6 +104135,7 @@ + @@ -102028,6 +104222,13 @@ + + + + + + + @@ -102044,6 +104245,14 @@ + + + + + + + + @@ -102169,833 +104378,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -103047,7 +104429,7 @@ - + @@ -103066,22 +104448,22 @@ - + - + - + - + - + - + @@ -103350,11 +104732,11 @@ - + - + @@ -103400,16 +104782,6 @@ - - - - - - - - - - @@ -103457,7 +104829,7 @@ - + @@ -103500,17 +104872,11 @@ - - - - - - - - - + + + @@ -103554,43 +104920,22 @@ - - - - - - - - - - - - - - - - - - - - - + - + - + - - - + + @@ -103627,7 +104972,6 @@ - @@ -103696,62 +105040,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -103769,29 +105057,36 @@ - - - + - + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - @@ -103812,28 +105107,13 @@ - + + + - - - - - - - - - - - - - - - - - @@ -103843,6 +105123,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -103851,6 +105171,12 @@ + + + + + + @@ -103863,13 +105189,29 @@ + + + + + + + + + + + + + + + + - + @@ -104148,7 +105490,6 @@ - @@ -104157,12 +105498,6 @@ - - - - - - @@ -104171,10 +105506,6 @@ - - - - @@ -104328,7 +105659,7 @@ - + @@ -104520,166 +105851,178 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + - + @@ -104706,10 +106049,86 @@ - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -104739,15 +106158,15 @@ - + - + - + - + @@ -104928,7 +106347,7 @@ - + @@ -104966,25 +106385,25 @@ - + - + - + - + - + - + - + @@ -105623,7 +107042,7 @@ - + @@ -105633,40 +107052,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -105738,340 +107157,346 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - + + + + - - - - + - + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + @@ -106225,13 +107650,13 @@ - + - + - + @@ -106534,7 +107959,7 @@ - + @@ -106544,46 +107969,46 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -106730,18 +108155,18 @@ - + - + - + - + - + @@ -107051,7 +108476,7 @@ - + @@ -107166,7 +108591,7 @@ - + @@ -107350,48 +108775,24 @@ - + + - + + + + - + - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - + - @@ -107464,32 +108865,32 @@ - + - + - + - + - + - + - + - + @@ -107622,23 +109023,7 @@ - - - - - - - - - - - - - - - - - + @@ -107663,7 +109048,7 @@ - + @@ -107713,75 +109098,87 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -107923,47 +109320,66 @@ - + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + @@ -108020,7 +109436,7 @@ - + @@ -108081,22 +109497,22 @@ - + - + - + - + - + - + @@ -108141,27 +109557,27 @@ - + - + - + - + - + - + - + - + @@ -108344,81 +109760,81 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -108483,15 +109899,9 @@ - - - - - - - - + + @@ -108735,6 +110145,9 @@ + + + @@ -109217,14 +110630,6 @@ - - - - - - - - @@ -109727,6 +111132,14 @@ + + + + + + + + @@ -109740,73 +111153,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -109937,124 +111350,160 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + - - + + - - + + - + - + - - - - + - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -110298,7 +111747,7 @@ - + @@ -110425,28 +111874,28 @@ - + - + - + - + - + - + - + - + @@ -111028,95 +112477,101 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -111156,7 +112611,7 @@ - + @@ -111229,112 +112684,112 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -111370,355 +112825,355 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + @@ -111988,7 +113443,7 @@ - + @@ -112031,37 +113486,40 @@ - + - + - + - + - + - + - + - + - + - + - + + + + @@ -112073,88 +113531,88 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112416,7 +113874,7 @@ - + @@ -112471,10 +113929,10 @@ - + - + @@ -112541,90 +113999,102 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112634,18 +114104,18 @@ - + - + - + - + - + @@ -112728,7 +114198,7 @@ - + @@ -112742,36 +114212,39 @@ + + + - + - + - + - + - + - + - + - + - + - + @@ -113165,98 +114638,113 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + @@ -113291,135 +114779,135 @@ - + - - - - - - - - - - - - - + - + - + - + - + - + + + + + + + + + + - + - - + + - - + + - - - - - - - - - - - + + - + - + - + - - - - + - + - + + + + - + - + - - - - + - - - - - - - + - + - + + + + + + + - + - + + + + - + - + - - + + + + + + + + + + + + + + - + - + - + - + - + @@ -113685,28 +115173,28 @@ - + - + - + - + - + - + - + @@ -113773,22 +115261,22 @@ - + - + - + - + - + - + @@ -113806,7 +115294,7 @@ - + @@ -113824,40 +115312,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -113877,7 +115365,7 @@ - + @@ -113896,40 +115384,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -113939,18 +115427,24 @@ - + - + - + - + - + + + + + + + @@ -114023,15 +115517,18 @@ - + - + - + + + + - + @@ -114045,89 +115542,101 @@ - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + + + + - + + + + + + + + + + - + + + + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114158,110 +115667,110 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114281,7 +115790,7 @@ - + @@ -114295,15 +115804,15 @@ - + - + - + - + @@ -114389,12 +115898,12 @@ - + - + - + @@ -114408,83 +115917,95 @@ - + - + - + - + - + - + - + - + - + - - + + - - - - - - - - - - - - - - + + - + - + - + - + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + @@ -114534,79 +116055,85 @@ - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + @@ -114621,7 +116148,7 @@ - + @@ -114681,7 +116208,7 @@ - + @@ -114736,145 +116263,145 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115137,7 +116664,7 @@ - + @@ -115147,13 +116674,13 @@ - + - + @@ -115284,7 +116811,7 @@ - + @@ -115321,25 +116848,25 @@ - + - + - + - + - + - + - + @@ -115348,7 +116875,7 @@ - + @@ -115418,10 +116945,10 @@ - + - + @@ -115462,35 +116989,35 @@ - + - + - + - + - + - + - + - + - + @@ -115842,7 +117369,7 @@ - + @@ -115882,169 +117409,169 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -116058,21 +117585,33 @@ - + - + - + - + - + - + + + + + + + + + + + + + @@ -116127,102 +117666,108 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + @@ -116235,7 +117780,7 @@ - + @@ -116246,7 +117791,7 @@ - + @@ -116259,22 +117804,22 @@ - + - + - + - + - + - + @@ -116323,7 +117868,7 @@ - + @@ -116342,46 +117887,46 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -116702,12 +118247,12 @@ - + - + - + @@ -116969,13 +118514,17 @@ - + - + + + + + @@ -117962,21 +119511,21 @@ - + - + - + - + - + - + @@ -118081,7 +119630,7 @@ - + @@ -118136,130 +119685,130 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -119444,190 +120993,202 @@ - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - - - - - - + - + - + - - - - - - - - - - - - - - - + - - - - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -119656,21 +121217,21 @@ - + - + - + - + - + - + @@ -119702,21 +121263,21 @@ - + - + - + - + - + - + @@ -119741,6 +121302,12 @@ + + + + + + @@ -120203,23 +121770,23 @@ - + - + - + - + - + - + @@ -120555,7 +122122,7 @@ - + @@ -120653,22 +122220,22 @@ - + - + - + - + - + - + @@ -120725,7 +122292,7 @@ - + @@ -120744,25 +122311,25 @@ - + - + - + - + - + - + - + @@ -121130,7 +122697,7 @@ - + @@ -121376,70 +122943,70 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -122216,6 +123783,7 @@ + @@ -124224,15 +125792,15 @@ - + - + - + - + @@ -124481,7 +126049,7 @@ - + @@ -124503,68 +126071,68 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -124667,46 +126235,46 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -124715,7 +126283,7 @@ - + @@ -124862,7 +126430,7 @@ - + @@ -124956,40 +126524,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -125004,70 +126572,70 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -125076,49 +126644,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -125468,750 +127036,798 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - - - - + + - + - - - - - - - - + + - + - - + + - - + + - + + + + - + - + - - + + - - - - - - - - - - - - - - + + - + - - + + - - + + - - + + - - + + - - + + - - - - - + + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + @@ -126438,7 +128054,7 @@ - + @@ -126975,7 +128591,7 @@ - + @@ -127006,56 +128622,56 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + @@ -127119,34 +128735,34 @@ - + - + - + - + - + - + - + - + - + - + @@ -127158,7 +128774,7 @@ - + @@ -127222,15 +128838,15 @@ - + - + - + - + @@ -127393,7 +129009,7 @@ - + @@ -127506,23 +129122,23 @@ - + - + - + - + - + - + - + @@ -127530,26 +129146,26 @@ - + - + - + - + - + - + - + - + @@ -128789,7 +130405,7 @@ - + @@ -128884,9 +130500,9 @@ - + - + @@ -128894,7 +130510,7 @@ - + @@ -128920,12 +130536,12 @@ - + - + - + @@ -128948,7 +130564,7 @@ - + @@ -128959,189 +130575,201 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -129210,12 +130838,12 @@ - + - + - + @@ -129231,24 +130859,24 @@ - + - + - + - + - + - + - + @@ -129402,12 +131030,12 @@ - + - + - + @@ -129441,7 +131069,7 @@ - + @@ -129821,12 +131449,12 @@ - + - + - + @@ -129879,15 +131507,15 @@ - + - + - + - + @@ -129904,90 +131532,108 @@ - + - + - + - + - + - + - + + + + + + + - + - + - + - + - + - + - + - - + + - + - + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -129999,9 +131645,9 @@ - - - + + + @@ -130473,45 +132119,57 @@ - + - + - + - + - + - - - - - - - - - - - + + - - - - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -130719,29 +132377,29 @@ - + - + - + - + - + - + - + - + - + @@ -130750,34 +132408,34 @@ - + - + - + - + - + - + - + - + - + - + @@ -130786,21 +132444,21 @@ - + - + - + - + - + - + @@ -130877,28 +132535,28 @@ - + - + - + - + - + - + - + - + - + @@ -131002,23 +132660,23 @@ - + - + - + - + - + - + @@ -131083,27 +132741,27 @@ - + - + - + - + - + - + - + - + @@ -131115,93 +132773,105 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -131227,7 +132897,7 @@ - + @@ -131331,7 +133001,7 @@ - + @@ -131571,7 +133241,7 @@ - + @@ -131732,18 +133402,18 @@ - + - + - + @@ -131754,21 +133424,21 @@ - + - + - + - + @@ -131786,7 +133456,7 @@ - + @@ -131813,7 +133483,7 @@ - + @@ -132048,7 +133718,7 @@ - + @@ -132056,7 +133726,7 @@ - + @@ -132074,7 +133744,7 @@ - + @@ -132109,42 +133779,42 @@ - + - + - + - + - + - + - + @@ -132153,172 +133823,172 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132326,148 +133996,148 @@ - + - + - + - + - + - + - + - - + + - - + + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132475,7 +134145,7 @@ - + @@ -132542,7 +134212,7 @@ - + @@ -132557,15 +134227,15 @@ - - + + - + @@ -132577,7 +134247,7 @@ - + @@ -132770,17 +134440,17 @@ - - + + - + - + - + @@ -132871,19 +134541,19 @@ - + - + - + @@ -132925,9 +134595,9 @@ - + - + @@ -132935,7 +134605,7 @@ - + @@ -133029,100 +134699,100 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133193,8 +134863,8 @@ - - + + @@ -133204,7 +134874,7 @@ - + @@ -133256,7 +134926,7 @@ - + @@ -133283,11 +134953,11 @@ - + - + @@ -133323,7 +134993,7 @@ - + @@ -133353,13 +135023,13 @@ - + - + @@ -133376,7 +135046,6 @@ - @@ -133388,10 +135057,6 @@ - - - - @@ -133413,7 +135078,7 @@ - + @@ -133460,6 +135125,10 @@ + + + + @@ -133479,7 +135148,7 @@ - + @@ -133487,15 +135156,18 @@ - - - - + + + + + + + @@ -133504,7 +135176,7 @@ - + @@ -133699,8 +135371,9 @@ - - + + + @@ -133764,7 +135437,7 @@ - + @@ -133856,7 +135529,7 @@ - + @@ -134260,15 +135933,15 @@ - + - + - + @@ -134313,7 +135986,7 @@ - + @@ -134341,7 +136014,7 @@ - + @@ -134382,11 +136055,15 @@ - - - + + + + + + + @@ -134512,87 +136189,90 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + + + + - + - + - + @@ -134618,79 +136298,82 @@ - + - + - + - + - + - + - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -134731,17 +136414,17 @@ - + - + - + - + - + @@ -134755,23 +136438,6 @@ - - - - - - - - - - - - - - - - - @@ -135028,6 +136694,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -135245,7 +136957,7 @@ - + @@ -135432,12 +137144,12 @@ - + - + - + @@ -135463,43 +137175,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -135515,6 +137227,17 @@ + + + + + + + + + + + @@ -135604,6 +137327,7 @@ + @@ -135647,7 +137371,7 @@ - + @@ -135681,6 +137405,11 @@ + + + + + @@ -135860,11 +137589,13 @@ - + + + @@ -135931,7 +137662,7 @@ - + @@ -136098,12 +137829,12 @@ - + - + @@ -136167,7 +137898,7 @@ - + @@ -136178,90 +137909,90 @@ - - + + - - - + + + - - - + + + - - - - + + + + - - + + - - + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - - + + + + - - - - + + + + - - - - - + + + + + @@ -136325,46 +138056,17 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + @@ -136568,12 +138270,19 @@ + + - + + + + + + @@ -136611,7 +138320,7 @@ - + @@ -136745,18 +138454,18 @@ - - + + - - - + + + - - - + + + @@ -136806,8 +138515,8 @@ - - + + @@ -136840,6 +138549,10 @@ + + + + @@ -136898,12 +138611,12 @@ - + - + - + @@ -136918,16 +138631,16 @@ - - + + - - - - - - + + + + + + @@ -136941,6 +138654,11 @@ + + + + + @@ -136949,14 +138667,14 @@ - + - + @@ -137174,6 +138892,11 @@ + + + + + @@ -137199,7 +138922,7 @@ - + @@ -137211,11 +138934,11 @@ - + - + @@ -137223,7 +138946,7 @@ - + @@ -137351,8 +139074,6 @@ - - @@ -137374,6 +139095,8 @@ + + @@ -137424,7 +139147,7 @@ - + @@ -137433,20 +139156,20 @@ - - + + - - + + - - + + - - + + @@ -137543,9 +139266,9 @@ - - - + + + @@ -137560,21 +139283,21 @@ - - + + - + - + - + - - + + @@ -137584,11 +139307,11 @@ - + - - + + @@ -137597,7 +139320,11 @@ - + + + + + @@ -137757,30 +139484,42 @@ - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -137920,7 +139659,7 @@ - + @@ -138089,38 +139828,41 @@ - - + + - - - - - - - - + + - - - - + - + + + + - + - - + + - - + + - - + + + + + + + + + + + @@ -138514,50 +140256,50 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -138739,15 +140481,15 @@ - + - + - + - + @@ -138759,6 +140501,8 @@ + + @@ -138778,45 +140522,46 @@ - - - + + + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + - - - - + + + + @@ -138824,52 +140569,56 @@ - - - + + + - - - - - + + + + + - + - - + + - - + + + + + + - - - - + + + + - - + + - - + + - - - + + + - - + + @@ -138882,6 +140631,13 @@ + + + + + + + @@ -138899,32 +140655,21 @@ - + - + - + - + - - - - - - - - - - - @@ -138961,84 +140706,26 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + @@ -139070,9 +140757,15 @@ + + + + + + @@ -139106,6 +140799,14 @@ + + + + + + + + @@ -139191,11 +140892,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -139207,7 +141001,6 @@ - @@ -139215,46 +141008,46 @@ - - - - + + + + - - - - - + + + + + - - - - + + + + - - - + + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -139274,11 +141067,15 @@ - - - - - + + + + + + + + + @@ -139320,6 +141117,7 @@ + @@ -139373,6 +141171,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -139383,6 +141236,14 @@ + + + + + + + + @@ -139577,7 +141438,7 @@ - + @@ -139598,7 +141459,14 @@ - + + + + + + + + @@ -139607,6 +141475,12 @@ + + + + + + @@ -139713,6 +141587,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -139727,7 +141634,7 @@ - + @@ -139740,28 +141647,28 @@ - + - + - + - + - + - + - + - + @@ -139801,6 +141708,7 @@ + @@ -139810,10 +141718,26 @@ + + + + + + + + + + + + + + + + @@ -139827,7 +141751,7 @@ - + @@ -139841,6 +141765,11 @@ + + + + + @@ -139859,6 +141788,11 @@ + + + + + @@ -139877,6 +141811,12 @@ + + + + + + @@ -140120,6 +142060,10 @@ + + + + @@ -140250,6 +142194,7 @@ + @@ -140287,6 +142232,11 @@ + + + + + @@ -140364,6 +142314,15 @@ + + + + + + + + + @@ -140371,6 +142330,12 @@ + + + + + + @@ -140395,10 +142360,17 @@ - - - - + + + + + + + + + + + @@ -140436,9 +142408,6 @@ - - - @@ -140556,7 +142525,7 @@ - + @@ -140590,37 +142559,37 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -140640,7 +142609,7 @@ - + @@ -140682,11 +142651,11 @@ - + - + @@ -140698,7 +142667,7 @@ - + @@ -140714,7 +142683,7 @@ - + @@ -140731,17 +142700,17 @@ - + - + - + @@ -140787,7 +142756,7 @@ - + @@ -140795,7 +142764,7 @@ - + @@ -140861,6 +142830,12 @@ + + + + + + @@ -140934,29 +142909,81 @@ - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -141155,6 +143182,12 @@ + + + + + + @@ -141209,14 +143242,14 @@ - + - + @@ -141297,6 +143330,15 @@ + + + + + + + + + @@ -141307,6 +143349,12 @@ + + + + + + @@ -141463,6 +143511,7 @@ + @@ -141519,11 +143568,6 @@ - - - - - @@ -141597,34 +143641,38 @@ - - - - - - - + + + + + + + - - - - + + + + - - - + + + - - - + + + + + + + @@ -141657,7 +143705,7 @@ - + @@ -141687,8 +143735,8 @@ - - + + @@ -141702,6 +143750,14 @@ + + + + + + + + @@ -141825,7 +143881,7 @@ - + @@ -141839,7 +143895,7 @@ - + @@ -141897,6 +143953,13 @@ + + + + + + + @@ -141919,6 +143982,12 @@ + + + + + + @@ -141950,6 +144019,18 @@ + + + + + + + + + + + + @@ -142027,6 +144108,16 @@ + + + + + + + + + + @@ -142036,6 +144127,10 @@ + + + + @@ -142048,30 +144143,34 @@ - - - - + + + - - - + + + - - - - - + + + + + - - - + + + + + + + + @@ -142185,6 +144284,13 @@ + + + + + + + @@ -142235,15 +144341,15 @@ - + - + - + - + @@ -142268,13 +144374,13 @@ - - - + + + - - + + @@ -142291,11 +144397,16 @@ - - + + + + + + + @@ -142316,7 +144427,7 @@ - + @@ -142433,7 +144544,7 @@ - + @@ -142447,6 +144558,9 @@ + + + @@ -142465,64 +144579,20 @@ + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -142550,7 +144620,6 @@ - @@ -142777,91 +144846,91 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -142882,140 +144951,150 @@ - - + + - - - + + + - + - + - + - + - + - + - + + + + + + + + + + + - + - - + + - - - - + + + + - - + + - - - - + + + + - - + + - - - + + + - - + + - - + + - - - - + + + + - - - - - - + + + + + + - - - + + + - - - + + + - - + + - - - + + + - - + + - - - - + + + + @@ -143034,12 +145113,12 @@ - + - + @@ -143057,23 +145136,23 @@ - - + + - - + + - - + + - - + + @@ -143145,48 +145224,54 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -143322,7 +145407,7 @@ - + @@ -143335,24 +145420,24 @@ - + - + - + - + - + - + - + @@ -143456,9 +145541,9 @@ - + - + @@ -143480,7 +145565,7 @@ - + @@ -143505,25 +145590,25 @@ - + - + - + - + - + - + - + @@ -143573,6 +145658,10 @@ + + + + @@ -143589,6 +145678,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -143652,7 +145772,7 @@ - + @@ -143670,8 +145790,8 @@ - - + + @@ -143679,6 +145799,11 @@ + + + + + @@ -143690,6 +145815,13 @@ + + + + + + + @@ -143706,6 +145838,13 @@ + + + + + + + @@ -143766,7 +145905,7 @@ - + @@ -143825,14 +145964,14 @@ - + - + @@ -143899,7 +146038,7 @@ - + @@ -143944,7 +146083,7 @@ - + @@ -144102,10 +146241,10 @@ - + - + @@ -144339,7 +146478,7 @@ - + @@ -144347,46 +146486,46 @@ - - - - - + + + + + - - - + + + - - - - - + + + + + - - - + + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -144487,22 +146626,22 @@ - + - + - + - + @@ -144582,17 +146721,17 @@ - + - + - + @@ -144600,13 +146739,17 @@ - - + + + + + + @@ -145649,7 +147792,7 @@ - + @@ -145818,61 +147961,61 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -145881,16 +148024,16 @@ - + - + - + - + @@ -146335,7 +148478,7 @@ - + @@ -147707,7 +149850,7 @@ - + @@ -147871,7 +150014,7 @@ - + @@ -149094,7 +151237,7 @@ - + @@ -149309,7 +151452,7 @@ - + @@ -149445,7 +151588,7 @@ - + @@ -150715,14 +152858,14 @@ - + - + @@ -150896,7 +153039,7 @@ - + @@ -150966,7 +153109,7 @@ - + @@ -151135,61 +153278,61 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -151198,16 +153341,16 @@ - + - + - + - + @@ -151506,7 +153649,7 @@ - + @@ -151596,7 +153739,7 @@ - + @@ -153201,7 +155344,7 @@ - + @@ -155069,219 +157212,219 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -155292,14 +157435,14 @@ - + - + @@ -155307,7 +157450,7 @@ - + @@ -155320,46 +157463,46 @@ - + - + - + - + - + - + - + @@ -155367,7 +157510,7 @@ - + @@ -155376,59 +157519,59 @@ - + - + - + - + - + - + - + - + - + @@ -155436,26 +157579,26 @@ - + - + - + - + @@ -155464,7 +157607,7 @@ - + @@ -155472,7 +157615,7 @@ - + @@ -155481,7 +157624,7 @@ - + @@ -155491,7 +157634,7 @@ - + @@ -155499,7 +157642,7 @@ - + @@ -155640,12 +157783,12 @@ - + - + @@ -155700,7 +157843,7 @@ - + @@ -156220,7 +158363,7 @@ - + @@ -156241,14 +158384,14 @@ - + - + @@ -156264,14 +158407,14 @@ - + - + @@ -156279,12 +158422,12 @@ - + - + @@ -156292,18 +158435,18 @@ - + - + - + @@ -156914,7 +159057,7 @@ - + @@ -157017,19 +159160,19 @@ - + - + - + @@ -157100,7 +159243,7 @@ - + @@ -157205,9 +159348,9 @@ - + - + @@ -157248,62 +159391,62 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -157331,22 +159474,22 @@ - + - + - + - + - + - + @@ -157399,12 +159542,12 @@ - + - + - + @@ -157448,10 +159591,10 @@ - + - + @@ -157464,7 +159607,7 @@ - + @@ -157489,19 +159632,19 @@ - + - + - + - + - + @@ -157589,16 +159732,23 @@ - + - + + + + + + + + @@ -157666,7 +159816,7 @@ - + @@ -158540,6 +160690,14 @@ + + + + + + + + @@ -158576,9 +160734,9 @@ - - - + + + @@ -160069,7 +162227,7 @@ - + @@ -160115,19 +162273,19 @@ - + - + - + - + - + @@ -160136,10 +162294,10 @@ - + - + @@ -160152,29 +162310,29 @@ - + - + - + - + - + - + - + @@ -161426,7 +163584,7 @@ - + @@ -161472,19 +163630,19 @@ - + - + - + - + - + @@ -161493,10 +163651,10 @@ - + - + @@ -161681,7 +163839,7 @@ - + @@ -161697,22 +163855,22 @@ - + - + - + - + - + - + @@ -162116,29 +164274,29 @@ - + - + - + - + - + - + - + @@ -163915,7 +166073,7 @@ - + @@ -163946,16 +166104,16 @@ - + - + - + - + @@ -164068,152 +166226,152 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -164317,81 +166475,81 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -164405,195 +166563,207 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -164657,6 +166827,6 @@ diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index 729e97a071e9..299d350411e6 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -399,6 +399,7 @@ __dev_kfree_skb_any devm_add_action devm_backlight_device_register + devm_blk_ksm_init devm_clk_bulk_get devm_clk_bulk_get_all devm_clk_get @@ -1170,6 +1171,7 @@ irq_work_queue irq_work_queue_on irq_work_sync + is_dma_buf_file is_vmalloc_addr jiffies jiffies_to_msecs @@ -1921,8 +1923,10 @@ rproc_add_subdev rproc_alloc rproc_boot + rproc_coredump_add_custom_segment rproc_coredump_add_segment rproc_coredump_set_elf_info + rproc_coredump_using_sections rproc_del rproc_free rproc_get_by_child @@ -2662,5 +2666,15 @@ xa_find_after xa_load xa_store + xhci_alloc_command + xhci_alloc_erst + xhci_free_command + xhci_free_erst + xhci_get_endpoint_index + xhci_queue_stop_endpoint + xhci_ring_alloc + xhci_ring_cmd_db + xhci_ring_free + xhci_trb_virt_to_dma zone_watermark_ok zone_watermark_ok_safe diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index efbd7acf5105..a0d89af0fb3d 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1712,7 +1712,6 @@ endmenu config ARM64_SVE bool "ARM Scalable Vector Extension support" default y - depends on !KVM || ARM64_VHE help The Scalable Vector Extension (SVE) is an extension to the AArch64 execution state which complements and extends the SIMD functionality @@ -1741,12 +1740,6 @@ config ARM64_SVE booting the kernel. If unsure and you are not observing these symptoms, you should assume that it is safe to say Y. - CPUs that support SVE are architecturally required to support the - Virtualization Host Extensions (VHE), so the kernel makes no - provision for supporting SVE alongside KVM without VHE enabled. - Thus, you will need to enable CONFIG_ARM64_VHE if you want to support - KVM in the same kernel image. - config ARM64_MODULE_PLTS bool "Use PLTs to allow module memory to spill over into vmalloc area" depends on MODULES diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 2da979bb515d..02199e267fa3 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -86,8 +86,6 @@ CONFIG_CRYPTO_AES_ARM64_CE_BLK=y CONFIG_KPROBES=y CONFIG_JUMP_LABEL=y CONFIG_SHADOW_CALL_STACK=y -CONFIG_LTO_CLANG_FULL=y -CONFIG_CFI_CLANG=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y @@ -503,6 +501,9 @@ CONFIG_IIO_BUFFER=y CONFIG_IIO_TRIGGER=y CONFIG_PWM=y CONFIG_GENERIC_PHY=y +CONFIG_POWERCAP=y +CONFIG_DTPM=y +CONFIG_DTPM_CPU=y CONFIG_RAS=y CONFIG_ANDROID=y CONFIG_ANDROID_BINDER_IPC=y @@ -619,17 +620,13 @@ CONFIG_DEBUG_INFO_DWARF4=y CONFIG_HEADERS_INSTALL=y # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y -CONFIG_UBSAN=y -CONFIG_UBSAN_TRAP=y -CONFIG_UBSAN_LOCAL_BOUNDS=y -# CONFIG_UBSAN_MISC is not set CONFIG_PAGE_OWNER=y CONFIG_DEBUG_STACK_USAGE=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_KASAN=y CONFIG_KASAN_HW_TAGS=y CONFIG_KFENCE=y -CONFIG_KFENCE_SAMPLE_INTERVAL=0 +CONFIG_KFENCE_SAMPLE_INTERVAL=500 CONFIG_PANIC_ON_OOPS=y CONFIG_PANIC_TIMEOUT=-1 CONFIG_WQ_WATCHDOG=y diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 8cded93f99c3..ac4e833a559f 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -676,11 +676,11 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU .endm /* - * Set SCTLR_EL1 to the passed value, and invalidate the local icache + * Set SCTLR_ELx to the @reg value, and invalidate the local icache * in the process. This is called when setting the MMU on. */ -.macro set_sctlr_el1, reg - msr sctlr_el1, \reg +.macro set_sctlr, sreg, reg + msr \sreg, \reg isb /* * Invalidate the local I-cache so that any instructions fetched @@ -692,6 +692,14 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU isb .endm +.macro set_sctlr_el1, reg + set_sctlr sctlr_el1, \reg +.endm + +.macro set_sctlr_el2, reg + set_sctlr sctlr_el2, \reg +.endm + /* * Check whether to yield to another runnable task from kernel mode NEON code * (which runs with preemption disabled). diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 87fc3b173677..58949ca811fd 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -607,6 +607,7 @@ void check_local_cpu_capabilities(void); u64 read_sanitised_ftr_reg(u32 id); u64 __read_sysreg_by_encoding(u32 sys_id); +int copy_ftr_reg(u32 id, struct arm64_ftr_reg *dst); static inline bool cpu_supports_mixed_endian_el0(void) { diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index bec5f14b622a..05c9c55768b8 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -130,6 +130,15 @@ static inline void sve_user_enable(void) sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN); } +#define sve_cond_update_zcr_vq(val, reg) \ + do { \ + u64 __zcr = read_sysreg_s((reg)); \ + u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \ + __new |= (val) & ZCR_ELx_LEN_MASK; \ + if (__zcr != __new) \ + write_sysreg_s(__new, (reg)); \ + } while (0) + /* * Probing and setup functions. * Calls to these functions must be serialised with one another. diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index af43367534c7..a2563992d2dc 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -6,6 +6,8 @@ * Author: Catalin Marinas */ +#include + .macro fpsimd_save state, tmpnr stp q0, q1, [\state, #16 * 0] stp q2, q3, [\state, #16 * 2] @@ -230,8 +232,7 @@ str w\nxtmp, [\xpfpsr, #4] .endm -.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2 - sve_load_vq \xvqminus1, x\nxtmp, \xtmp2 +.macro __sve_load nxbase, xpfpsr, nxtmp _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34 _sve_ldr_p 0, \nxbase _sve_wrffr 0 @@ -242,3 +243,8 @@ ldr w\nxtmp, [\xpfpsr, #4] msr fpcr, x\nxtmp .endm + +.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2 + sve_load_vq \xvqminus1, x\nxtmp, \xtmp2 + __sve_load \nxbase, \xpfpsr, \nxtmp +.endm diff --git a/arch/arm64/include/asm/hyp_image.h b/arch/arm64/include/asm/hyp_image.h index 737ded6b6d0d..b4b3076a76fb 100644 --- a/arch/arm64/include/asm/hyp_image.h +++ b/arch/arm64/include/asm/hyp_image.h @@ -10,11 +10,15 @@ #define __HYP_CONCAT(a, b) a ## b #define HYP_CONCAT(a, b) __HYP_CONCAT(a, b) +#ifndef __KVM_NVHE_HYPERVISOR__ /* * KVM nVHE code has its own symbol namespace prefixed with __kvm_nvhe_, * to separate it from the kernel proper. */ #define kvm_nvhe_sym(sym) __kvm_nvhe_##sym +#else +#define kvm_nvhe_sym(sym) sym +#endif #ifdef LINKER_SCRIPT @@ -56,6 +60,9 @@ */ #define KVM_NVHE_ALIAS(sym) kvm_nvhe_sym(sym) = sym; +/* Defines a linker script alias for KVM nVHE hyp symbols */ +#define KVM_NVHE_ALIAS_HYP(first, sec) kvm_nvhe_sym(first) = kvm_nvhe_sym(sec); + #endif /* LINKER_SCRIPT */ #endif /* __ARM64_HYP_IMAGE_H__ */ diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 99dc28f9876f..c453b829783d 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -57,6 +57,12 @@ #define __KVM_HOST_SMCCC_FUNC___kvm_get_mdcr_el2 12 #define __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs 13 #define __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_aprs 14 +#define __KVM_HOST_SMCCC_FUNC___pkvm_init 15 +#define __KVM_HOST_SMCCC_FUNC___pkvm_create_mappings 16 +#define __KVM_HOST_SMCCC_FUNC___pkvm_create_private_mapping 17 +#define __KVM_HOST_SMCCC_FUNC___pkvm_cpu_set_vector 18 +#define __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize 19 +#define __KVM_HOST_SMCCC_FUNC___pkvm_mark_hyp 20 #ifndef __ASSEMBLY__ @@ -154,6 +160,9 @@ struct kvm_nvhe_init_params { unsigned long tpidr_el2; unsigned long stack_hyp_va; phys_addr_t pgd_pa; + unsigned long hcr_el2; + unsigned long vttbr; + unsigned long vtcr; }; /* Translate a kernel address @ptr into its equivalent linear mapping */ diff --git a/arch/arm64/include/asm/kvm_cpufeature.h b/arch/arm64/include/asm/kvm_cpufeature.h new file mode 100644 index 000000000000..48cba6cecd71 --- /dev/null +++ b/arch/arm64/include/asm/kvm_cpufeature.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 - Google LLC + * Author: Quentin Perret + */ + +#include + +#ifndef KVM_HYP_CPU_FTR_REG +#if defined(__KVM_NVHE_HYPERVISOR__) +#define KVM_HYP_CPU_FTR_REG(name) extern struct arm64_ftr_reg name +#else +#define KVM_HYP_CPU_FTR_REG(name) extern struct arm64_ftr_reg kvm_nvhe_sym(name) +#endif +#endif + +KVM_HYP_CPU_FTR_REG(arm64_ftr_reg_ctrel0); +KVM_HYP_CPU_FTR_REG(arm64_ftr_reg_id_aa64mmfr0_el1); +KVM_HYP_CPU_FTR_REG(arm64_ftr_reg_id_aa64mmfr1_el1); diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 528e5d8a425a..0a68de8703d0 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -95,7 +95,7 @@ struct kvm_s2_mmu { /* The last vcpu id that ran on each physical CPU */ int __percpu *last_vcpu_ran; - struct kvm *kvm; + struct kvm_arch *arch; }; struct kvm_arch_memory_slot { @@ -375,8 +375,10 @@ struct kvm_vcpu_arch { }; /* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */ -#define vcpu_sve_pffr(vcpu) ((void *)((char *)((vcpu)->arch.sve_state) + \ - sve_ffr_offset((vcpu)->arch.sve_max_vl))) +#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \ + sve_ffr_offset((vcpu)->arch.sve_max_vl)) + +#define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl) #define vcpu_sve_state_size(vcpu) ({ \ size_t __size_ret; \ @@ -385,7 +387,7 @@ struct kvm_vcpu_arch { if (WARN_ON(!sve_vl_valid((vcpu)->arch.sve_max_vl))) { \ __size_ret = 0; \ } else { \ - __vcpu_vq = sve_vq_from_vl((vcpu)->arch.sve_max_vl); \ + __vcpu_vq = vcpu_sve_max_vq(vcpu); \ __size_ret = SVE_SIG_REGS_SIZE(__vcpu_vq); \ } \ \ @@ -594,6 +596,7 @@ int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); void kvm_arm_halt_guest(struct kvm *kvm); void kvm_arm_resume_guest(struct kvm *kvm); +#ifndef __KVM_NVHE_HYPERVISOR__ #define kvm_call_hyp_nvhe(f, ...) \ ({ \ struct arm_smccc_res res; \ @@ -633,6 +636,11 @@ void kvm_arm_resume_guest(struct kvm *kvm); \ ret; \ }) +#else /* __KVM_NVHE_HYPERVISOR__ */ +#define kvm_call_hyp(f, ...) f(__VA_ARGS__) +#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__) +#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__) +#endif /* __KVM_NVHE_HYPERVISOR__ */ void force_vm_exit(const cpumask_t *mask); void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot); @@ -695,19 +703,6 @@ static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt) ctxt_sys_reg(cpu_ctxt, MPIDR_EL1) = read_cpuid_mpidr(); } -static inline bool kvm_arch_requires_vhe(void) -{ - /* - * The Arm architecture specifies that implementation of SVE - * requires VHE also to be implemented. The KVM code for arm64 - * relies on this when SVE is present: - */ - if (system_supports_sve()) - return true; - - return false; -} - void kvm_arm_vcpu_ptrauth_trap(struct kvm_vcpu *vcpu); static inline void kvm_arch_hardware_unsetup(void) {} @@ -748,9 +743,13 @@ void kvm_clr_pmu_events(u32 clr); void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu); void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu); + +void setup_kvm_el2_caps(void); #else static inline void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr) {} static inline void kvm_clr_pmu_events(u32 clr) {} + +static inline void setup_kvm_el2_caps(void) {} #endif void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu); @@ -774,5 +773,12 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); (test_bit(KVM_ARM_VCPU_PMU_V3, (vcpu)->arch.features)) int kvm_trng_call(struct kvm_vcpu *vcpu); +#ifdef CONFIG_KVM +extern phys_addr_t hyp_mem_base; +extern phys_addr_t hyp_mem_size; +void __init kvm_hyp_reserve(void); +#else +static inline void kvm_hyp_reserve(void) { } +#endif #endif /* __ARM64_KVM_HOST_H__ */ diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 32ae676236b6..de40a565d7e5 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -90,6 +90,8 @@ void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu); void __fpsimd_save_state(struct user_fpsimd_state *fp_regs); void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs); +void __sve_save_state(void *sve_pffr, u32 *fpsr); +void __sve_restore_state(void *sve_pffr, u32 *fpsr); #ifndef __KVM_NVHE_HYPERVISOR__ void activate_traps_vhe_load(struct kvm_vcpu *vcpu); @@ -106,4 +108,12 @@ void __noreturn __hyp_do_panic(struct kvm_cpu_context *host_ctxt, u64 spsr, u64 elr, u64 par); #endif +#ifdef __KVM_NVHE_HYPERVISOR__ +void __pkvm_init_switch_pgd(phys_addr_t phys, unsigned long size, + phys_addr_t pgd, void *sp, void *cont_fn); +int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus, + unsigned long *per_cpu_base, u32 hyp_va_bits); +void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); +#endif + #endif /* __ARM64_KVM_HYP_H__ */ diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 90873851f677..9d64fa73ee67 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -166,7 +166,15 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu); phys_addr_t kvm_mmu_get_httbr(void); phys_addr_t kvm_get_idmap_vector(void); -int kvm_mmu_init(void); +int kvm_mmu_init(u32 *hyp_va_bits); + +static inline void *__kvm_vector_slot2addr(void *base, + enum arm64_hyp_spectre_vector slot) +{ + int idx = slot - (slot != HYP_VECTOR_DIRECT); + + return base + (idx * SZ_2K); +} struct kvm; @@ -262,9 +270,9 @@ static __always_inline u64 kvm_get_vttbr(struct kvm_s2_mmu *mmu) * Must be called from hyp code running at EL2 with an updated VTTBR * and interrupts disabled. */ -static __always_inline void __load_guest_stage2(struct kvm_s2_mmu *mmu) +static __always_inline void __load_stage2(struct kvm_s2_mmu *mmu, unsigned long vtcr) { - write_sysreg(kern_hyp_va(mmu->kvm)->arch.vtcr, vtcr_el2); + write_sysreg(vtcr, vtcr_el2); write_sysreg(kvm_get_vttbr(mmu), vttbr_el2); /* @@ -275,5 +283,14 @@ static __always_inline void __load_guest_stage2(struct kvm_s2_mmu *mmu) asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); } +static __always_inline void __load_guest_stage2(struct kvm_s2_mmu *mmu) +{ + __load_stage2(mmu, kern_hyp_va(mmu->arch)->vtcr); +} + +static inline struct kvm *kvm_s2_mmu_to_kvm(struct kvm_s2_mmu *mmu) +{ + return container_of(mmu->arch, struct kvm, arch); +} #endif /* __ASSEMBLY__ */ #endif /* __ARM64_KVM_MMU_H__ */ diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 8886d43cfb11..7382bdfb6284 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -11,22 +11,77 @@ #include #include +#define KVM_PGTABLE_MAX_LEVELS 4U + +static inline u64 kvm_get_parange(u64 mmfr0) +{ + u64 parange = cpuid_feature_extract_unsigned_field(mmfr0, + ID_AA64MMFR0_PARANGE_SHIFT); + if (parange > ID_AA64MMFR0_PARANGE_MAX) + parange = ID_AA64MMFR0_PARANGE_MAX; + + return parange; +} + typedef u64 kvm_pte_t; +/** + * struct kvm_pgtable_mm_ops - Memory management callbacks. + * @zalloc_page: Allocate a single zeroed memory page. The @arg parameter + * can be used by the walker to pass a memcache. The + * initial refcount of the page is 1. + * @zalloc_pages_exact: Allocate an exact number of zeroed memory pages. The + * @size parameter is in bytes, and is rounded-up to the + * next page boundary. The resulting allocation is + * physically contiguous. + * @free_pages_exact: Free an exact number of memory pages previously + * allocated by zalloc_pages_exact. + * @get_page: Increment the refcount on a page. + * @put_page: Decrement the refcount on a page. When the refcount + * reaches 0 the page is automatically freed. + * @page_count: Return the refcount of a page. + * @phys_to_virt: Convert a physical address into a virtual address mapped + * in the current context. + * @virt_to_phys: Convert a virtual address mapped in the current context + * into a physical address. + */ +struct kvm_pgtable_mm_ops { + void* (*zalloc_page)(void *arg); + void* (*zalloc_pages_exact)(size_t size); + void (*free_pages_exact)(void *addr, size_t size); + void (*get_page)(void *addr); + void (*put_page)(void *addr); + int (*page_count)(void *addr); + void* (*phys_to_virt)(phys_addr_t phys); + phys_addr_t (*virt_to_phys)(void *addr); +}; + +/** + * enum kvm_pgtable_stage2_flags - Stage-2 page-table flags. + * @KVM_PGTABLE_S2_NOFWB: Don't enforce Normal-WB even if the CPUs have + * ARM64_HAS_STAGE2_FWB. + */ +enum kvm_pgtable_stage2_flags { + KVM_PGTABLE_S2_NOFWB = BIT(0), +}; + /** * struct kvm_pgtable - KVM page-table. * @ia_bits: Maximum input address size, in bits. * @start_level: Level at which the page-table walk starts. * @pgd: Pointer to the first top-level entry of the page-table. + * @mm_ops: Memory management callbacks. * @mmu: Stage-2 KVM MMU struct. Unused for stage-1 page-tables. */ struct kvm_pgtable { u32 ia_bits; u32 start_level; kvm_pte_t *pgd; + struct kvm_pgtable_mm_ops *mm_ops; /* Stage-2 only */ struct kvm_s2_mmu *mmu; + enum kvm_pgtable_stage2_flags flags; }; /** @@ -49,6 +104,16 @@ enum kvm_pgtable_prot { #define PAGE_HYP_RO (KVM_PGTABLE_PROT_R) #define PAGE_HYP_DEVICE (PAGE_HYP | KVM_PGTABLE_PROT_DEVICE) +/** + * struct kvm_mem_range - Range of Intermediate Physical Addresses + * @start: Start of the range. + * @end: End of the range. + */ +struct kvm_mem_range { + u64 start; + u64 end; +}; + /** * enum kvm_pgtable_walk_flags - Flags to control a depth-first page-table walk. * @KVM_PGTABLE_WALK_LEAF: Visit leaf entries, including invalid @@ -86,10 +151,12 @@ struct kvm_pgtable_walker { * kvm_pgtable_hyp_init() - Initialise a hypervisor stage-1 page-table. * @pgt: Uninitialised page-table structure to initialise. * @va_bits: Maximum virtual address bits. + * @mm_ops: Memory management callbacks. * * Return: 0 on success, negative error code on failure. */ -int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits); +int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits, + struct kvm_pgtable_mm_ops *mm_ops); /** * kvm_pgtable_hyp_destroy() - Destroy an unused hypervisor stage-1 page-table. @@ -122,14 +189,38 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt); int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, enum kvm_pgtable_prot prot); +/** + * kvm_get_vtcr() - Helper to construct VTCR_EL2 + * @mmfr0: Sanitized value of SYS_ID_AA64MMFR0_EL1 register. + * @mmfr1: Sanitized value of SYS_ID_AA64MMFR1_EL1 register. + * @phys_shfit: Value to set in VTCR_EL2.T0SZ. + * + * The VTCR value is common across all the physical CPUs on the system. + * We use system wide sanitised values to fill in different fields, + * except for Hardware Management of Access Flags. HA Flag is set + * unconditionally on all CPUs, as it is safe to run with or without + * the feature and the bit is RES0 on CPUs that don't support it. + * + * Return: VTCR_EL2 value + */ +u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift); + /** * kvm_pgtable_stage2_init() - Initialise a guest stage-2 page-table. * @pgt: Uninitialised page-table structure to initialise. - * @kvm: KVM structure representing the guest virtual machine. + * @arch: Arch-specific KVM structure representing the guest virtual + * machine. + * @mm_ops: Memory management callbacks. + * @flags: Stage-2 configuration flags. * * Return: 0 on success, negative error code on failure. */ -int kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm *kvm); +int kvm_pgtable_stage2_init_flags(struct kvm_pgtable *pgt, struct kvm_arch *arch, + struct kvm_pgtable_mm_ops *mm_ops, + enum kvm_pgtable_stage2_flags flags); + +#define kvm_pgtable_stage2_init(pgt, arch, mm_ops) \ + kvm_pgtable_stage2_init_flags(pgt, arch, mm_ops, 0) /** * kvm_pgtable_stage2_destroy() - Destroy an unused guest stage-2 page-table. @@ -147,8 +238,8 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt); * @size: Size of the mapping. * @phys: Physical address of the memory to map. * @prot: Permissions and attributes for the mapping. - * @mc: Cache of pre-allocated GFP_PGTABLE_USER memory from which to - * allocate page-table pages. + * @mc: Cache of pre-allocated and zeroed memory from which to allocate + * page-table pages. * * The offset of @addr within a page is ignored, @size is rounded-up to * the next page boundary and @phys is rounded-down to the previous page @@ -170,7 +261,28 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt); */ int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, enum kvm_pgtable_prot prot, - struct kvm_mmu_memory_cache *mc); + void *mc); + +/** + * kvm_pgtable_stage2_set_owner() - Annotate invalid mappings with metadata + * encoding the ownership of a page in the + * IPA space. + * @pgt: Page-table structure initialised by kvm_pgtable_stage2_init(). + * @addr: Base intermediate physical address to annotate. + * @size: Size of the annotated range. + * @mc: Cache of pre-allocated and zeroed memory from which to allocate + * page-table pages. + * @owner_id: Unique identifier for the owner of the page. + * + * By default, all page-tables are owned by identifier 0. This function can be + * used to mark portions of the IPA space as owned by other entities. When a + * stage 2 is used with identity-mappings, these annotations allow to use the + * page-table data structure as a simple rmap. + * + * Return: 0 on success, negative error code on failure. + */ +int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size, + void *mc, u8 owner_id); /** * kvm_pgtable_stage2_unmap() - Remove a mapping from a guest stage-2 page-table. @@ -311,4 +423,23 @@ int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size); int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, struct kvm_pgtable_walker *walker); +/** + * kvm_pgtable_stage2_find_range() - Find a range of Intermediate Physical + * Addresses with compatible permission + * attributes. + * @pgt: Page-table structure initialised by kvm_pgtable_stage2_init(). + * @addr: Address that must be covered by the range. + * @prot: Protection attributes that the range must be compatible with. + * @range: Range structure used to limit the search space at call time and + * that will hold the result. + * + * The offset of @addr within a page is ignored. An IPA is compatible with @prot + * iff its corresponding stage-2 page-table entry has default ownership and, if + * valid, is mapped with protection attributes identical to @prot. + * + * Return: 0 on success, negative error code on failure. + */ +int kvm_pgtable_stage2_find_range(struct kvm_pgtable *pgt, u64 addr, + enum kvm_pgtable_prot prot, + struct kvm_mem_range *range); #endif /* __ARM64_KVM_PGTABLE_H__ */ diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index 9a65fb528110..079f4e9a3e84 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -71,10 +71,10 @@ extern bool arm64_use_ng_mappings; #define PAGE_KERNEL_EXEC __pgprot(PROT_NORMAL & ~PTE_PXN) #define PAGE_KERNEL_EXEC_CONT __pgprot((PROT_NORMAL & ~PTE_PXN) | PTE_CONT) -#define PAGE_S2_MEMATTR(attr) \ +#define PAGE_S2_MEMATTR(attr, has_fwb) \ ({ \ u64 __val; \ - if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) \ + if (has_fwb) \ __val = PTE_S2_MEMATTR(MT_S2_FWB_ ## attr); \ else \ __val = PTE_S2_MEMATTR(MT_S2_ ## attr); \ diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h index 2f36b16a5b5d..e4ad9db53af1 100644 --- a/arch/arm64/include/asm/sections.h +++ b/arch/arm64/include/asm/sections.h @@ -13,6 +13,7 @@ extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[]; extern char __hyp_text_start[], __hyp_text_end[]; extern char __hyp_rodata_start[], __hyp_rodata_end[]; extern char __hyp_reloc_begin[], __hyp_reloc_end[]; +extern char __hyp_bss_start[], __hyp_bss_end[]; extern char __idmap_text_start[], __idmap_text_end[]; extern char __initdata_begin[], __initdata_end[]; extern char __inittext_begin[], __inittext_end[]; diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 269347146a66..1093196db31d 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -624,9 +624,6 @@ #define SCTLR_ELx_A (BIT(1)) #define SCTLR_ELx_M (BIT(0)) -#define SCTLR_ELx_FLAGS (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | \ - SCTLR_ELx_SA | SCTLR_ELx_I | SCTLR_ELx_IESB) - /* SCTLR_EL2 specific flags. */ #define SCTLR_EL2_RES1 ((BIT(4)) | (BIT(5)) | (BIT(11)) | (BIT(16)) | \ (BIT(18)) | (BIT(22)) | (BIT(23)) | (BIT(28)) | \ @@ -638,6 +635,10 @@ #define ENDIAN_SET_EL2 0 #endif +#define INIT_SCTLR_EL2_MMU_ON \ + (SCTLR_ELx_M | SCTLR_ELx_C | SCTLR_ELx_SA | SCTLR_ELx_I | \ + SCTLR_ELx_IESB | SCTLR_ELx_WXN | ENDIAN_SET_EL2 | SCTLR_EL2_RES1) + #define INIT_SCTLR_EL2_MMU_OFF \ (SCTLR_EL2_RES1 | ENDIAN_SET_EL2) diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 826069a8b247..0050f113ce99 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -121,6 +121,9 @@ int main(void) DEFINE(NVHE_INIT_TPIDR_EL2, offsetof(struct kvm_nvhe_init_params, tpidr_el2)); DEFINE(NVHE_INIT_STACK_HYP_VA, offsetof(struct kvm_nvhe_init_params, stack_hyp_va)); DEFINE(NVHE_INIT_PGD_PA, offsetof(struct kvm_nvhe_init_params, pgd_pa)); + DEFINE(NVHE_INIT_HCR_EL2, offsetof(struct kvm_nvhe_init_params, hcr_el2)); + DEFINE(NVHE_INIT_VTTBR, offsetof(struct kvm_nvhe_init_params, vttbr)); + DEFINE(NVHE_INIT_VTCR, offsetof(struct kvm_nvhe_init_params, vtcr)); #endif #ifdef CONFIG_CPU_PM DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); diff --git a/arch/arm64/kernel/cpu-reset.S b/arch/arm64/kernel/cpu-reset.S index 37721eb6f9a1..d47ff63a5b66 100644 --- a/arch/arm64/kernel/cpu-reset.S +++ b/arch/arm64/kernel/cpu-reset.S @@ -30,10 +30,7 @@ * flat identity mapping. */ SYM_CODE_START(__cpu_soft_restart) - /* Clear sctlr_el1 flags. */ - mrs x12, sctlr_el1 - mov_q x13, SCTLR_ELx_FLAGS - bic x12, x12, x13 + mov_q x12, INIT_SCTLR_EL1_MMU_OFF pre_disable_mmu_workaround /* * either disable EL1&0 translation regime or disable EL2&0 translation diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 31489f6bc5af..ea573522c5eb 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1199,6 +1199,18 @@ u64 read_sanitised_ftr_reg(u32 id) } EXPORT_SYMBOL_GPL(read_sanitised_ftr_reg); +int copy_ftr_reg(u32 id, struct arm64_ftr_reg *dst) +{ + struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id); + + if (!regp) + return -EINVAL; + + *dst = *regp; + + return 0; +} + #define read_sysreg_case(r) \ case r: val = read_sysreg_s(r); break; @@ -2930,6 +2942,7 @@ void __init setup_cpu_features(void) setup_system_capabilities(); setup_elf_hwcaps(arm64_elf_hwcaps); + setup_kvm_el2_caps(); if (system_supports_32bit_el0()) setup_elf_hwcaps(compat_elf_hwcaps); diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index 5aa9ed1e9ec6..d5dc2b792651 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -104,6 +104,36 @@ KVM_NVHE_ALIAS(kvm_arm_hyp_percpu_base); /* PMU available static key */ KVM_NVHE_ALIAS(kvm_arm_pmu_available); +/* Position-independent library routines */ +KVM_NVHE_ALIAS_HYP(clear_page, __pi_clear_page); +KVM_NVHE_ALIAS_HYP(copy_page, __pi_copy_page); +KVM_NVHE_ALIAS_HYP(memcpy, __pi_memcpy); +KVM_NVHE_ALIAS_HYP(memset, __pi_memset); + +#ifdef CONFIG_KASAN +KVM_NVHE_ALIAS_HYP(__memcpy, __pi_memcpy); +KVM_NVHE_ALIAS_HYP(__memset, __pi_memset); +#endif + +/* Kernel memory sections */ +KVM_NVHE_ALIAS(__start_rodata); +KVM_NVHE_ALIAS(__end_rodata); +KVM_NVHE_ALIAS(__bss_start); +KVM_NVHE_ALIAS(__bss_stop); + +/* Hyp memory sections */ +KVM_NVHE_ALIAS(__hyp_idmap_text_start); +KVM_NVHE_ALIAS(__hyp_idmap_text_end); +KVM_NVHE_ALIAS(__hyp_text_start); +KVM_NVHE_ALIAS(__hyp_text_end); +KVM_NVHE_ALIAS(__hyp_bss_start); +KVM_NVHE_ALIAS(__hyp_bss_end); +KVM_NVHE_ALIAS(__hyp_rodata_start); +KVM_NVHE_ALIAS(__hyp_rodata_end); + +/* pKVM static key */ +KVM_NVHE_ALIAS(kvm_protected_mode_initialized); + #endif /* CONFIG_KVM */ #endif /* __ARM64_KERNEL_IMAGE_VARS_H */ diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 08156be75569..e5e65f785c43 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -46,6 +46,8 @@ #include #include +#include + static const char *handler[]= { "Synchronous Abort", "IRQ", @@ -404,6 +406,7 @@ void do_undefinstr(struct pt_regs *regs) if (call_undef_hook(regs) == 0) return; + trace_android_rvh_do_undefinstr(regs, user_mode(regs)); BUG_ON(!user_mode(regs)); force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); } @@ -765,6 +768,7 @@ asmlinkage void notrace bad_mode(struct pt_regs *regs, int reason, unsigned int handler[reason], smp_processor_id(), esr, esr_get_class_string(esr)); + trace_android_rvh_bad_mode(regs, esr, reason); __show_regs(regs); local_daif_mask(); panic("bad mode"); @@ -830,6 +834,8 @@ void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr) pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n", smp_processor_id(), esr, esr_get_class_string(esr)); + + trace_android_rvh_arm64_serror_panic(regs, esr); if (regs) __show_regs(regs); diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 93cef9607c0e..f86d781e7180 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -5,24 +5,7 @@ * Written by Martin Mares */ -#define RO_EXCEPTION_TABLE_ALIGN 8 -#define RUNTIME_DISCARD_EXIT - -#include -#include #include -#include -#include -#include - -#include "image.h" - -OUTPUT_ARCH(aarch64) -ENTRY(_text) - -jiffies = jiffies_64; - - #ifdef CONFIG_KVM #define HYPERVISOR_EXTABLE \ . = ALIGN(SZ_8); \ @@ -32,9 +15,11 @@ jiffies = jiffies_64; #define HYPERVISOR_DATA_SECTIONS \ HYP_SECTION_NAME(.rodata) : { \ + . = ALIGN(PAGE_SIZE); \ __hyp_rodata_start = .; \ *(HYP_SECTION_NAME(.data..ro_after_init)) \ *(HYP_SECTION_NAME(.rodata)) \ + . = ALIGN(PAGE_SIZE); \ __hyp_rodata_end = .; \ } @@ -51,29 +36,52 @@ jiffies = jiffies_64; __hyp_reloc_end = .; \ } +#define BSS_FIRST_SECTIONS \ + __hyp_bss_start = .; \ + *(HYP_SECTION_NAME(.bss)) \ + . = ALIGN(PAGE_SIZE); \ + __hyp_bss_end = .; + +/* + * We require that __hyp_bss_start and __bss_start are aligned, and enforce it + * with an assertion. But the BSS_SECTION macro places an empty .sbss section + * between them, which can in some cases cause the linker to misalign them. To + * work around the issue, force a page alignment for __bss_start. + */ +#define SBSS_ALIGN PAGE_SIZE #else /* CONFIG_KVM */ #define HYPERVISOR_EXTABLE #define HYPERVISOR_DATA_SECTIONS #define HYPERVISOR_PERCPU_SECTION #define HYPERVISOR_RELOC_SECTION +#define SBSS_ALIGN 0 #endif +#define RO_EXCEPTION_TABLE_ALIGN 8 +#define RUNTIME_DISCARD_EXIT + +#include +#include +#include +#include +#include + +#include "image.h" + +OUTPUT_ARCH(aarch64) +ENTRY(_text) + +jiffies = jiffies_64; + #define HYPERVISOR_TEXT \ - /* \ - * Align to 4 KB so that \ - * a) the HYP vector table is at its minimum \ - * alignment of 2048 bytes \ - * b) the HYP init code will not cross a page \ - * boundary if its size does not exceed \ - * 4 KB (see related ASSERT() below) \ - */ \ - . = ALIGN(SZ_4K); \ + . = ALIGN(PAGE_SIZE); \ __hyp_idmap_text_start = .; \ *(.hyp.idmap.text) \ __hyp_idmap_text_end = .; \ __hyp_text_start = .; \ *(.hyp.text) \ HYPERVISOR_EXTABLE \ + . = ALIGN(PAGE_SIZE); \ __hyp_text_end = .; #define IDMAP_TEXT \ @@ -278,7 +286,7 @@ SECTIONS __pecoff_data_rawsize = ABSOLUTE(. - __initdata_begin); _edata = .; - BSS_SECTION(0, 0, 0) + BSS_SECTION(SBSS_ALIGN, 0, 0) . = ALIGN(PAGE_SIZE); init_pg_dir = .; @@ -311,11 +319,12 @@ SECTIONS #include "image-vars.h" /* - * The HYP init code and ID map text can't be longer than a page each, - * and should not cross a page boundary. + * The HYP init code and ID map text can't be longer than a page each. The + * former is page-aligned, but the latter may not be with 16K or 64K pages, so + * it should also not cross a page boundary. */ -ASSERT(__hyp_idmap_text_end - (__hyp_idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K, - "HYP init code too big or misaligned") +ASSERT(__hyp_idmap_text_end - __hyp_idmap_text_start <= PAGE_SIZE, + "HYP init code too big") ASSERT(__idmap_text_end - (__idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K, "ID map text too big or misaligned") #ifdef CONFIG_HIBERNATION @@ -326,6 +335,9 @@ ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1)) ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) == PAGE_SIZE, "Entry trampoline text too big") #endif +#ifdef CONFIG_KVM +ASSERT(__hyp_bss_start == __bss_start, "HYP and Host BSS are misaligned") +#endif /* * If padding is applied before .head.text, virt<->phys conversions will fail. */ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index a822c7b1e5e3..9acf77589489 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1359,16 +1359,9 @@ static unsigned long nvhe_percpu_order(void) /* A lookup table holding the hypervisor VA for each vector slot */ static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS]; -static int __kvm_vector_slot2idx(enum arm64_hyp_spectre_vector slot) -{ - return slot - (slot != HYP_VECTOR_DIRECT); -} - static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot) { - int idx = __kvm_vector_slot2idx(slot); - - hyp_spectre_vector_selector[slot] = base + (idx * SZ_2K); + hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot); } static int kvm_init_vector_slots(void) @@ -1397,22 +1390,18 @@ static int kvm_init_vector_slots(void) return 0; } -static void cpu_init_hyp_mode(void) +static void cpu_prepare_hyp_mode(int cpu) { - struct kvm_nvhe_init_params *params = this_cpu_ptr_nvhe_sym(kvm_init_params); - struct arm_smccc_res res; + struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu); unsigned long tcr; - /* Switch from the HYP stub to our own HYP init vector */ - __hyp_set_vectors(kvm_get_idmap_vector()); - /* * Calculate the raw per-cpu offset without a translation from the * kernel's mapping to the linear mapping, and store it in tpidr_el2 * so that we can use adr_l to access per-cpu variables in EL2. * Also drop the KASAN tag which gets in the way... */ - params->tpidr_el2 = (unsigned long)kasan_reset_tag(this_cpu_ptr_nvhe_sym(__per_cpu_start)) - + params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) - (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start)); params->mair_el2 = read_sysreg(mair_el1); @@ -1436,14 +1425,28 @@ static void cpu_init_hyp_mode(void) tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET; params->tcr_el2 = tcr; - params->stack_hyp_va = kern_hyp_va(__this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE); + params->stack_hyp_va = kern_hyp_va(per_cpu(kvm_arm_hyp_stack_page, cpu) + PAGE_SIZE); params->pgd_pa = kvm_mmu_get_httbr(); + if (is_protected_kvm_enabled()) + params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS; + else + params->hcr_el2 = HCR_HOST_NVHE_FLAGS; + params->vttbr = params->vtcr = 0; /* * Flush the init params from the data cache because the struct will * be read while the MMU is off. */ kvm_flush_dcache_to_poc(params, sizeof(*params)); +} + +static void hyp_install_host_vector(void) +{ + struct kvm_nvhe_init_params *params; + struct arm_smccc_res res; + + /* Switch from the HYP stub to our own HYP init vector */ + __hyp_set_vectors(kvm_get_idmap_vector()); /* * Call initialization code, and switch to the full blown HYP code. @@ -1452,8 +1455,14 @@ static void cpu_init_hyp_mode(void) * cpus_have_const_cap() wrapper. */ BUG_ON(!system_capabilities_finalized()); + params = this_cpu_ptr_nvhe_sym(kvm_init_params); arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res); WARN_ON(res.a0 != SMCCC_RET_SUCCESS); +} + +static void cpu_init_hyp_mode(void) +{ + hyp_install_host_vector(); /* * Disabling SSBD on a non-VHE system requires us to enable SSBS @@ -1496,7 +1505,10 @@ static void cpu_set_hyp_vector(void) struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data); void *vector = hyp_spectre_vector_selector[data->slot]; - *this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector; + if (!is_protected_kvm_enabled()) + *this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector; + else + kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot); } static void cpu_hyp_reinit(void) @@ -1504,13 +1516,14 @@ static void cpu_hyp_reinit(void) kvm_init_host_cpu_context(&this_cpu_ptr_hyp_sym(kvm_host_data)->host_ctxt); cpu_hyp_reset(); - cpu_set_hyp_vector(); if (is_kernel_in_hyp_mode()) kvm_timer_init_vhe(); else cpu_init_hyp_mode(); + cpu_set_hyp_vector(); + kvm_arm_init_debug(); if (vgic_present) @@ -1706,18 +1719,59 @@ static void teardown_hyp_mode(void) } } +static int do_pkvm_init(u32 hyp_va_bits) +{ + void *per_cpu_base = kvm_ksym_ref(kvm_arm_hyp_percpu_base); + int ret; + + preempt_disable(); + hyp_install_host_vector(); + ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size, + num_possible_cpus(), kern_hyp_va(per_cpu_base), + hyp_va_bits); + preempt_enable(); + + return ret; +} + +static int kvm_hyp_init_protection(u32 hyp_va_bits) +{ + void *addr = phys_to_virt(hyp_mem_base); + int ret; + + ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP); + if (ret) + return ret; + + ret = do_pkvm_init(hyp_va_bits); + if (ret) + return ret; + + free_hyp_pgds(); + + return 0; +} + /** * Inits Hyp-mode on all online CPUs */ static int init_hyp_mode(void) { + u32 hyp_va_bits; int cpu; - int err = 0; + int err = -ENOMEM; + + /* + * The protected Hyp-mode cannot be initialized if the memory pool + * allocation has failed. + */ + if (is_protected_kvm_enabled() && !hyp_mem_base) + goto out_err; /* * Allocate Hyp PGD and setup Hyp identity mapping */ - err = kvm_mmu_init(); + err = kvm_mmu_init(&hyp_va_bits); if (err) goto out_err; @@ -1778,7 +1832,19 @@ static int init_hyp_mode(void) goto out_err; } - err = create_hyp_mappings(kvm_ksym_ref(__bss_start), + /* + * .hyp.bss is guaranteed to be placed at the beginning of the .bss + * section thanks to an assertion in the linker script. Map it RW and + * the rest of .bss RO. + */ + err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start), + kvm_ksym_ref(__hyp_bss_end), PAGE_HYP); + if (err) { + kvm_err("Cannot map hyp bss section: %d\n", err); + goto out_err; + } + + err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end), kvm_ksym_ref(__bss_stop), PAGE_HYP_RO); if (err) { kvm_err("Cannot map bss section\n"); @@ -1799,19 +1865,19 @@ static int init_hyp_mode(void) } } - /* - * Map Hyp percpu pages - */ for_each_possible_cpu(cpu) { char *percpu_begin = (char *)kvm_arm_hyp_percpu_base[cpu]; char *percpu_end = percpu_begin + nvhe_percpu_size(); + /* Map Hyp percpu pages */ err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP); - if (err) { kvm_err("Cannot map hyp percpu region\n"); goto out_err; } + + /* Prepare the CPU initialization parameters */ + cpu_prepare_hyp_mode(cpu); } if (is_protected_kvm_enabled()) { @@ -1821,6 +1887,14 @@ static int init_hyp_mode(void) goto out_err; } + if (is_protected_kvm_enabled()) { + err = kvm_hyp_init_protection(hyp_va_bits); + if (err) { + kvm_err("Failed to init hyp memory protection\n"); + goto out_err; + } + } + return 0; out_err: @@ -1829,6 +1903,72 @@ out_err: return err; } +void _kvm_host_prot_finalize(void *discard) +{ + WARN_ON(kvm_call_hyp_nvhe(__pkvm_prot_finalize)); +} + +static inline int pkvm_mark_hyp(phys_addr_t start, phys_addr_t end) +{ + return kvm_call_hyp_nvhe(__pkvm_mark_hyp, start, end); +} + +#define pkvm_mark_hyp_section(__section) \ + pkvm_mark_hyp(__pa_symbol(__section##_start), \ + __pa_symbol(__section##_end)) + +static int finalize_hyp_mode(void) +{ + int cpu, ret; + + if (!is_protected_kvm_enabled()) + return 0; + + ret = pkvm_mark_hyp_section(__hyp_idmap_text); + if (ret) + return ret; + + ret = pkvm_mark_hyp_section(__hyp_text); + if (ret) + return ret; + + ret = pkvm_mark_hyp_section(__hyp_rodata); + if (ret) + return ret; + + ret = pkvm_mark_hyp_section(__hyp_bss); + if (ret) + return ret; + + ret = pkvm_mark_hyp(hyp_mem_base, hyp_mem_base + hyp_mem_size); + if (ret) + return ret; + + for_each_possible_cpu(cpu) { + phys_addr_t start = virt_to_phys((void *)kvm_arm_hyp_percpu_base[cpu]); + phys_addr_t end = start + (PAGE_SIZE << nvhe_percpu_order()); + + ret = pkvm_mark_hyp(start, end); + if (ret) + return ret; + + start = virt_to_phys((void *)per_cpu(kvm_arm_hyp_stack_page, cpu)); + end = start + PAGE_SIZE; + ret = pkvm_mark_hyp(start, end); + if (ret) + return ret; + } + + /* + * Flip the static key upfront as that may no longer be possible + * once the host stage 2 is installed. + */ + static_branch_enable(&kvm_protected_mode_initialized); + on_each_cpu(_kvm_host_prot_finalize, NULL, 1); + + return 0; +} + static void check_kvm_target_cpu(void *ret) { *(int *)ret = kvm_target_cpu(); @@ -1903,11 +2043,6 @@ int kvm_arch_init(void *opaque) in_hyp_mode = is_kernel_in_hyp_mode(); - if (!in_hyp_mode && kvm_arch_requires_vhe()) { - kvm_pr_unimpl("CPU unsupported in non-VHE mode, not initializing\n"); - return -ENODEV; - } - if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) || cpus_have_final_cap(ARM64_WORKAROUND_1508412)) kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \ @@ -1945,8 +2080,15 @@ int kvm_arch_init(void *opaque) if (err) goto out_hyp; + if (!in_hyp_mode) { + err = finalize_hyp_mode(); + if (err) { + kvm_err("Failed to finalize Hyp protection\n"); + goto out_hyp; + } + } + if (is_protected_kvm_enabled()) { - static_branch_enable(&kvm_protected_mode_initialized); kvm_info("Protected nVHE mode initialized successfully\n"); } else if (in_hyp_mode) { kvm_info("VHE mode initialized successfully\n"); diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c index 3e081d556e81..5621020b28de 100644 --- a/arch/arm64/kvm/fpsimd.c +++ b/arch/arm64/kvm/fpsimd.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -42,6 +43,17 @@ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu) if (ret) goto error; + if (vcpu->arch.sve_state) { + void *sve_end; + + sve_end = vcpu->arch.sve_state + vcpu_sve_state_size(vcpu); + + ret = create_hyp_mappings(vcpu->arch.sve_state, sve_end, + PAGE_HYP); + if (ret) + goto error; + } + vcpu->arch.host_thread_info = kern_hyp_va(ti); vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd); error: @@ -109,11 +121,17 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) local_irq_save(flags); if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) { - fpsimd_save_and_flush_cpu_state(); + if (guest_has_sve) { + __vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); - if (guest_has_sve) - __vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_s(SYS_ZCR_EL12); - } else if (host_has_sve) { + /* Restore the VL that was saved when bound to the CPU */ + if (!has_vhe()) + sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, + SYS_ZCR_EL1); + } + + fpsimd_save_and_flush_cpu_state(); + } else if (has_vhe() && host_has_sve) { /* * The FPSIMD/SVE state in the CPU has not been touched, and we * have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 9bbd30e62799..c763808cacdf 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -299,7 +299,7 @@ static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) memset(vqs, 0, sizeof(vqs)); - max_vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); + max_vq = vcpu_sve_max_vq(vcpu); for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) if (sve_vq_available(vq)) vqs[vq_word(vq)] |= vq_mask(vq); @@ -427,7 +427,7 @@ static int sve_reg_to_region(struct sve_state_reg_region *region, if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) return -ENOENT; - vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); + vq = vcpu_sve_max_vq(vcpu); reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) - SVE_SIG_REGS_OFFSET; @@ -437,7 +437,7 @@ static int sve_reg_to_region(struct sve_state_reg_region *region, if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) return -ENOENT; - vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); + vq = vcpu_sve_max_vq(vcpu); reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) - SVE_SIG_REGS_OFFSET; diff --git a/arch/arm64/kvm/hyp/Makefile b/arch/arm64/kvm/hyp/Makefile index 687598e41b21..b726332eec49 100644 --- a/arch/arm64/kvm/hyp/Makefile +++ b/arch/arm64/kvm/hyp/Makefile @@ -10,4 +10,4 @@ subdir-ccflags-y := -I$(incdir) \ -DDISABLE_BRANCH_PROFILING \ $(DISABLE_STACKLEAK_PLUGIN) -obj-$(CONFIG_KVM) += vhe/ nvhe/ pgtable.o +obj-$(CONFIG_KVM) += vhe/ nvhe/ pgtable.o reserved_mem.o diff --git a/arch/arm64/kvm/hyp/fpsimd.S b/arch/arm64/kvm/hyp/fpsimd.S index 01f114aa47b0..3c635929771a 100644 --- a/arch/arm64/kvm/hyp/fpsimd.S +++ b/arch/arm64/kvm/hyp/fpsimd.S @@ -19,3 +19,13 @@ SYM_FUNC_START(__fpsimd_restore_state) fpsimd_restore x0, 1 ret SYM_FUNC_END(__fpsimd_restore_state) + +SYM_FUNC_START(__sve_restore_state) + __sve_load 0, x1, 2 + ret +SYM_FUNC_END(__sve_restore_state) + +SYM_FUNC_START(__sve_save_state) + sve_save 0, x1, 2 + ret +SYM_FUNC_END(__sve_save_state) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 6c1f51f25eb3..8d980a15df71 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -160,19 +160,9 @@ static inline bool __translate_far_to_hpfar(u64 far, u64 *hpfar) return true; } -static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) +static inline bool __get_fault_info(u64 esr, struct kvm_vcpu_fault_info *fault) { - u8 ec; - u64 esr; - u64 hpfar, far; - - esr = vcpu->arch.fault.esr_el2; - ec = ESR_ELx_EC(esr); - - if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW) - return true; - - far = read_sysreg_el2(SYS_FAR); + fault->far_el2 = read_sysreg_el2(SYS_FAR); /* * The HPFAR can be invalid if the stage 2 fault did not @@ -188,39 +178,63 @@ static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) if (!(esr & ESR_ELx_S1PTW) && (cpus_have_final_cap(ARM64_WORKAROUND_834220) || (esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) { - if (!__translate_far_to_hpfar(far, &hpfar)) + if (!__translate_far_to_hpfar(fault->far_el2, &fault->hpfar_el2)) return false; } else { - hpfar = read_sysreg(hpfar_el2); + fault->hpfar_el2 = read_sysreg(hpfar_el2); } - vcpu->arch.fault.far_el2 = far; - vcpu->arch.fault.hpfar_el2 = hpfar; return true; } +static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) +{ + u8 ec; + u64 esr; + + esr = vcpu->arch.fault.esr_el2; + ec = ESR_ELx_EC(esr); + + if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW) + return true; + + return __get_fault_info(esr, &vcpu->arch.fault); +} + +static inline void __hyp_sve_save_host(struct kvm_vcpu *vcpu) +{ + struct thread_struct *thread; + + thread = container_of(vcpu->arch.host_fpsimd_state, struct thread_struct, + uw.fpsimd_state); + + __sve_save_state(sve_pffr(thread), &vcpu->arch.host_fpsimd_state->fpsr); +} + +static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) +{ + sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2); + __sve_restore_state(vcpu_sve_pffr(vcpu), + &vcpu->arch.ctxt.fp_regs.fpsr); + write_sysreg_el1(__vcpu_sys_reg(vcpu, ZCR_EL1), SYS_ZCR); +} + /* Check for an FPSIMD/SVE trap and handle as appropriate */ static inline bool __hyp_handle_fpsimd(struct kvm_vcpu *vcpu) { - bool vhe, sve_guest, sve_host; + bool sve_guest, sve_host; u8 esr_ec; + u64 reg; if (!system_supports_fpsimd()) return false; - /* - * Currently system_supports_sve() currently implies has_vhe(), - * so the check is redundant. However, has_vhe() can be determined - * statically and helps the compiler remove dead code. - */ - if (has_vhe() && system_supports_sve()) { + if (system_supports_sve()) { sve_guest = vcpu_has_sve(vcpu); sve_host = vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE; - vhe = true; } else { sve_guest = false; sve_host = false; - vhe = has_vhe(); } esr_ec = kvm_vcpu_trap_get_class(vcpu); @@ -229,53 +243,38 @@ static inline bool __hyp_handle_fpsimd(struct kvm_vcpu *vcpu) return false; /* Don't handle SVE traps for non-SVE vcpus here: */ - if (!sve_guest) - if (esr_ec != ESR_ELx_EC_FP_ASIMD) - return false; + if (!sve_guest && esr_ec != ESR_ELx_EC_FP_ASIMD) + return false; /* Valid trap. Switch the context: */ - - if (vhe) { - u64 reg = read_sysreg(cpacr_el1) | CPACR_EL1_FPEN; - + if (has_vhe()) { + reg = CPACR_EL1_FPEN; if (sve_guest) reg |= CPACR_EL1_ZEN; - write_sysreg(reg, cpacr_el1); + sysreg_clear_set(cpacr_el1, 0, reg); } else { - write_sysreg(read_sysreg(cptr_el2) & ~(u64)CPTR_EL2_TFP, - cptr_el2); - } + reg = CPTR_EL2_TFP; + if (sve_guest) + reg |= CPTR_EL2_TZ; + sysreg_clear_set(cptr_el2, reg, 0); + } isb(); if (vcpu->arch.flags & KVM_ARM64_FP_HOST) { - /* - * In the SVE case, VHE is assumed: it is enforced by - * Kconfig and kvm_arch_init(). - */ - if (sve_host) { - struct thread_struct *thread = container_of( - vcpu->arch.host_fpsimd_state, - struct thread_struct, uw.fpsimd_state); - - sve_save_state(sve_pffr(thread), - &vcpu->arch.host_fpsimd_state->fpsr); - } else { + if (sve_host) + __hyp_sve_save_host(vcpu); + else __fpsimd_save_state(vcpu->arch.host_fpsimd_state); - } vcpu->arch.flags &= ~KVM_ARM64_FP_HOST; } - if (sve_guest) { - sve_load_state(vcpu_sve_pffr(vcpu), - &vcpu->arch.ctxt.fp_regs.fpsr, - sve_vq_from_vl(vcpu->arch.sve_max_vl) - 1); - write_sysreg_s(__vcpu_sys_reg(vcpu, ZCR_EL1), SYS_ZCR_EL12); - } else { + if (sve_guest) + __hyp_sve_restore_guest(vcpu); + else __fpsimd_restore_state(&vcpu->arch.ctxt.fp_regs); - } /* Skip restoring fpexc32 for AArch64 guests */ if (!(read_sysreg(hcr_el2) & HCR_RW)) diff --git a/arch/arm64/kvm/hyp/include/nvhe/early_alloc.h b/arch/arm64/kvm/hyp/include/nvhe/early_alloc.h new file mode 100644 index 000000000000..dc61aaa56f31 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/early_alloc.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __KVM_HYP_EARLY_ALLOC_H +#define __KVM_HYP_EARLY_ALLOC_H + +#include + +void hyp_early_alloc_init(void *virt, unsigned long size); +unsigned long hyp_early_alloc_nr_used_pages(void); +void *hyp_early_alloc_page(void *arg); +void *hyp_early_alloc_contig(unsigned int nr_pages); + +extern struct kvm_pgtable_mm_ops hyp_early_alloc_mm_ops; + +#endif /* __KVM_HYP_EARLY_ALLOC_H */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/gfp.h b/arch/arm64/kvm/hyp/include/nvhe/gfp.h new file mode 100644 index 000000000000..55b3f0ce5bc8 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/gfp.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __KVM_HYP_GFP_H +#define __KVM_HYP_GFP_H + +#include + +#include +#include + +#define HYP_NO_ORDER UINT_MAX + +struct hyp_pool { + /* + * Spinlock protecting concurrent changes to the memory pool as well as + * the struct hyp_page of the pool's pages until we have a proper atomic + * API at EL2. + */ + hyp_spinlock_t lock; + struct list_head free_area[MAX_ORDER]; + phys_addr_t range_start; + phys_addr_t range_end; + unsigned int max_order; +}; + +static inline void hyp_page_ref_inc(struct hyp_page *p) +{ + struct hyp_pool *pool = hyp_page_to_pool(p); + + hyp_spin_lock(&pool->lock); + p->refcount++; + hyp_spin_unlock(&pool->lock); +} + +static inline int hyp_page_ref_dec_and_test(struct hyp_page *p) +{ + struct hyp_pool *pool = hyp_page_to_pool(p); + int ret; + + hyp_spin_lock(&pool->lock); + p->refcount--; + ret = (p->refcount == 0); + hyp_spin_unlock(&pool->lock); + + return ret; +} + +static inline void hyp_set_page_refcounted(struct hyp_page *p) +{ + struct hyp_pool *pool = hyp_page_to_pool(p); + + hyp_spin_lock(&pool->lock); + if (p->refcount) { + hyp_spin_unlock(&pool->lock); + hyp_panic(); + } + p->refcount = 1; + hyp_spin_unlock(&pool->lock); +} + +/* Allocation */ +void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order); +void hyp_get_page(void *addr); +void hyp_put_page(void *addr); + +/* Used pages cannot be freed */ +int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, + unsigned int reserved_pages); +#endif /* __KVM_HYP_GFP_H */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h new file mode 100644 index 000000000000..42d81ec739fa --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Google LLC + * Author: Quentin Perret + */ + +#ifndef __KVM_NVHE_MEM_PROTECT__ +#define __KVM_NVHE_MEM_PROTECT__ +#include +#include +#include +#include +#include + +struct host_kvm { + struct kvm_arch arch; + struct kvm_pgtable pgt; + struct kvm_pgtable_mm_ops mm_ops; + hyp_spinlock_t lock; +}; +extern struct host_kvm host_kvm; + +int __pkvm_prot_finalize(void); +int __pkvm_mark_hyp(phys_addr_t start, phys_addr_t end); + +int kvm_host_prepare_stage2(void *mem_pgt_pool, void *dev_pgt_pool); +void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt); + +static __always_inline void __load_host_stage2(void) +{ + if (static_branch_likely(&kvm_protected_mode_initialized)) + __load_stage2(&host_kvm.arch.mmu, host_kvm.arch.vtcr); + else + write_sysreg(0, vttbr_el2); +} +#endif /* __KVM_NVHE_MEM_PROTECT__ */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/memory.h b/arch/arm64/kvm/hyp/include/nvhe/memory.h new file mode 100644 index 000000000000..d2fb307c5952 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/memory.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __KVM_HYP_MEMORY_H +#define __KVM_HYP_MEMORY_H + +#include + +#include + +struct hyp_pool; +struct hyp_page { + unsigned int refcount; + unsigned int order; + struct hyp_pool *pool; + struct list_head node; +}; + +extern s64 hyp_physvirt_offset; +extern u64 __hyp_vmemmap; +#define hyp_vmemmap ((struct hyp_page *)__hyp_vmemmap) + +#define __hyp_pa(virt) ((phys_addr_t)(virt) + hyp_physvirt_offset) +#define __hyp_va(phys) ((void *)((phys_addr_t)(phys) - hyp_physvirt_offset)) + +static inline void *hyp_phys_to_virt(phys_addr_t phys) +{ + return __hyp_va(phys); +} + +static inline phys_addr_t hyp_virt_to_phys(void *addr) +{ + return __hyp_pa(addr); +} + +#define hyp_phys_to_pfn(phys) ((phys) >> PAGE_SHIFT) +#define hyp_pfn_to_phys(pfn) ((phys_addr_t)((pfn) << PAGE_SHIFT)) +#define hyp_phys_to_page(phys) (&hyp_vmemmap[hyp_phys_to_pfn(phys)]) +#define hyp_virt_to_page(virt) hyp_phys_to_page(__hyp_pa(virt)) +#define hyp_virt_to_pfn(virt) hyp_phys_to_pfn(__hyp_pa(virt)) + +#define hyp_page_to_pfn(page) ((struct hyp_page *)(page) - hyp_vmemmap) +#define hyp_page_to_phys(page) hyp_pfn_to_phys((hyp_page_to_pfn(page))) +#define hyp_page_to_virt(page) __hyp_va(hyp_page_to_phys(page)) +#define hyp_page_to_pool(page) (((struct hyp_page *)page)->pool) + +static inline int hyp_page_count(void *addr) +{ + struct hyp_page *p = hyp_virt_to_page(addr); + + return p->refcount; +} + +#endif /* __KVM_HYP_MEMORY_H */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h new file mode 100644 index 000000000000..0095f6289742 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __KVM_HYP_MM_H +#define __KVM_HYP_MM_H + +#include +#include +#include +#include + +#include +#include + +#define HYP_MEMBLOCK_REGIONS 128 +extern struct memblock_region kvm_nvhe_sym(hyp_memory)[]; +extern unsigned int kvm_nvhe_sym(hyp_memblock_nr); +extern struct kvm_pgtable pkvm_pgtable; +extern hyp_spinlock_t pkvm_pgd_lock; +extern struct hyp_pool hpool; +extern u64 __io_map_base; + +int hyp_create_idmap(u32 hyp_va_bits); +int hyp_map_vectors(void); +int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back); +int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot); +int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot); +int __pkvm_create_mappings(unsigned long start, unsigned long size, + unsigned long phys, enum kvm_pgtable_prot prot); +unsigned long __pkvm_create_private_mapping(phys_addr_t phys, size_t size, + enum kvm_pgtable_prot prot); + +static inline void hyp_vmemmap_range(phys_addr_t phys, unsigned long size, + unsigned long *start, unsigned long *end) +{ + unsigned long nr_pages = size >> PAGE_SHIFT; + struct hyp_page *p = hyp_phys_to_page(phys); + + *start = (unsigned long)p; + *end = *start + nr_pages * sizeof(struct hyp_page); + *start = ALIGN_DOWN(*start, PAGE_SIZE); + *end = ALIGN(*end, PAGE_SIZE); +} + +static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages) +{ + unsigned long total = 0, i; + + /* Provision the worst case scenario */ + for (i = 0; i < KVM_PGTABLE_MAX_LEVELS; i++) { + nr_pages = DIV_ROUND_UP(nr_pages, PTRS_PER_PTE); + total += nr_pages; + } + + return total; +} + +static inline unsigned long __hyp_pgtable_total_pages(void) +{ + unsigned long res = 0, i; + + /* Cover all of memory with page-granularity */ + for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) { + struct memblock_region *reg = &kvm_nvhe_sym(hyp_memory)[i]; + res += __hyp_pgtable_max_pages(reg->size >> PAGE_SHIFT); + } + + return res; +} + +static inline unsigned long hyp_s1_pgtable_pages(void) +{ + unsigned long res; + + res = __hyp_pgtable_total_pages(); + + /* Allow 1 GiB for private mappings */ + res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT); + + return res; +} + +static inline unsigned long host_s2_mem_pgtable_pages(void) +{ + /* + * Include an extra 16 pages to safely upper-bound the worst case of + * concatenated pgds. + */ + return __hyp_pgtable_total_pages() + 16; +} + +static inline unsigned long host_s2_dev_pgtable_pages(void) +{ + /* Allow 1 GiB for MMIO mappings */ + return __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT); +} + +#endif /* __KVM_HYP_MM_H */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/spinlock.h b/arch/arm64/kvm/hyp/include/nvhe/spinlock.h new file mode 100644 index 000000000000..76b537f8d1c6 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/spinlock.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * A stand-alone ticket spinlock implementation for use by the non-VHE + * KVM hypervisor code running at EL2. + * + * Copyright (C) 2020 Google LLC + * Author: Will Deacon + * + * Heavily based on the implementation removed by c11090474d70 which was: + * Copyright (C) 2012 ARM Ltd. + */ + +#ifndef __ARM64_KVM_NVHE_SPINLOCK_H__ +#define __ARM64_KVM_NVHE_SPINLOCK_H__ + +#include +#include + +typedef union hyp_spinlock { + u32 __val; + struct { +#ifdef __AARCH64EB__ + u16 next, owner; +#else + u16 owner, next; +#endif + }; +} hyp_spinlock_t; + +#define hyp_spin_lock_init(l) \ +do { \ + *(l) = (hyp_spinlock_t){ .__val = 0 }; \ +} while (0) + +static inline void hyp_spin_lock(hyp_spinlock_t *lock) +{ + u32 tmp; + hyp_spinlock_t lockval, newval; + + asm volatile( + /* Atomically increment the next ticket. */ + ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ +" prfm pstl1strm, %3\n" +"1: ldaxr %w0, %3\n" +" add %w1, %w0, #(1 << 16)\n" +" stxr %w2, %w1, %3\n" +" cbnz %w2, 1b\n", + /* LSE atomics */ +" mov %w2, #(1 << 16)\n" +" ldadda %w2, %w0, %3\n" + __nops(3)) + + /* Did we get the lock? */ +" eor %w1, %w0, %w0, ror #16\n" +" cbz %w1, 3f\n" + /* + * No: spin on the owner. Send a local event to avoid missing an + * unlock before the exclusive load. + */ +" sevl\n" +"2: wfe\n" +" ldaxrh %w2, %4\n" +" eor %w1, %w2, %w0, lsr #16\n" +" cbnz %w1, 2b\n" + /* We got the lock. Critical section starts here. */ +"3:" + : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) + : "Q" (lock->owner) + : "memory"); +} + +static inline void hyp_spin_unlock(hyp_spinlock_t *lock) +{ + u64 tmp; + + asm volatile( + ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " ldrh %w1, %0\n" + " add %w1, %w1, #1\n" + " stlrh %w1, %0", + /* LSE atomics */ + " mov %w1, #1\n" + " staddlh %w1, %0\n" + __nops(1)) + : "=Q" (lock->owner), "=&r" (tmp) + : + : "memory"); +} + +#endif /* __ARM64_KVM_NVHE_SPINLOCK_H__ */ diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 5d1264914cbb..a6c007816bd9 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -9,10 +9,15 @@ ccflags-y := -D__KVM_NVHE_HYPERVISOR__ -D__DISABLE_EXPORTS hostprogs := gen-hyprel HOST_EXTRACFLAGS += -I$(objtree)/include +lib-objs := clear_page.o copy_page.o memcpy.o memset.o +lib-objs := $(addprefix ../../../lib/, $(lib-objs)) + obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \ - hyp-main.o hyp-smp.o psci-relay.o + hyp-main.o hyp-smp.o psci-relay.o early_alloc.o stub.o page_alloc.o \ + cache.o setup.o mm.o mem_protect.o obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ - ../fpsimd.o ../hyp-entry.o ../exception.o + ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o +obj-y += $(lib-objs) ## ## Build rules for compiling nVHE hyp code diff --git a/arch/arm64/kvm/hyp/nvhe/cache.S b/arch/arm64/kvm/hyp/nvhe/cache.S new file mode 100644 index 000000000000..36cef6915428 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/cache.S @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Code copied from arch/arm64/mm/cache.S. + */ + +#include +#include +#include + +SYM_FUNC_START_PI(__flush_dcache_area) + dcache_by_line_op civac, sy, x0, x1, x2, x3 + ret +SYM_FUNC_END_PI(__flush_dcache_area) diff --git a/arch/arm64/kvm/hyp/nvhe/early_alloc.c b/arch/arm64/kvm/hyp/nvhe/early_alloc.c new file mode 100644 index 000000000000..1306c430ab87 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/early_alloc.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Google LLC + * Author: Quentin Perret + */ + +#include + +#include +#include + +struct kvm_pgtable_mm_ops hyp_early_alloc_mm_ops; +s64 __ro_after_init hyp_physvirt_offset; + +static unsigned long base; +static unsigned long end; +static unsigned long cur; + +unsigned long hyp_early_alloc_nr_used_pages(void) +{ + return (cur - base) >> PAGE_SHIFT; +} + +void *hyp_early_alloc_contig(unsigned int nr_pages) +{ + unsigned long size = (nr_pages << PAGE_SHIFT); + void *ret = (void *)cur; + + if (!nr_pages) + return NULL; + + if (end - cur < size) + return NULL; + + cur += size; + memset(ret, 0, size); + + return ret; +} + +void *hyp_early_alloc_page(void *arg) +{ + return hyp_early_alloc_contig(1); +} + +void hyp_early_alloc_init(void *virt, unsigned long size) +{ + base = cur = (unsigned long)virt; + end = base + size; + + hyp_early_alloc_mm_ops.zalloc_page = hyp_early_alloc_page; + hyp_early_alloc_mm_ops.phys_to_virt = hyp_phys_to_virt; + hyp_early_alloc_mm_ops.virt_to_phys = hyp_virt_to_phys; +} diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index c631e29fb001..c953fb4b9a13 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -83,11 +83,6 @@ SYM_CODE_END(__kvm_hyp_init) * x0: struct kvm_nvhe_init_params PA */ SYM_CODE_START_LOCAL(___kvm_hyp_init) -alternative_if ARM64_KVM_PROTECTED_MODE - mov_q x1, HCR_HOST_NVHE_PROTECTED_FLAGS - msr hcr_el2, x1 -alternative_else_nop_endif - ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] msr tpidr_el2, x1 @@ -97,6 +92,15 @@ alternative_else_nop_endif ldr x1, [x0, #NVHE_INIT_MAIR_EL2] msr mair_el2, x1 + ldr x1, [x0, #NVHE_INIT_HCR_EL2] + msr hcr_el2, x1 + + ldr x1, [x0, #NVHE_INIT_VTTBR] + msr vttbr_el2, x1 + + ldr x1, [x0, #NVHE_INIT_VTCR] + msr vtcr_el2, x1 + ldr x1, [x0, #NVHE_INIT_PGD_PA] phys_to_ttbr x2, x1 alternative_if ARM64_HAS_CNP @@ -115,15 +119,10 @@ alternative_else_nop_endif /* Invalidate the stale TLBs from Bootloader */ tlbi alle2 + tlbi vmalls12e1 dsb sy - /* - * Preserve all the RES1 bits while setting the default flags, - * as well as the EE bit on BE. Drop the A flag since the compiler - * is allowed to generate unaligned accesses. - */ - mov_q x0, (SCTLR_EL2_RES1 | (SCTLR_ELx_FLAGS & ~SCTLR_ELx_A)) -CPU_BE( orr x0, x0, #SCTLR_ELx_EE) + mov_q x0, INIT_SCTLR_EL2_MMU_ON alternative_if ARM64_HAS_ADDRESS_AUTH mov_q x1, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \ SCTLR_ELx_ENDA | SCTLR_ELx_ENDB) @@ -221,9 +220,7 @@ SYM_CODE_START(__kvm_handle_stub_hvc) mov x0, xzr reset: /* Reset kvm back to the hyp stub. */ - mrs x5, sctlr_el2 - mov_q x6, SCTLR_ELx_FLAGS - bic x5, x5, x6 // Clear SCTL_M and etc + mov_q x5, INIT_SCTLR_EL2_MMU_OFF pre_disable_mmu_workaround msr sctlr_el2, x5 isb @@ -244,4 +241,31 @@ alternative_else_nop_endif SYM_CODE_END(__kvm_handle_stub_hvc) +SYM_FUNC_START(__pkvm_init_switch_pgd) + /* Turn the MMU off */ + pre_disable_mmu_workaround + mrs x2, sctlr_el2 + bic x3, x2, #SCTLR_ELx_M + msr sctlr_el2, x3 + isb + + tlbi alle2 + + /* Install the new pgtables */ + ldr x3, [x0, #NVHE_INIT_PGD_PA] + phys_to_ttbr x4, x3 +alternative_if ARM64_HAS_CNP + orr x4, x4, #TTBR_CNP_BIT +alternative_else_nop_endif + msr ttbr0_el2, x4 + + /* Set the new stack pointer */ + ldr x0, [x0, #NVHE_INIT_STACK_HYP_VA] + mov sp, x0 + + /* And turn the MMU back on! */ + set_sctlr_el2 x2 + ret x1 +SYM_FUNC_END(__pkvm_init_switch_pgd) + .popsection diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 936328207bde..5c29ee9c7437 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -6,12 +6,15 @@ #include +#include #include #include #include #include #include +#include +#include #include DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); @@ -106,6 +109,61 @@ static void handle___vgic_v3_restore_aprs(struct kvm_cpu_context *host_ctxt) __vgic_v3_restore_aprs(kern_hyp_va(cpu_if)); } +static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); + DECLARE_REG(unsigned long, size, host_ctxt, 2); + DECLARE_REG(unsigned long, nr_cpus, host_ctxt, 3); + DECLARE_REG(unsigned long *, per_cpu_base, host_ctxt, 4); + DECLARE_REG(u32, hyp_va_bits, host_ctxt, 5); + + /* + * __pkvm_init() will return only if an error occurred, otherwise it + * will tail-call in __pkvm_init_finalise() which will have to deal + * with the host context directly. + */ + cpu_reg(host_ctxt, 1) = __pkvm_init(phys, size, nr_cpus, per_cpu_base, + hyp_va_bits); +} + +static void handle___pkvm_cpu_set_vector(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(enum arm64_hyp_spectre_vector, slot, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = pkvm_cpu_set_vector(slot); +} + +static void handle___pkvm_create_mappings(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(unsigned long, start, host_ctxt, 1); + DECLARE_REG(unsigned long, size, host_ctxt, 2); + DECLARE_REG(unsigned long, phys, host_ctxt, 3); + DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 4); + + cpu_reg(host_ctxt, 1) = __pkvm_create_mappings(start, size, phys, prot); +} + +static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); + DECLARE_REG(size_t, size, host_ctxt, 2); + DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 3); + + cpu_reg(host_ctxt, 1) = __pkvm_create_private_mapping(phys, size, prot); +} + +static void handle___pkvm_prot_finalize(struct kvm_cpu_context *host_ctxt) +{ + cpu_reg(host_ctxt, 1) = __pkvm_prot_finalize(); +} + +static void handle___pkvm_mark_hyp(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(phys_addr_t, start, host_ctxt, 1); + DECLARE_REG(phys_addr_t, end, host_ctxt, 2); + + cpu_reg(host_ctxt, 1) = __pkvm_mark_hyp(start, end); +} typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x @@ -125,6 +183,12 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__kvm_get_mdcr_el2), HANDLE_FUNC(__vgic_v3_save_aprs), HANDLE_FUNC(__vgic_v3_restore_aprs), + HANDLE_FUNC(__pkvm_init), + HANDLE_FUNC(__pkvm_cpu_set_vector), + HANDLE_FUNC(__pkvm_create_mappings), + HANDLE_FUNC(__pkvm_create_private_mapping), + HANDLE_FUNC(__pkvm_prot_finalize), + HANDLE_FUNC(__pkvm_mark_hyp), }; static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) @@ -177,6 +241,16 @@ void handle_trap(struct kvm_cpu_context *host_ctxt) case ESR_ELx_EC_SMC64: handle_host_smc(host_ctxt); break; + case ESR_ELx_EC_SVE: + sysreg_clear_set(cptr_el2, CPTR_EL2_TZ, 0); + isb(); + sve_cond_update_zcr_vq(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2); + break; + case ESR_ELx_EC_IABT_LOW: + fallthrough; + case ESR_ELx_EC_DABT_LOW: + handle_host_mem_abort(host_ctxt); + break; default: hyp_panic(); } diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c index 879559057dee..cc829b9db0da 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c @@ -38,3 +38,10 @@ unsigned long __hyp_per_cpu_offset(unsigned int cpu) elf_base = (unsigned long)&__per_cpu_start; return this_cpu_base - elf_base; } + +/* + * Define the CPU feature registers variables that will hold the copies of + * the host's sanitized values. + */ +#define KVM_HYP_CPU_FTR_REG(name) struct arm64_ftr_reg name +#include diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S index cd119d82d8e3..f4562f417d3f 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S @@ -25,4 +25,5 @@ SECTIONS { BEGIN_HYP_SECTION(.data..percpu) PERCPU_INPUT(L1_CACHE_BYTES) END_HYP_SECTION + HYP_SECTION(.bss) } diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c new file mode 100644 index 000000000000..c472c3becf40 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Google LLC + * Author: Quentin Perret + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +extern unsigned long hyp_nr_cpus; +struct host_kvm host_kvm; + +struct hyp_pool host_s2_mem; +struct hyp_pool host_s2_dev; + +static const u8 pkvm_hyp_id = 1; + +static void *host_s2_zalloc_pages_exact(size_t size) +{ + return hyp_alloc_pages(&host_s2_mem, get_order(size)); +} + +static void *host_s2_zalloc_page(void *pool) +{ + return hyp_alloc_pages(pool, 0); +} + +static int prepare_s2_pools(void *mem_pgt_pool, void *dev_pgt_pool) +{ + unsigned long nr_pages, pfn; + int ret; + + pfn = hyp_virt_to_pfn(mem_pgt_pool); + nr_pages = host_s2_mem_pgtable_pages(); + ret = hyp_pool_init(&host_s2_mem, pfn, nr_pages, 0); + if (ret) + return ret; + + pfn = hyp_virt_to_pfn(dev_pgt_pool); + nr_pages = host_s2_dev_pgtable_pages(); + ret = hyp_pool_init(&host_s2_dev, pfn, nr_pages, 0); + if (ret) + return ret; + + host_kvm.mm_ops = (struct kvm_pgtable_mm_ops) { + .zalloc_pages_exact = host_s2_zalloc_pages_exact, + .zalloc_page = host_s2_zalloc_page, + .phys_to_virt = hyp_phys_to_virt, + .virt_to_phys = hyp_virt_to_phys, + .page_count = hyp_page_count, + .get_page = hyp_get_page, + .put_page = hyp_put_page, + }; + + return 0; +} + +static void prepare_host_vtcr(void) +{ + u32 parange, phys_shift; + u64 mmfr0, mmfr1; + + mmfr0 = arm64_ftr_reg_id_aa64mmfr0_el1.sys_val; + mmfr1 = arm64_ftr_reg_id_aa64mmfr1_el1.sys_val; + + /* The host stage 2 is id-mapped, so use parange for T0SZ */ + parange = kvm_get_parange(mmfr0); + phys_shift = id_aa64mmfr0_parange_to_phys_shift(parange); + + host_kvm.arch.vtcr = kvm_get_vtcr(mmfr0, mmfr1, phys_shift); +} + +int kvm_host_prepare_stage2(void *mem_pgt_pool, void *dev_pgt_pool) +{ + struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu; + int ret; + + prepare_host_vtcr(); + hyp_spin_lock_init(&host_kvm.lock); + + ret = prepare_s2_pools(mem_pgt_pool, dev_pgt_pool); + if (ret) + return ret; + + ret = kvm_pgtable_stage2_init_flags(&host_kvm.pgt, &host_kvm.arch, + &host_kvm.mm_ops, KVM_PGTABLE_S2_NOFWB); + if (ret) + return ret; + + mmu->pgd_phys = __hyp_pa(host_kvm.pgt.pgd); + mmu->arch = &host_kvm.arch; + mmu->pgt = &host_kvm.pgt; + mmu->vmid.vmid_gen = 0; + mmu->vmid.vmid = 0; + + return 0; +} + +int __pkvm_prot_finalize(void) +{ + struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu; + struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params); + + params->vttbr = kvm_get_vttbr(mmu); + params->vtcr = host_kvm.arch.vtcr; + params->hcr_el2 |= HCR_VM; + kvm_flush_dcache_to_poc(params, sizeof(*params)); + + write_sysreg(params->hcr_el2, hcr_el2); + __load_stage2(&host_kvm.arch.mmu, host_kvm.arch.vtcr); + + /* + * Make sure to have an ISB before the TLB maintenance below but only + * when __load_stage2() doesn't include one already. + */ + asm(ALTERNATIVE("isb", "nop", ARM64_WORKAROUND_SPECULATIVE_AT)); + + /* Invalidate stale HCR bits that may be cached in TLBs */ + __tlbi(vmalls12e1); + dsb(nsh); + isb(); + + return 0; +} + +static int host_stage2_unmap_dev_all(void) +{ + struct kvm_pgtable *pgt = &host_kvm.pgt; + struct memblock_region *reg; + u64 addr = 0; + int i, ret; + + /* Unmap all non-memory regions to recycle the pages */ + for (i = 0; i < hyp_memblock_nr; i++, addr = reg->base + reg->size) { + reg = &hyp_memory[i]; + ret = kvm_pgtable_stage2_unmap(pgt, addr, reg->base - addr); + if (ret) + return ret; + } + return kvm_pgtable_stage2_unmap(pgt, addr, BIT(pgt->ia_bits) - addr); +} + +static bool find_mem_range(phys_addr_t addr, struct kvm_mem_range *range) +{ + int cur, left = 0, right = hyp_memblock_nr; + struct memblock_region *reg; + phys_addr_t end; + + range->start = 0; + range->end = ULONG_MAX; + + /* The list of memblock regions is sorted, binary search it */ + while (left < right) { + cur = (left + right) >> 1; + reg = &hyp_memory[cur]; + end = reg->base + reg->size; + if (addr < reg->base) { + right = cur; + range->end = reg->base; + } else if (addr >= end) { + left = cur + 1; + range->start = end; + } else { + range->start = reg->base; + range->end = end; + return true; + } + } + + return false; +} + +static bool range_is_memory(u64 start, u64 end) +{ + struct kvm_mem_range r1, r2; + + if (!find_mem_range(start, &r1) || !find_mem_range(end, &r2)) + return false; + if (r1.start != r2.start) + return false; + + return true; +} + +static inline int __host_stage2_idmap(u64 start, u64 end, + enum kvm_pgtable_prot prot, + struct hyp_pool *pool) +{ + return kvm_pgtable_stage2_map(&host_kvm.pgt, start, end - start, start, + prot, pool); +} + +static int host_stage2_idmap(u64 addr) +{ + enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W; + struct kvm_mem_range range; + bool is_memory = find_mem_range(addr, &range); + struct hyp_pool *pool = is_memory ? &host_s2_mem : &host_s2_dev; + int ret; + + if (is_memory) + prot |= KVM_PGTABLE_PROT_X; + + hyp_spin_lock(&host_kvm.lock); + ret = kvm_pgtable_stage2_find_range(&host_kvm.pgt, addr, prot, &range); + if (ret) + goto unlock; + + ret = __host_stage2_idmap(range.start, range.end, prot, pool); + if (is_memory || ret != -ENOMEM) + goto unlock; + + /* + * host_s2_mem has been provided with enough pages to cover all of + * memory with page granularity, so we should never hit the ENOMEM case. + * However, it is difficult to know how much of the MMIO range we will + * need to cover upfront, so we may need to 'recycle' the pages if we + * run out. + */ + ret = host_stage2_unmap_dev_all(); + if (ret) + goto unlock; + + ret = __host_stage2_idmap(range.start, range.end, prot, pool); + +unlock: + hyp_spin_unlock(&host_kvm.lock); + + return ret; +} + +int __pkvm_mark_hyp(phys_addr_t start, phys_addr_t end) +{ + int ret; + + /* + * host_stage2_unmap_dev_all() currently relies on MMIO mappings being + * non-persistent, so don't allow changing page ownership in MMIO range. + */ + if (!range_is_memory(start, end)) + return -EINVAL; + + hyp_spin_lock(&host_kvm.lock); + ret = kvm_pgtable_stage2_set_owner(&host_kvm.pgt, start, end - start, + &host_s2_mem, pkvm_hyp_id); + hyp_spin_unlock(&host_kvm.lock); + + return ret != -EAGAIN ? ret : 0; +} + +void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) +{ + struct kvm_vcpu_fault_info fault; + u64 esr, addr; + int ret = 0; + + esr = read_sysreg_el2(SYS_ESR); + if (!__get_fault_info(esr, &fault)) + hyp_panic(); + + addr = (fault.hpfar_el2 & HPFAR_MASK) << 8; + ret = host_stage2_idmap(addr); + if (ret && ret != -EAGAIN) + hyp_panic(); +} diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c new file mode 100644 index 000000000000..a8efdf0f9003 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Google LLC + * Author: Quentin Perret + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +struct kvm_pgtable pkvm_pgtable; +hyp_spinlock_t pkvm_pgd_lock; +u64 __io_map_base; + +struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS]; +unsigned int hyp_memblock_nr; + +int __pkvm_create_mappings(unsigned long start, unsigned long size, + unsigned long phys, enum kvm_pgtable_prot prot) +{ + int err; + + hyp_spin_lock(&pkvm_pgd_lock); + err = kvm_pgtable_hyp_map(&pkvm_pgtable, start, size, phys, prot); + hyp_spin_unlock(&pkvm_pgd_lock); + + return err; +} + +unsigned long __pkvm_create_private_mapping(phys_addr_t phys, size_t size, + enum kvm_pgtable_prot prot) +{ + unsigned long addr; + int err; + + hyp_spin_lock(&pkvm_pgd_lock); + + size = PAGE_ALIGN(size + offset_in_page(phys)); + addr = __io_map_base; + __io_map_base += size; + + /* Are we overflowing on the vmemmap ? */ + if (__io_map_base > __hyp_vmemmap) { + __io_map_base -= size; + addr = (unsigned long)ERR_PTR(-ENOMEM); + goto out; + } + + err = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, size, phys, prot); + if (err) { + addr = (unsigned long)ERR_PTR(err); + goto out; + } + + addr = addr + offset_in_page(phys); +out: + hyp_spin_unlock(&pkvm_pgd_lock); + + return addr; +} + +int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot) +{ + unsigned long start = (unsigned long)from; + unsigned long end = (unsigned long)to; + unsigned long virt_addr; + phys_addr_t phys; + + start = start & PAGE_MASK; + end = PAGE_ALIGN(end); + + for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) { + int err; + + phys = hyp_virt_to_phys((void *)virt_addr); + err = __pkvm_create_mappings(virt_addr, PAGE_SIZE, phys, prot); + if (err) + return err; + } + + return 0; +} + +int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back) +{ + unsigned long start, end; + + hyp_vmemmap_range(phys, size, &start, &end); + + return __pkvm_create_mappings(start, end - start, back, PAGE_HYP); +} + +static void *__hyp_bp_vect_base; +int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot) +{ + void *vector; + + switch (slot) { + case HYP_VECTOR_DIRECT: { + vector = __kvm_hyp_vector; + break; + } + case HYP_VECTOR_SPECTRE_DIRECT: { + vector = __bp_harden_hyp_vecs; + break; + } + case HYP_VECTOR_INDIRECT: + case HYP_VECTOR_SPECTRE_INDIRECT: { + vector = (void *)__hyp_bp_vect_base; + break; + } + default: + return -EINVAL; + } + + vector = __kvm_vector_slot2addr(vector, slot); + *this_cpu_ptr(&kvm_hyp_vector) = (unsigned long)vector; + + return 0; +} + +int hyp_map_vectors(void) +{ + phys_addr_t phys; + void *bp_base; + + if (!cpus_have_const_cap(ARM64_SPECTRE_V3A)) + return 0; + + phys = __hyp_pa(__bp_harden_hyp_vecs); + bp_base = (void *)__pkvm_create_private_mapping(phys, + __BP_HARDEN_HYP_VECS_SZ, + PAGE_HYP_EXEC); + if (IS_ERR_OR_NULL(bp_base)) + return PTR_ERR(bp_base); + + __hyp_bp_vect_base = bp_base; + + return 0; +} + +int hyp_create_idmap(u32 hyp_va_bits) +{ + unsigned long start, end; + + start = hyp_virt_to_phys((void *)__hyp_idmap_text_start); + start = ALIGN_DOWN(start, PAGE_SIZE); + + end = hyp_virt_to_phys((void *)__hyp_idmap_text_end); + end = ALIGN(end, PAGE_SIZE); + + /* + * One half of the VA space is reserved to linearly map portions of + * memory -- see va_layout.c for more details. The other half of the VA + * space contains the trampoline page, and needs some care. Split that + * second half in two and find the quarter of VA space not conflicting + * with the idmap to place the IOs and the vmemmap. IOs use the lower + * half of the quarter and the vmemmap the upper half. + */ + __io_map_base = start & BIT(hyp_va_bits - 2); + __io_map_base ^= BIT(hyp_va_bits - 2); + __hyp_vmemmap = __io_map_base | BIT(hyp_va_bits - 3); + + return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC); +} diff --git a/arch/arm64/kvm/hyp/nvhe/page_alloc.c b/arch/arm64/kvm/hyp/nvhe/page_alloc.c new file mode 100644 index 000000000000..237e03bf0cb1 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/page_alloc.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Google LLC + * Author: Quentin Perret + */ + +#include +#include + +u64 __hyp_vmemmap; + +/* + * Index the hyp_vmemmap to find a potential buddy page, but make no assumption + * about its current state. + * + * Example buddy-tree for a 4-pages physically contiguous pool: + * + * o : Page 3 + * / + * o-o : Page 2 + * / + * / o : Page 1 + * / / + * o---o-o : Page 0 + * Order 2 1 0 + * + * Example of requests on this pool: + * __find_buddy_nocheck(pool, page 0, order 0) => page 1 + * __find_buddy_nocheck(pool, page 0, order 1) => page 2 + * __find_buddy_nocheck(pool, page 1, order 0) => page 0 + * __find_buddy_nocheck(pool, page 2, order 0) => page 3 + */ +static struct hyp_page *__find_buddy_nocheck(struct hyp_pool *pool, + struct hyp_page *p, + unsigned int order) +{ + phys_addr_t addr = hyp_page_to_phys(p); + + addr ^= (PAGE_SIZE << order); + + /* + * Don't return a page outside the pool range -- it belongs to + * something else and may not be mapped in hyp_vmemmap. + */ + if (addr < pool->range_start || addr >= pool->range_end) + return NULL; + + return hyp_phys_to_page(addr); +} + +/* Find a buddy page currently available for allocation */ +static struct hyp_page *__find_buddy_avail(struct hyp_pool *pool, + struct hyp_page *p, + unsigned int order) +{ + struct hyp_page *buddy = __find_buddy_nocheck(pool, p, order); + + if (!buddy || buddy->order != order || list_empty(&buddy->node)) + return NULL; + + return buddy; + +} + +static void __hyp_attach_page(struct hyp_pool *pool, + struct hyp_page *p) +{ + unsigned int order = p->order; + struct hyp_page *buddy; + + memset(hyp_page_to_virt(p), 0, PAGE_SIZE << p->order); + + /* + * Only the first struct hyp_page of a high-order page (otherwise known + * as the 'head') should have p->order set. The non-head pages should + * have p->order = HYP_NO_ORDER. Here @p may no longer be the head + * after coallescing, so make sure to mark it HYP_NO_ORDER proactively. + */ + p->order = HYP_NO_ORDER; + for (; (order + 1) < pool->max_order; order++) { + buddy = __find_buddy_avail(pool, p, order); + if (!buddy) + break; + + /* Take the buddy out of its list, and coallesce with @p */ + list_del_init(&buddy->node); + buddy->order = HYP_NO_ORDER; + p = min(p, buddy); + } + + /* Mark the new head, and insert it */ + p->order = order; + list_add_tail(&p->node, &pool->free_area[order]); +} + +static void hyp_attach_page(struct hyp_page *p) +{ + struct hyp_pool *pool = hyp_page_to_pool(p); + + hyp_spin_lock(&pool->lock); + __hyp_attach_page(pool, p); + hyp_spin_unlock(&pool->lock); +} + +static struct hyp_page *__hyp_extract_page(struct hyp_pool *pool, + struct hyp_page *p, + unsigned int order) +{ + struct hyp_page *buddy; + + list_del_init(&p->node); + while (p->order > order) { + /* + * The buddy of order n - 1 currently has HYP_NO_ORDER as it + * is covered by a higher-level page (whose head is @p). Use + * __find_buddy_nocheck() to find it and inject it in the + * free_list[n - 1], effectively splitting @p in half. + */ + p->order--; + buddy = __find_buddy_nocheck(pool, p, p->order); + buddy->order = p->order; + list_add_tail(&buddy->node, &pool->free_area[buddy->order]); + } + + return p; +} + +void hyp_put_page(void *addr) +{ + struct hyp_page *p = hyp_virt_to_page(addr); + + if (hyp_page_ref_dec_and_test(p)) + hyp_attach_page(p); +} + +void hyp_get_page(void *addr) +{ + struct hyp_page *p = hyp_virt_to_page(addr); + + hyp_page_ref_inc(p); +} + +void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order) +{ + unsigned int i = order; + struct hyp_page *p; + + hyp_spin_lock(&pool->lock); + + /* Look for a high-enough-order page */ + while (i < pool->max_order && list_empty(&pool->free_area[i])) + i++; + if (i >= pool->max_order) { + hyp_spin_unlock(&pool->lock); + return NULL; + } + + /* Extract it from the tree at the right order */ + p = list_first_entry(&pool->free_area[i], struct hyp_page, node); + p = __hyp_extract_page(pool, p, order); + + hyp_spin_unlock(&pool->lock); + hyp_set_page_refcounted(p); + + return hyp_page_to_virt(p); +} + +int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, + unsigned int reserved_pages) +{ + phys_addr_t phys = hyp_pfn_to_phys(pfn); + struct hyp_page *p; + int i; + + hyp_spin_lock_init(&pool->lock); + pool->max_order = min(MAX_ORDER, get_order(nr_pages << PAGE_SHIFT)); + for (i = 0; i < pool->max_order; i++) + INIT_LIST_HEAD(&pool->free_area[i]); + pool->range_start = phys; + pool->range_end = phys + (nr_pages << PAGE_SHIFT); + + /* Init the vmemmap portion */ + p = hyp_phys_to_page(phys); + memset(p, 0, sizeof(*p) * nr_pages); + for (i = 0; i < nr_pages; i++) { + p[i].pool = pool; + INIT_LIST_HEAD(&p[i].node); + } + + /* Attach the unused pages to the buddy tree */ + for (i = reserved_pages; i < nr_pages; i++) + __hyp_attach_page(pool, &p[i]); + + return 0; +} diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c b/arch/arm64/kvm/hyp/nvhe/psci-relay.c index 63de71c0481e..08508783ec3d 100644 --- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c +++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c @@ -11,6 +11,7 @@ #include #include +#include #include void kvm_hyp_cpu_entry(unsigned long r0); @@ -20,9 +21,6 @@ void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); /* Config options set by the host. */ struct kvm_host_psci_config __ro_after_init kvm_host_psci_config; -s64 __ro_after_init hyp_physvirt_offset; - -#define __hyp_pa(x) ((phys_addr_t)((x)) + hyp_physvirt_offset) #define INVALID_CPU_ID UINT_MAX diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c new file mode 100644 index 000000000000..7488f53b0aa2 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020 Google LLC + * Author: Quentin Perret + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +struct hyp_pool hpool; +struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops; +unsigned long hyp_nr_cpus; + +#define hyp_percpu_size ((unsigned long)__per_cpu_end - \ + (unsigned long)__per_cpu_start) + +static void *vmemmap_base; +static void *hyp_pgt_base; +static void *host_s2_mem_pgt_base; +static void *host_s2_dev_pgt_base; + +static int divide_memory_pool(void *virt, unsigned long size) +{ + unsigned long vstart, vend, nr_pages; + + hyp_early_alloc_init(virt, size); + + hyp_vmemmap_range(__hyp_pa(virt), size, &vstart, &vend); + nr_pages = (vend - vstart) >> PAGE_SHIFT; + vmemmap_base = hyp_early_alloc_contig(nr_pages); + if (!vmemmap_base) + return -ENOMEM; + + nr_pages = hyp_s1_pgtable_pages(); + hyp_pgt_base = hyp_early_alloc_contig(nr_pages); + if (!hyp_pgt_base) + return -ENOMEM; + + nr_pages = host_s2_mem_pgtable_pages(); + host_s2_mem_pgt_base = hyp_early_alloc_contig(nr_pages); + if (!host_s2_mem_pgt_base) + return -ENOMEM; + + nr_pages = host_s2_dev_pgtable_pages(); + host_s2_dev_pgt_base = hyp_early_alloc_contig(nr_pages); + if (!host_s2_dev_pgt_base) + return -ENOMEM; + + return 0; +} + +static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size, + unsigned long *per_cpu_base, + u32 hyp_va_bits) +{ + void *start, *end, *virt = hyp_phys_to_virt(phys); + unsigned long pgt_size = hyp_s1_pgtable_pages() << PAGE_SHIFT; + int ret, i; + + /* Recreate the hyp page-table using the early page allocator */ + hyp_early_alloc_init(hyp_pgt_base, pgt_size); + ret = kvm_pgtable_hyp_init(&pkvm_pgtable, hyp_va_bits, + &hyp_early_alloc_mm_ops); + if (ret) + return ret; + + ret = hyp_create_idmap(hyp_va_bits); + if (ret) + return ret; + + ret = hyp_map_vectors(); + if (ret) + return ret; + + ret = hyp_back_vmemmap(phys, size, hyp_virt_to_phys(vmemmap_base)); + if (ret) + return ret; + + ret = pkvm_create_mappings(__hyp_text_start, __hyp_text_end, PAGE_HYP_EXEC); + if (ret) + return ret; + + ret = pkvm_create_mappings(__start_rodata, __end_rodata, PAGE_HYP_RO); + if (ret) + return ret; + + ret = pkvm_create_mappings(__hyp_rodata_start, __hyp_rodata_end, PAGE_HYP_RO); + if (ret) + return ret; + + ret = pkvm_create_mappings(__hyp_bss_start, __hyp_bss_end, PAGE_HYP); + if (ret) + return ret; + + ret = pkvm_create_mappings(__hyp_bss_end, __bss_stop, PAGE_HYP_RO); + if (ret) + return ret; + + ret = pkvm_create_mappings(virt, virt + size, PAGE_HYP); + if (ret) + return ret; + + for (i = 0; i < hyp_nr_cpus; i++) { + start = (void *)kern_hyp_va(per_cpu_base[i]); + end = start + PAGE_ALIGN(hyp_percpu_size); + ret = pkvm_create_mappings(start, end, PAGE_HYP); + if (ret) + return ret; + + end = (void *)per_cpu_ptr(&kvm_init_params, i)->stack_hyp_va; + start = end - PAGE_SIZE; + ret = pkvm_create_mappings(start, end, PAGE_HYP); + if (ret) + return ret; + } + + return 0; +} + +static void update_nvhe_init_params(void) +{ + struct kvm_nvhe_init_params *params; + unsigned long i; + + for (i = 0; i < hyp_nr_cpus; i++) { + params = per_cpu_ptr(&kvm_init_params, i); + params->pgd_pa = __hyp_pa(pkvm_pgtable.pgd); + __flush_dcache_area(params, sizeof(*params)); + } +} + +static void *hyp_zalloc_hyp_page(void *arg) +{ + return hyp_alloc_pages(&hpool, 0); +} + +void __noreturn __pkvm_init_finalise(void) +{ + struct kvm_host_data *host_data = this_cpu_ptr(&kvm_host_data); + struct kvm_cpu_context *host_ctxt = &host_data->host_ctxt; + unsigned long nr_pages, reserved_pages, pfn; + int ret; + + /* Now that the vmemmap is backed, install the full-fledged allocator */ + pfn = hyp_virt_to_pfn(hyp_pgt_base); + nr_pages = hyp_s1_pgtable_pages(); + reserved_pages = hyp_early_alloc_nr_used_pages(); + ret = hyp_pool_init(&hpool, pfn, nr_pages, reserved_pages); + if (ret) + goto out; + + ret = kvm_host_prepare_stage2(host_s2_mem_pgt_base, host_s2_dev_pgt_base); + if (ret) + goto out; + + pkvm_pgtable_mm_ops = (struct kvm_pgtable_mm_ops) { + .zalloc_page = hyp_zalloc_hyp_page, + .phys_to_virt = hyp_phys_to_virt, + .virt_to_phys = hyp_virt_to_phys, + .get_page = hyp_get_page, + .put_page = hyp_put_page, + }; + pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops; + +out: + /* + * We tail-called to here from handle___pkvm_init() and will not return, + * so make sure to propagate the return value to the host. + */ + cpu_reg(host_ctxt, 1) = ret; + + __host_enter(host_ctxt); +} + +int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus, + unsigned long *per_cpu_base, u32 hyp_va_bits) +{ + struct kvm_nvhe_init_params *params; + void *virt = hyp_phys_to_virt(phys); + void (*fn)(phys_addr_t params_pa, void *finalize_fn_va); + int ret; + + if (!PAGE_ALIGNED(phys) || !PAGE_ALIGNED(size)) + return -EINVAL; + + hyp_spin_lock_init(&pkvm_pgd_lock); + hyp_nr_cpus = nr_cpus; + + ret = divide_memory_pool(virt, size); + if (ret) + return ret; + + ret = recreate_hyp_mappings(phys, size, per_cpu_base, hyp_va_bits); + if (ret) + return ret; + + update_nvhe_init_params(); + + /* Jump in the idmap page to switch to the new page-tables */ + params = this_cpu_ptr(&kvm_init_params); + fn = (typeof(fn))__hyp_pa(__pkvm_init_switch_pgd); + fn(__hyp_pa(params), __pkvm_init_finalise); + + unreachable(); +} diff --git a/arch/arm64/kvm/hyp/nvhe/stub.c b/arch/arm64/kvm/hyp/nvhe/stub.c new file mode 100644 index 000000000000..c0aa6bbfd79d --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/stub.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Stubs for out-of-line function calls caused by re-using kernel + * infrastructure at EL2. + * + * Copyright (C) 2020 - Google LLC + */ + +#include + +#ifdef CONFIG_DEBUG_LIST +bool __list_add_valid(struct list_head *new, struct list_head *prev, + struct list_head *next) +{ + return true; +} + +bool __list_del_entry_valid(struct list_head *entry) +{ + return true; +} +#endif diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 84d2a7012f74..9cc4c9ebfd8f 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -28,6 +28,8 @@ #include #include +#include + /* Non-VHE specific context */ DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data); DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); @@ -41,9 +43,9 @@ static void __activate_traps(struct kvm_vcpu *vcpu) __activate_traps_common(vcpu); val = CPTR_EL2_DEFAULT; - val |= CPTR_EL2_TTA | CPTR_EL2_TZ | CPTR_EL2_TAM; + val |= CPTR_EL2_TTA | CPTR_EL2_TAM; if (!update_fp_enabled(vcpu)) { - val |= CPTR_EL2_TFP; + val |= CPTR_EL2_TFP | CPTR_EL2_TZ; __activate_traps_fpsimd32(vcpu); } @@ -68,7 +70,7 @@ static void __activate_traps(struct kvm_vcpu *vcpu) static void __deactivate_traps(struct kvm_vcpu *vcpu) { extern char __kvm_hyp_host_vector[]; - u64 mdcr_el2; + u64 mdcr_el2, cptr; ___deactivate_traps(vcpu); @@ -98,17 +100,14 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) mdcr_el2 |= MDCR_EL2_E2TB_MASK << MDCR_EL2_E2TB_SHIFT; write_sysreg(mdcr_el2, mdcr_el2); - if (is_protected_kvm_enabled()) - write_sysreg(HCR_HOST_NVHE_PROTECTED_FLAGS, hcr_el2); - else - write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2); - write_sysreg(CPTR_EL2_DEFAULT, cptr_el2); - write_sysreg(__kvm_hyp_host_vector, vbar_el2); -} + write_sysreg(this_cpu_ptr(&kvm_init_params)->hcr_el2, hcr_el2); -static void __load_host_stage2(void) -{ - write_sysreg(0, vttbr_el2); + cptr = CPTR_EL2_DEFAULT; + if (vcpu_has_sve(vcpu) && (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)) + cptr |= CPTR_EL2_TZ; + + write_sysreg(cptr, cptr_el2); + write_sysreg(__kvm_hyp_host_vector, vbar_el2); } /* Save VGICv3 state on non-VHE systems */ diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c index 229b06748c20..83dc3b271bc5 100644 --- a/arch/arm64/kvm/hyp/nvhe/tlb.c +++ b/arch/arm64/kvm/hyp/nvhe/tlb.c @@ -8,6 +8,8 @@ #include #include +#include + struct tlb_inv_context { u64 tcr; }; @@ -43,7 +45,7 @@ static void __tlb_switch_to_guest(struct kvm_s2_mmu *mmu, static void __tlb_switch_to_host(struct tlb_inv_context *cxt) { - write_sysreg(0, vttbr_el2); + __load_host_stage2(); if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { /* Ensure write of the host VMID */ diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 926fc07074f5..06cff4710504 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -9,8 +9,7 @@ #include #include - -#define KVM_PGTABLE_MAX_LEVELS 4U +#include #define KVM_PTE_VALID BIT(0) @@ -49,6 +48,11 @@ KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W | \ KVM_PTE_LEAF_ATTR_HI_S2_XN) +#define KVM_PTE_LEAF_ATTR_S2_IGNORED GENMASK(58, 55) + +#define KVM_INVALID_PTE_OWNER_MASK GENMASK(63, 56) +#define KVM_MAX_OWNER_ID 1 + struct kvm_pgtable_walk_data { struct kvm_pgtable *pgt; struct kvm_pgtable_walker *walker; @@ -68,21 +72,36 @@ static u64 kvm_granule_size(u32 level) return BIT(kvm_granule_shift(level)); } -static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level) -{ - u64 granule = kvm_granule_size(level); +#define KVM_PHYS_INVALID (-1ULL) +static bool kvm_phys_is_valid(u64 phys) +{ + return phys < BIT(id_aa64mmfr0_parange_to_phys_shift(ID_AA64MMFR0_PARANGE_MAX)); +} + +static bool kvm_level_supports_block_mapping(u32 level) +{ /* * Reject invalid block mappings and don't bother with 4TB mappings for * 52-bit PAs. */ - if (level == 0 || (PAGE_SIZE != SZ_4K && level == 1)) + return !(level == 0 || (PAGE_SIZE != SZ_4K && level == 1)); +} + +static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level) +{ + u64 granule = kvm_granule_size(level); + + if (!kvm_level_supports_block_mapping(level)) return false; if (granule > (end - addr)) return false; - return IS_ALIGNED(addr, granule) && IS_ALIGNED(phys, granule); + if (kvm_phys_is_valid(phys) && !IS_ALIGNED(phys, granule)) + return false; + + return IS_ALIGNED(addr, granule); } static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level) @@ -152,20 +171,20 @@ static kvm_pte_t kvm_phys_to_pte(u64 pa) return pte; } -static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte) +static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops) { - return __va(kvm_pte_to_phys(pte)); + return mm_ops->phys_to_virt(kvm_pte_to_phys(pte)); } -static void kvm_set_invalid_pte(kvm_pte_t *ptep) +static void kvm_clear_pte(kvm_pte_t *ptep) { - kvm_pte_t pte = *ptep; - WRITE_ONCE(*ptep, pte & ~KVM_PTE_VALID); + WRITE_ONCE(*ptep, 0); } -static void kvm_set_table_pte(kvm_pte_t *ptep, kvm_pte_t *childp) +static void kvm_set_table_pte(kvm_pte_t *ptep, kvm_pte_t *childp, + struct kvm_pgtable_mm_ops *mm_ops) { - kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(__pa(childp)); + kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp)); pte |= FIELD_PREP(KVM_PTE_TYPE, KVM_PTE_TYPE_TABLE); pte |= KVM_PTE_VALID; @@ -187,6 +206,11 @@ static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level) return pte; } +static kvm_pte_t kvm_init_invalid_leaf_owner(u8 owner_id) +{ + return FIELD_PREP(KVM_INVALID_PTE_OWNER_MASK, owner_id); +} + static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, u64 addr, u32 level, kvm_pte_t *ptep, enum kvm_pgtable_walk_flags flag) @@ -228,7 +252,7 @@ static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, goto out; } - childp = kvm_pte_follow(pte); + childp = kvm_pte_follow(pte, data->pgt->mm_ops); ret = __kvm_pgtable_walk(data, childp, level + 1); if (ret) goto out; @@ -303,12 +327,12 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, } struct hyp_map_data { - u64 phys; - kvm_pte_t attr; + u64 phys; + kvm_pte_t attr; + struct kvm_pgtable_mm_ops *mm_ops; }; -static int hyp_map_set_prot_attr(enum kvm_pgtable_prot prot, - struct hyp_map_data *data) +static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep) { bool device = prot & KVM_PGTABLE_PROT_DEVICE; u32 mtype = device ? MT_DEVICE_nGnRE : MT_NORMAL; @@ -333,7 +357,8 @@ static int hyp_map_set_prot_attr(enum kvm_pgtable_prot prot, attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_AP, ap); attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_SH, sh); attr |= KVM_PTE_LEAF_ATTR_LO_S1_AF; - data->attr = attr; + *ptep = attr; + return 0; } @@ -359,6 +384,8 @@ static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, enum kvm_pgtable_walk_flags flag, void * const arg) { kvm_pte_t *childp; + struct hyp_map_data *data = arg; + struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops; if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg)) return 0; @@ -366,11 +393,11 @@ static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1)) return -EINVAL; - childp = (kvm_pte_t *)get_zeroed_page(GFP_KERNEL); + childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL); if (!childp) return -ENOMEM; - kvm_set_table_pte(ptep, childp); + kvm_set_table_pte(ptep, childp, mm_ops); return 0; } @@ -380,6 +407,7 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, int ret; struct hyp_map_data map_data = { .phys = ALIGN_DOWN(phys, PAGE_SIZE), + .mm_ops = pgt->mm_ops, }; struct kvm_pgtable_walker walker = { .cb = hyp_map_walker, @@ -387,7 +415,7 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, .arg = &map_data, }; - ret = hyp_map_set_prot_attr(prot, &map_data); + ret = hyp_set_prot_attr(prot, &map_data.attr); if (ret) return ret; @@ -397,16 +425,18 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, return ret; } -int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits) +int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits, + struct kvm_pgtable_mm_ops *mm_ops) { u64 levels = ARM64_HW_PGTABLE_LEVELS(va_bits); - pgt->pgd = (kvm_pte_t *)get_zeroed_page(GFP_KERNEL); + pgt->pgd = (kvm_pte_t *)mm_ops->zalloc_page(NULL); if (!pgt->pgd) return -ENOMEM; pgt->ia_bits = va_bits; pgt->start_level = KVM_PGTABLE_MAX_LEVELS - levels; + pgt->mm_ops = mm_ops; pgt->mmu = NULL; return 0; } @@ -414,7 +444,9 @@ int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits) static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, enum kvm_pgtable_walk_flags flag, void * const arg) { - free_page((unsigned long)kvm_pte_follow(*ptep)); + struct kvm_pgtable_mm_ops *mm_ops = arg; + + mm_ops->put_page((void *)kvm_pte_follow(*ptep, mm_ops)); return 0; } @@ -423,30 +455,78 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt) struct kvm_pgtable_walker walker = { .cb = hyp_free_walker, .flags = KVM_PGTABLE_WALK_TABLE_POST, + .arg = pgt->mm_ops, }; WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker)); - free_page((unsigned long)pgt->pgd); + pgt->mm_ops->put_page(pgt->pgd); pgt->pgd = NULL; } struct stage2_map_data { u64 phys; kvm_pte_t attr; + u8 owner_id; kvm_pte_t *anchor; + kvm_pte_t *childp; struct kvm_s2_mmu *mmu; - struct kvm_mmu_memory_cache *memcache; + void *memcache; + + struct kvm_pgtable_mm_ops *mm_ops; }; -static int stage2_map_set_prot_attr(enum kvm_pgtable_prot prot, - struct stage2_map_data *data) +u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift) +{ + u64 vtcr = VTCR_EL2_FLAGS; + u8 lvls; + + vtcr |= kvm_get_parange(mmfr0) << VTCR_EL2_PS_SHIFT; + vtcr |= VTCR_EL2_T0SZ(phys_shift); + /* + * Use a minimum 2 level page table to prevent splitting + * host PMD huge pages at stage2. + */ + lvls = stage2_pgtable_levels(phys_shift); + if (lvls < 2) + lvls = 2; + vtcr |= VTCR_EL2_LVLS_TO_SL0(lvls); + + /* + * Enable the Hardware Access Flag management, unconditionally + * on all CPUs. The features is RES0 on CPUs without the support + * and must be ignored by the CPUs. + */ + vtcr |= VTCR_EL2_HA; + + /* Set the vmid bits */ + vtcr |= (get_vmid_bits(mmfr1) == 16) ? + VTCR_EL2_VS_16BIT : + VTCR_EL2_VS_8BIT; + + return vtcr; +} + +static bool stage2_has_fwb(struct kvm_pgtable *pgt) +{ + if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) + return false; + + return !(pgt->flags & KVM_PGTABLE_S2_NOFWB); +} + +static int stage2_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep, + struct kvm_pgtable *pgt) { bool device = prot & KVM_PGTABLE_PROT_DEVICE; - kvm_pte_t attr = device ? PAGE_S2_MEMATTR(DEVICE_nGnRE) : - PAGE_S2_MEMATTR(NORMAL); u32 sh = KVM_PTE_LEAF_ATTR_LO_S2_SH_IS; + kvm_pte_t attr; + + if (device) + attr = PAGE_S2_MEMATTR(DEVICE_nGnRE, stage2_has_fwb(pgt)); + else + attr = PAGE_S2_MEMATTR(NORMAL, stage2_has_fwb(pgt)); if (!(prot & KVM_PGTABLE_PROT_X)) attr |= KVM_PTE_LEAF_ATTR_HI_S2_XN; @@ -461,44 +541,78 @@ static int stage2_map_set_prot_attr(enum kvm_pgtable_prot prot, attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S2_SH, sh); attr |= KVM_PTE_LEAF_ATTR_LO_S2_AF; - data->attr = attr; + *ptep = attr; + return 0; } +static bool stage2_pte_needs_update(kvm_pte_t old, kvm_pte_t new) +{ + if (!kvm_pte_valid(old) || !kvm_pte_valid(new)) + return true; + + return ((old ^ new) & (~KVM_PTE_LEAF_ATTR_S2_PERMS)); +} + +static bool stage2_pte_is_counted(kvm_pte_t pte) +{ + /* + * The refcount tracks valid entries as well as invalid entries if they + * encode ownership of a page to another entity than the page-table + * owner, whose id is 0. + */ + return !!pte; +} + +static void stage2_put_pte(kvm_pte_t *ptep, struct kvm_s2_mmu *mmu, u64 addr, + u32 level, struct kvm_pgtable_mm_ops *mm_ops) +{ + /* + * Clear the existing PTE, and perform break-before-make with + * TLB maintenance if it was valid. + */ + if (kvm_pte_valid(*ptep)) { + kvm_clear_pte(ptep); + kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level); + } + + mm_ops->put_page(ptep); +} + static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, struct stage2_map_data *data) { kvm_pte_t new, old = *ptep; u64 granule = kvm_granule_size(level), phys = data->phys; - struct page *page = virt_to_page(ptep); + struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops; if (!kvm_block_mapping_supported(addr, end, phys, level)) return -E2BIG; - new = kvm_init_valid_leaf_pte(phys, data->attr, level); - if (kvm_pte_valid(old)) { + if (kvm_phys_is_valid(phys)) + new = kvm_init_valid_leaf_pte(phys, data->attr, level); + else + new = kvm_init_invalid_leaf_owner(data->owner_id); + + if (stage2_pte_is_counted(old)) { /* * Skip updating the PTE if we are trying to recreate the exact * same mapping or only change the access permissions. Instead, * the vCPU will exit one more time from guest if still needed * and then go through the path of relaxing permissions. */ - if (!((old ^ new) & (~KVM_PTE_LEAF_ATTR_S2_PERMS))) + if (!stage2_pte_needs_update(old, new)) return -EAGAIN; - /* - * There's an existing different valid leaf entry, so perform - * break-before-make. - */ - kvm_set_invalid_pte(ptep); - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level); - put_page(page); + stage2_put_pte(ptep, data->mmu, addr, level, mm_ops); } smp_store_release(ptep, new); - get_page(page); - data->phys += granule; + if (stage2_pte_is_counted(new)) + mm_ops->get_page(ptep); + if (kvm_phys_is_valid(phys)) + data->phys += granule; return 0; } @@ -512,7 +626,8 @@ static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level, if (!kvm_block_mapping_supported(addr, end, data->phys, level)) return 0; - kvm_set_invalid_pte(ptep); + data->childp = kvm_pte_follow(*ptep, data->mm_ops); + kvm_clear_pte(ptep); /* * Invalidate the whole stage-2, as we may have numerous leaf @@ -527,13 +642,13 @@ static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level, static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, struct stage2_map_data *data) { - int ret; + struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops; kvm_pte_t *childp, pte = *ptep; - struct page *page = virt_to_page(ptep); + int ret; if (data->anchor) { - if (kvm_pte_valid(pte)) - put_page(page); + if (stage2_pte_is_counted(pte)) + mm_ops->put_page(ptep); return 0; } @@ -548,7 +663,7 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, if (!data->memcache) return -ENOMEM; - childp = kvm_mmu_memory_cache_alloc(data->memcache); + childp = mm_ops->zalloc_page(data->memcache); if (!childp) return -ENOMEM; @@ -557,14 +672,11 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, * a table. Accesses beyond 'end' that fall within the new table * will be mapped lazily. */ - if (kvm_pte_valid(pte)) { - kvm_set_invalid_pte(ptep); - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level); - put_page(page); - } + if (stage2_pte_is_counted(pte)) + stage2_put_pte(ptep, data->mmu, addr, level, mm_ops); - kvm_set_table_pte(ptep, childp); - get_page(page); + kvm_set_table_pte(ptep, childp, mm_ops); + mm_ops->get_page(ptep); return 0; } @@ -573,19 +685,25 @@ static int stage2_map_walk_table_post(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, struct stage2_map_data *data) { + struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops; + kvm_pte_t *childp; int ret = 0; if (!data->anchor) return 0; - free_page((unsigned long)kvm_pte_follow(*ptep)); - put_page(virt_to_page(ptep)); - if (data->anchor == ptep) { + childp = data->childp; data->anchor = NULL; + data->childp = NULL; ret = stage2_map_walk_leaf(addr, end, level, ptep, data); + } else { + childp = kvm_pte_follow(*ptep, mm_ops); } + mm_ops->put_page(childp); + mm_ops->put_page(ptep); + return ret; } @@ -627,13 +745,14 @@ static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, enum kvm_pgtable_prot prot, - struct kvm_mmu_memory_cache *mc) + void *mc) { int ret; struct stage2_map_data map_data = { .phys = ALIGN_DOWN(phys, PAGE_SIZE), .mmu = pgt->mmu, .memcache = mc, + .mm_ops = pgt->mm_ops, }; struct kvm_pgtable_walker walker = { .cb = stage2_map_walker, @@ -643,7 +762,7 @@ int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, .arg = &map_data, }; - ret = stage2_map_set_prot_attr(prot, &map_data); + ret = stage2_set_prot_attr(prot, &map_data.attr, pgt); if (ret) return ret; @@ -652,38 +771,69 @@ int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, return ret; } +int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size, + void *mc, u8 owner_id) +{ + int ret; + struct stage2_map_data map_data = { + .phys = KVM_PHYS_INVALID, + .mmu = pgt->mmu, + .memcache = mc, + .mm_ops = pgt->mm_ops, + .owner_id = owner_id, + }; + struct kvm_pgtable_walker walker = { + .cb = stage2_map_walker, + .flags = KVM_PGTABLE_WALK_TABLE_PRE | + KVM_PGTABLE_WALK_LEAF | + KVM_PGTABLE_WALK_TABLE_POST, + .arg = &map_data, + }; + + if (owner_id > KVM_MAX_OWNER_ID) + return -EINVAL; + + ret = kvm_pgtable_walk(pgt, addr, size, &walker); + dsb(ishst); + return ret; +} + static void stage2_flush_dcache(void *addr, u64 size) { - if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) - return; - __flush_dcache_area(addr, size); } -static bool stage2_pte_cacheable(kvm_pte_t pte) +static bool stage2_pte_cacheable(kvm_pte_t pte, struct kvm_pgtable *pgt) { u64 memattr = pte & KVM_PTE_LEAF_ATTR_LO_S2_MEMATTR; - return memattr == PAGE_S2_MEMATTR(NORMAL); + return memattr == PAGE_S2_MEMATTR(NORMAL, stage2_has_fwb(pgt)); } static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, enum kvm_pgtable_walk_flags flag, void * const arg) { - struct kvm_s2_mmu *mmu = arg; + struct kvm_pgtable *pgt = arg; + struct kvm_s2_mmu *mmu = pgt->mmu; + struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops; kvm_pte_t pte = *ptep, *childp = NULL; bool need_flush = false; - if (!kvm_pte_valid(pte)) + if (!kvm_pte_valid(pte)) { + if (stage2_pte_is_counted(pte)) { + kvm_clear_pte(ptep); + mm_ops->put_page(ptep); + } return 0; + } if (kvm_pte_table(pte, level)) { - childp = kvm_pte_follow(pte); + childp = kvm_pte_follow(pte, mm_ops); - if (page_count(virt_to_page(childp)) != 1) + if (mm_ops->page_count(childp) != 1) return 0; - } else if (stage2_pte_cacheable(pte)) { - need_flush = true; + } else if (stage2_pte_cacheable(pte, pgt)) { + need_flush = !stage2_has_fwb(pgt); } /* @@ -691,17 +841,15 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, * block entry and rely on the remaining portions being faulted * back lazily. */ - kvm_set_invalid_pte(ptep); - kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level); - put_page(virt_to_page(ptep)); + stage2_put_pte(ptep, mmu, addr, level, mm_ops); if (need_flush) { - stage2_flush_dcache(kvm_pte_follow(pte), + stage2_flush_dcache(kvm_pte_follow(pte, mm_ops), kvm_granule_size(level)); } if (childp) - free_page((unsigned long)childp); + mm_ops->put_page(childp); return 0; } @@ -710,7 +858,7 @@ int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size) { struct kvm_pgtable_walker walker = { .cb = stage2_unmap_walker, - .arg = pgt->mmu, + .arg = pgt, .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST, }; @@ -842,12 +990,14 @@ static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, enum kvm_pgtable_walk_flags flag, void * const arg) { + struct kvm_pgtable *pgt = arg; + struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops; kvm_pte_t pte = *ptep; - if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pte)) + if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pte, pgt)) return 0; - stage2_flush_dcache(kvm_pte_follow(pte), kvm_granule_size(level)); + stage2_flush_dcache(kvm_pte_follow(pte, mm_ops), kvm_granule_size(level)); return 0; } @@ -856,30 +1006,35 @@ int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) struct kvm_pgtable_walker walker = { .cb = stage2_flush_walker, .flags = KVM_PGTABLE_WALK_LEAF, + .arg = pgt, }; - if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) + if (stage2_has_fwb(pgt)) return 0; return kvm_pgtable_walk(pgt, addr, size, &walker); } -int kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm *kvm) +int kvm_pgtable_stage2_init_flags(struct kvm_pgtable *pgt, struct kvm_arch *arch, + struct kvm_pgtable_mm_ops *mm_ops, + enum kvm_pgtable_stage2_flags flags) { size_t pgd_sz; - u64 vtcr = kvm->arch.vtcr; + u64 vtcr = arch->vtcr; u32 ia_bits = VTCR_EL2_IPA(vtcr); u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr); u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0; pgd_sz = kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE; - pgt->pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT | __GFP_ZERO); + pgt->pgd = mm_ops->zalloc_pages_exact(pgd_sz); if (!pgt->pgd) return -ENOMEM; + pgt->flags = flags; pgt->ia_bits = ia_bits; pgt->start_level = start_level; - pgt->mmu = &kvm->arch.mmu; + pgt->mm_ops = mm_ops; + pgt->mmu = &arch->mmu; /* Ensure zeroed PGD pages are visible to the hardware walker */ dsb(ishst); @@ -890,15 +1045,16 @@ static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, enum kvm_pgtable_walk_flags flag, void * const arg) { + struct kvm_pgtable_mm_ops *mm_ops = arg; kvm_pte_t pte = *ptep; - if (!kvm_pte_valid(pte)) + if (!stage2_pte_is_counted(pte)) return 0; - put_page(virt_to_page(ptep)); + mm_ops->put_page(ptep); if (kvm_pte_table(pte, level)) - free_page((unsigned long)kvm_pte_follow(pte)); + mm_ops->put_page(kvm_pte_follow(pte, mm_ops)); return 0; } @@ -910,10 +1066,85 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt) .cb = stage2_free_walker, .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST, + .arg = pgt->mm_ops, }; WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker)); pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE; - free_pages_exact(pgt->pgd, pgd_sz); + pgt->mm_ops->free_pages_exact(pgt->pgd, pgd_sz); pgt->pgd = NULL; } + +#define KVM_PTE_LEAF_S2_COMPAT_MASK (KVM_PTE_LEAF_ATTR_S2_PERMS | \ + KVM_PTE_LEAF_ATTR_LO_S2_MEMATTR | \ + KVM_PTE_LEAF_ATTR_S2_IGNORED) + +static int stage2_check_permission_walker(u64 addr, u64 end, u32 level, + kvm_pte_t *ptep, + enum kvm_pgtable_walk_flags flag, + void * const arg) +{ + kvm_pte_t old_attr, pte = *ptep, *new_attr = arg; + + /* + * Compatible mappings are either invalid and owned by the page-table + * owner (whose id is 0), or valid with matching permission attributes. + */ + if (kvm_pte_valid(pte)) { + old_attr = pte & KVM_PTE_LEAF_S2_COMPAT_MASK; + if (old_attr != *new_attr) + return -EEXIST; + } else if (pte) { + return -EEXIST; + } + + return 0; +} + +int kvm_pgtable_stage2_find_range(struct kvm_pgtable *pgt, u64 addr, + enum kvm_pgtable_prot prot, + struct kvm_mem_range *range) +{ + kvm_pte_t attr; + struct kvm_pgtable_walker check_perm_walker = { + .cb = stage2_check_permission_walker, + .flags = KVM_PGTABLE_WALK_LEAF, + .arg = &attr, + }; + u64 granule, start, end; + u32 level; + int ret; + + ret = stage2_set_prot_attr(prot, &attr, pgt); + if (ret) + return ret; + attr &= KVM_PTE_LEAF_S2_COMPAT_MASK; + + for (level = pgt->start_level; level < KVM_PGTABLE_MAX_LEVELS; level++) { + granule = kvm_granule_size(level); + start = ALIGN_DOWN(addr, granule); + end = start + granule; + + if (!kvm_level_supports_block_mapping(level)) + continue; + + if (start < range->start || range->end < end) + continue; + + /* + * Check the presence of existing mappings with incompatible + * permissions within the current block range, and try one level + * deeper if one is found. + */ + ret = kvm_pgtable_walk(pgt, start, granule, &check_perm_walker); + if (ret != -EEXIST) + break; + } + + if (!ret) { + range->start = start; + range->end = end; + } + + return ret; +} diff --git a/arch/arm64/kvm/hyp/reserved_mem.c b/arch/arm64/kvm/hyp/reserved_mem.c new file mode 100644 index 000000000000..83ca23ac259b --- /dev/null +++ b/arch/arm64/kvm/hyp/reserved_mem.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 - Google LLC + * Author: Quentin Perret + */ + +#include +#include +#include + +#include + +#include +#include + +static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory); +static unsigned int *hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr); + +phys_addr_t hyp_mem_base; +phys_addr_t hyp_mem_size; + +static int cmp_hyp_memblock(const void *p1, const void *p2) +{ + const struct memblock_region *r1 = p1; + const struct memblock_region *r2 = p2; + + return r1->base < r2->base ? -1 : (r1->base > r2->base); +} + +static void __init sort_memblock_regions(void) +{ + sort(hyp_memory, + *hyp_memblock_nr_ptr, + sizeof(struct memblock_region), + cmp_hyp_memblock, + NULL); +} + +static int __init register_memblock_regions(void) +{ + struct memblock_region *reg; + + for_each_mem_region(reg) { + if (*hyp_memblock_nr_ptr >= HYP_MEMBLOCK_REGIONS) + return -ENOMEM; + + hyp_memory[*hyp_memblock_nr_ptr] = *reg; + (*hyp_memblock_nr_ptr)++; + } + sort_memblock_regions(); + + return 0; +} + +void __init kvm_hyp_reserve(void) +{ + u64 nr_pages, prev, hyp_mem_pages = 0; + int ret; + + if (!is_hyp_mode_available() || is_kernel_in_hyp_mode()) + return; + + if (kvm_get_mode() != KVM_MODE_PROTECTED) + return; + + ret = register_memblock_regions(); + if (ret) { + *hyp_memblock_nr_ptr = 0; + kvm_err("Failed to register hyp memblocks: %d\n", ret); + return; + } + + hyp_mem_pages += hyp_s1_pgtable_pages(); + hyp_mem_pages += host_s2_mem_pgtable_pages(); + hyp_mem_pages += host_s2_dev_pgtable_pages(); + + /* + * The hyp_vmemmap needs to be backed by pages, but these pages + * themselves need to be present in the vmemmap, so compute the number + * of pages needed by looking for a fixed point. + */ + nr_pages = 0; + do { + prev = nr_pages; + nr_pages = hyp_mem_pages + prev; + nr_pages = DIV_ROUND_UP(nr_pages * sizeof(struct hyp_page), PAGE_SIZE); + nr_pages += __hyp_pgtable_max_pages(nr_pages); + } while (nr_pages != prev); + hyp_mem_pages += nr_pages; + + /* + * Try to allocate a PMD-aligned region to reduce TLB pressure once + * this is unmapped from the host stage-2, and fallback to PAGE_SIZE. + */ + hyp_mem_size = hyp_mem_pages << PAGE_SHIFT; + hyp_mem_base = memblock_find_in_range(0, memblock_end_of_DRAM(), + ALIGN(hyp_mem_size, PMD_SIZE), + PMD_SIZE); + if (!hyp_mem_base) + hyp_mem_base = memblock_find_in_range(0, memblock_end_of_DRAM(), + hyp_mem_size, PAGE_SIZE); + else + hyp_mem_size = ALIGN(hyp_mem_size, PMD_SIZE); + + if (!hyp_mem_base) { + kvm_err("Failed to reserve hyp memory\n"); + return; + } + memblock_reserve(hyp_mem_base, hyp_mem_size); + + kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20, + hyp_mem_base); +} diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 8711894db8c2..0f16b70befa8 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -88,6 +88,44 @@ static bool kvm_is_device_pfn(unsigned long pfn) return !pfn_valid(pfn); } +static void *stage2_memcache_zalloc_page(void *arg) +{ + struct kvm_mmu_memory_cache *mc = arg; + + /* Allocated with __GFP_ZERO, so no need to zero */ + return kvm_mmu_memory_cache_alloc(mc); +} + +static void *kvm_host_zalloc_pages_exact(size_t size) +{ + return alloc_pages_exact(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO); +} + +static void kvm_host_get_page(void *addr) +{ + get_page(virt_to_page(addr)); +} + +static void kvm_host_put_page(void *addr) +{ + put_page(virt_to_page(addr)); +} + +static int kvm_host_page_count(void *addr) +{ + return page_count(virt_to_page(addr)); +} + +static phys_addr_t kvm_host_pa(void *addr) +{ + return __pa(addr); +} + +static void *kvm_host_va(phys_addr_t phys) +{ + return __va(phys); +} + /* * Unmapping vs dcache management: * @@ -127,7 +165,7 @@ static bool kvm_is_device_pfn(unsigned long pfn) static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, bool may_block) { - struct kvm *kvm = mmu->kvm; + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); phys_addr_t end = start + size; assert_spin_locked(&kvm->mmu_lock); @@ -183,15 +221,39 @@ void free_hyp_pgds(void) if (hyp_pgtable) { kvm_pgtable_hyp_destroy(hyp_pgtable); kfree(hyp_pgtable); + hyp_pgtable = NULL; } mutex_unlock(&kvm_hyp_pgd_mutex); } +static bool kvm_host_owns_hyp_mappings(void) +{ + if (static_branch_likely(&kvm_protected_mode_initialized)) + return false; + + /* + * This can happen at boot time when __create_hyp_mappings() is called + * after the hyp protection has been enabled, but the static key has + * not been flipped yet. + */ + if (!hyp_pgtable && is_protected_kvm_enabled()) + return false; + + WARN_ON(!hyp_pgtable); + + return true; +} + static int __create_hyp_mappings(unsigned long start, unsigned long size, unsigned long phys, enum kvm_pgtable_prot prot) { int err; + if (!kvm_host_owns_hyp_mappings()) { + return kvm_call_hyp_nvhe(__pkvm_create_mappings, + start, size, phys, prot); + } + mutex_lock(&kvm_hyp_pgd_mutex); err = kvm_pgtable_hyp_map(hyp_pgtable, start, size, phys, prot); mutex_unlock(&kvm_hyp_pgd_mutex); @@ -253,6 +315,16 @@ static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size, unsigned long base; int ret = 0; + if (!kvm_host_owns_hyp_mappings()) { + base = kvm_call_hyp_nvhe(__pkvm_create_private_mapping, + phys_addr, size, prot); + if (IS_ERR_OR_NULL((void *)base)) + return PTR_ERR((void *)base); + *haddr = base; + + return 0; + } + mutex_lock(&kvm_hyp_pgd_mutex); /* @@ -351,6 +423,17 @@ int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size, return 0; } +static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = { + .zalloc_page = stage2_memcache_zalloc_page, + .zalloc_pages_exact = kvm_host_zalloc_pages_exact, + .free_pages_exact = free_pages_exact, + .get_page = kvm_host_get_page, + .put_page = kvm_host_put_page, + .page_count = kvm_host_page_count, + .phys_to_virt = kvm_host_va, + .virt_to_phys = kvm_host_pa, +}; + /** * kvm_init_stage2_mmu - Initialise a S2 MMU strucrure * @kvm: The pointer to the KVM structure @@ -374,7 +457,7 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) if (!pgt) return -ENOMEM; - err = kvm_pgtable_stage2_init(pgt, kvm); + err = kvm_pgtable_stage2_init(pgt, &kvm->arch, &kvm_s2_mm_ops); if (err) goto out_free_pgtable; @@ -387,7 +470,7 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) for_each_possible_cpu(cpu) *per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1; - mmu->kvm = kvm; + mmu->arch = &kvm->arch; mmu->pgt = pgt; mmu->pgd_phys = __pa(pgt->pgd); mmu->vmid.vmid_gen = 0; @@ -469,7 +552,7 @@ void stage2_unmap_vm(struct kvm *kvm) void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) { - struct kvm *kvm = mmu->kvm; + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); struct kvm_pgtable *pgt = NULL; spin_lock(&kvm->mmu_lock); @@ -538,7 +621,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, */ static void stage2_wp_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end) { - struct kvm *kvm = mmu->kvm; + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); stage2_apply_range_resched(kvm, addr, end, kvm_pgtable_stage2_wrprotect); } @@ -1208,10 +1291,22 @@ static int kvm_map_idmap_text(void) return err; } -int kvm_mmu_init(void) +static void *kvm_hyp_zalloc_page(void *arg) +{ + return (void *)get_zeroed_page(GFP_KERNEL); +} + +static struct kvm_pgtable_mm_ops kvm_hyp_mm_ops = { + .zalloc_page = kvm_hyp_zalloc_page, + .get_page = kvm_host_get_page, + .put_page = kvm_host_put_page, + .phys_to_virt = kvm_host_va, + .virt_to_phys = kvm_host_pa, +}; + +int kvm_mmu_init(u32 *hyp_va_bits) { int err; - u32 hyp_va_bits; hyp_idmap_start = __pa_symbol(__hyp_idmap_text_start); hyp_idmap_start = ALIGN_DOWN(hyp_idmap_start, PAGE_SIZE); @@ -1225,8 +1320,8 @@ int kvm_mmu_init(void) */ BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK); - hyp_va_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET); - kvm_debug("Using %u-bit virtual addresses at EL2\n", hyp_va_bits); + *hyp_va_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET); + kvm_debug("Using %u-bit virtual addresses at EL2\n", *hyp_va_bits); kvm_debug("IDMAP page: %lx\n", hyp_idmap_start); kvm_debug("HYP VA range: %lx:%lx\n", kern_hyp_va(PAGE_OFFSET), @@ -1251,7 +1346,7 @@ int kvm_mmu_init(void) goto out; } - err = kvm_pgtable_hyp_init(hyp_pgtable, hyp_va_bits); + err = kvm_pgtable_hyp_init(hyp_pgtable, *hyp_va_bits, &kvm_hyp_mm_ops); if (err) goto out_free_pgtable; diff --git a/arch/arm64/kvm/perf.c b/arch/arm64/kvm/perf.c index 739164324afe..8f860ae56bb7 100644 --- a/arch/arm64/kvm/perf.c +++ b/arch/arm64/kvm/perf.c @@ -55,7 +55,8 @@ int kvm_perf_init(void) * hardware performance counters. This could ensure the presence of * a physical PMU and CONFIG_PERF_EVENT is selected. */ - if (IS_ENABLED(CONFIG_ARM_PMU) && perf_num_counters() > 0) + if (IS_ENABLED(CONFIG_ARM_PMU) && perf_num_counters() > 0 + && !is_protected_kvm_enabled()) static_branch_enable(&kvm_arm_pmu_available); return perf_register_guest_info_callbacks(&kvm_guest_cbs); diff --git a/arch/arm64/kvm/pmu.c b/arch/arm64/kvm/pmu.c index faf32a44ba04..03a6c1f4a09a 100644 --- a/arch/arm64/kvm/pmu.c +++ b/arch/arm64/kvm/pmu.c @@ -33,7 +33,7 @@ void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr) { struct kvm_host_data *ctx = this_cpu_ptr_hyp_sym(kvm_host_data); - if (!ctx || !kvm_pmu_switch_needed(attr)) + if (!kvm_arm_support_pmu_v3() || !ctx || !kvm_pmu_switch_needed(attr)) return; if (!attr->exclude_host) @@ -49,7 +49,7 @@ void kvm_clr_pmu_events(u32 clr) { struct kvm_host_data *ctx = this_cpu_ptr_hyp_sym(kvm_host_data); - if (!ctx) + if (!kvm_arm_support_pmu_v3() || !ctx) return; ctx->pmu_events.events_host &= ~clr; @@ -172,7 +172,7 @@ void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) struct kvm_host_data *host; u32 events_guest, events_host; - if (!has_vhe()) + if (!kvm_arm_support_pmu_v3() || !has_vhe()) return; preempt_disable(); @@ -193,7 +193,7 @@ void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) struct kvm_host_data *host; u32 events_guest, events_host; - if (!has_vhe()) + if (!kvm_arm_support_pmu_v3() || !has_vhe()) return; host = this_cpu_ptr_hyp_sym(kvm_host_data); diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 9d3d09a89894..e6f095062843 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -74,10 +74,6 @@ static int kvm_vcpu_enable_sve(struct kvm_vcpu *vcpu) if (!system_supports_sve()) return -EINVAL; - /* Verify that KVM startup enforced this when SVE was detected: */ - if (WARN_ON(!has_vhe())) - return -EINVAL; - vcpu->arch.sve_max_vl = kvm_sve_max_vl; /* @@ -331,19 +327,10 @@ int kvm_set_ipa_limit(void) return 0; } -/* - * Configure the VTCR_EL2 for this VM. The VTCR value is common - * across all the physical CPUs on the system. We use system wide - * sanitised values to fill in different fields, except for Hardware - * Management of Access Flags. HA Flag is set unconditionally on - * all CPUs, as it is safe to run with or without the feature and - * the bit is RES0 on CPUs that don't support it. - */ int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type) { - u64 vtcr = VTCR_EL2_FLAGS, mmfr0; - u32 parange, phys_shift; - u8 lvls; + u64 mmfr0, mmfr1; + u32 phys_shift; if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK) return -EINVAL; @@ -363,33 +350,8 @@ int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type) } mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); - parange = cpuid_feature_extract_unsigned_field(mmfr0, - ID_AA64MMFR0_PARANGE_SHIFT); - if (parange > ID_AA64MMFR0_PARANGE_MAX) - parange = ID_AA64MMFR0_PARANGE_MAX; - vtcr |= parange << VTCR_EL2_PS_SHIFT; + mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); + kvm->arch.vtcr = kvm_get_vtcr(mmfr0, mmfr1, phys_shift); - vtcr |= VTCR_EL2_T0SZ(phys_shift); - /* - * Use a minimum 2 level page table to prevent splitting - * host PMD huge pages at stage2. - */ - lvls = stage2_pgtable_levels(phys_shift); - if (lvls < 2) - lvls = 2; - vtcr |= VTCR_EL2_LVLS_TO_SL0(lvls); - - /* - * Enable the Hardware Access Flag management, unconditionally - * on all CPUs. The features is RES0 on CPUs without the support - * and must be ignored by the CPUs. - */ - vtcr |= VTCR_EL2_HA; - - /* Set the vmid bits */ - vtcr |= (kvm_get_vmid_bits() == 16) ? - VTCR_EL2_VS_16BIT : - VTCR_EL2_VS_8BIT; - kvm->arch.vtcr = vtcr; return 0; } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 4f2f1e3145de..dfb3b4f9ca84 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -2775,3 +2776,24 @@ void kvm_sys_reg_table_init(void) /* Clear all higher bits. */ cache_levels &= (1 << (i*3))-1; } + +#define CPU_FTR_REG_HYP_COPY(id, name) \ + { .sys_id = id, .dst = (struct arm64_ftr_reg *)&kvm_nvhe_sym(name) } +struct __ftr_reg_copy_entry { + u32 sys_id; + struct arm64_ftr_reg *dst; +} hyp_ftr_regs[] __initdata = { + CPU_FTR_REG_HYP_COPY(SYS_CTR_EL0, arm64_ftr_reg_ctrel0), + CPU_FTR_REG_HYP_COPY(SYS_ID_AA64MMFR0_EL1, arm64_ftr_reg_id_aa64mmfr0_el1), + CPU_FTR_REG_HYP_COPY(SYS_ID_AA64MMFR1_EL1, arm64_ftr_reg_id_aa64mmfr1_el1), +}; + +void __init setup_kvm_el2_caps(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(hyp_ftr_regs); i++) { + WARN(copy_ftr_reg(hyp_ftr_regs[i].sys_id, hyp_ftr_regs[i].dst), + "%u feature register not found\n", hyp_ftr_regs[i].sys_id); + } +} diff --git a/arch/arm64/lib/clear_page.S b/arch/arm64/lib/clear_page.S index 073acbf02a7c..b84b179edba3 100644 --- a/arch/arm64/lib/clear_page.S +++ b/arch/arm64/lib/clear_page.S @@ -14,7 +14,7 @@ * Parameters: * x0 - dest */ -SYM_FUNC_START(clear_page) +SYM_FUNC_START_PI(clear_page) mrs x1, dczid_el0 and w1, w1, #0xf mov x2, #4 @@ -25,5 +25,5 @@ SYM_FUNC_START(clear_page) tst x0, #(PAGE_SIZE - 1) b.ne 1b ret -SYM_FUNC_END(clear_page) +SYM_FUNC_END_PI(clear_page) EXPORT_SYMBOL(clear_page) diff --git a/arch/arm64/lib/copy_page.S b/arch/arm64/lib/copy_page.S index e7a793961408..29144f4cd449 100644 --- a/arch/arm64/lib/copy_page.S +++ b/arch/arm64/lib/copy_page.S @@ -17,7 +17,7 @@ * x0 - dest * x1 - src */ -SYM_FUNC_START(copy_page) +SYM_FUNC_START_PI(copy_page) alternative_if ARM64_HAS_NO_HW_PREFETCH // Prefetch three cache lines ahead. prfm pldl1strm, [x1, #128] @@ -75,5 +75,5 @@ alternative_else_nop_endif stnp x16, x17, [x0, #112 - 256] ret -SYM_FUNC_END(copy_page) +SYM_FUNC_END_PI(copy_page) EXPORT_SYMBOL(copy_page) diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index c1654be1f128..8f213efbed43 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -433,6 +434,8 @@ void __init bootmem_init(void) dma_pernuma_cma_reserve(); + kvm_hyp_reserve(); + /* * sparse_init() tries to allocate memory from memblock, so must be * done after the fixed reservations diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index ef9f1d5e989d..af75d5909a0f 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "bpf_jit.h" @@ -1098,6 +1099,8 @@ skip_init_ctx: goto out_off; } bpf_jit_binary_lock_ro(header); + trace_android_vh_set_memory_ro((unsigned long)header, header->pages); + trace_android_vh_set_memory_x((unsigned long)header, header->pages); } else { jit_data->ctx = ctx; jit_data->image = image_ptr; diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index e4b5a0c6a6ad..7586acb562aa 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -56,6 +56,7 @@ CONFIG_CMDLINE="stack_depot_disable=on" CONFIG_PM_WAKELOCKS=y CONFIG_PM_WAKELOCKS_LIMIT=0 # CONFIG_PM_WAKELOCKS_GC is not set +CONFIG_ENERGY_MODEL=y CONFIG_CPU_FREQ_STAT=y CONFIG_CPU_FREQ_TIMES=y CONFIG_CPU_FREQ_GOV_POWERSAVE=y @@ -443,6 +444,9 @@ CONFIG_PM_DEVFREQ_EVENT=y CONFIG_IIO=y CONFIG_IIO_BUFFER=y CONFIG_IIO_TRIGGER=y +CONFIG_POWERCAP=y +CONFIG_DTPM=y +CONFIG_DTPM_CPU=y CONFIG_RAS=y CONFIG_ANDROID=y CONFIG_ANDROID_BINDER_IPC=y @@ -561,15 +565,11 @@ CONFIG_DEBUG_INFO_DWARF4=y CONFIG_HEADERS_INSTALL=y # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_MAGIC_SYSRQ=y -CONFIG_UBSAN=y -CONFIG_UBSAN_TRAP=y -CONFIG_UBSAN_LOCAL_BOUNDS=y -# CONFIG_UBSAN_MISC is not set CONFIG_PAGE_OWNER=y CONFIG_DEBUG_STACK_USAGE=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_KFENCE=y -CONFIG_KFENCE_SAMPLE_INTERVAL=0 +CONFIG_KFENCE_SAMPLE_INTERVAL=500 CONFIG_PANIC_ON_OOPS=y CONFIG_PANIC_TIMEOUT=-1 CONFIG_WQ_WATCHDOG=y diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index d40e431c2aca..8ba5e8c9642b 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -40,6 +40,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include /* * Export tracepoints that act as a bare tracehook (ie: have no trace event @@ -100,9 +106,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_busiest_queue); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_migrate_queued_task); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_resume_cpus); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_energy_efficient_cpu); -EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_sugov_sched_attr); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_iowait); -EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_sugov_update); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpu_overutilized); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_setaffinity); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_cpus_allowed); @@ -170,12 +174,44 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_compl_command); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_send_uic_command); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_send_tm_command); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_check_int_errors); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_update_sdev); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_wake_up_sync); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_wake_flags); -EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_uclamp_eff_value); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_uclamp_eff_get); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpufreq_transition); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cgroup_set_task); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_syscall_prctl_finished); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_tick); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_wakeup_ignore); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_replace_next_task_fair); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_build_sched_domains); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alter_mutex_list_add); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_unlock_slowpath); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_wake_finish); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_undefinstr); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_bad_mode); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_arm64_serror_panic); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_insert); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_node_delete); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_node_replace); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_lookup); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_commit_creds); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_exit_creds); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_override_creds); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_revert_creds); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_x); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_nx); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_ro); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_rw); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_module_permit_before_init); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_module_permit_after_init); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_util_est_update); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_meminfo_proc_show); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_exit_mm); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_pages_slowpath); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_mem); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_print_slabinfo_header); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cache_show); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_typec_tcpci_override_toggling); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_typec_tcpci_check_contaminant); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_typec_tcpci_get_vbus); diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 3e2b899cb931..afef1a6a1e52 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -648,8 +648,8 @@ static ssize_t writeback_store(struct device *dev, if (strncmp(buf, PAGE_WB_SIG, sizeof(PAGE_WB_SIG) - 1)) return -EINVAL; - ret = kstrtol(buf + sizeof(PAGE_WB_SIG) - 1, 10, &index); - if (ret || index >= nr_pages) + if (kstrtol(buf + sizeof(PAGE_WB_SIG) - 1, 10, &index) || + index >= nr_pages) return -EINVAL; nr_pages = 1; @@ -673,7 +673,7 @@ static ssize_t writeback_store(struct device *dev, goto release_init_lock; } - while (nr_pages--) { + for (; nr_pages != 0; index++, nr_pages--) { struct bio_vec bvec; bvec.bv_page = page; diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index 5b09a4c16be6..c203eddfb772 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -534,13 +534,13 @@ static int system_heap_create(void) for (i = 0; i < NUM_ORDERS; i++) { pools[i] = dmabuf_page_pool_create(order_flags[i], orders[i]); - if (IS_ERR(pools[i])) { + if (!pools[i]) { int j; pr_err("%s: page pool creation failed!\n", __func__); for (j = 0; j < i; j++) dmabuf_page_pool_destroy(pools[j]); - return PTR_ERR(pools[i]); + return -ENOMEM; } } diff --git a/drivers/mmc/core/crypto.c b/drivers/mmc/core/crypto.c index 419a368f8402..67557808cada 100644 --- a/drivers/mmc/core/crypto.c +++ b/drivers/mmc/core/crypto.c @@ -31,18 +31,11 @@ void mmc_crypto_prepare_req(struct mmc_queue_req *mqrq) struct request *req = mmc_queue_req_to_req(mqrq); struct mmc_request *mrq = &mqrq->brq.mrq; - if (!req->crypt_keyslot) + if (!req->crypt_ctx) return; - mrq->crypto_enabled = true; - mrq->crypto_key_slot = blk_ksm_get_slot_idx(req->crypt_keyslot); - - /* - * For now we assume that all MMC drivers set max_dun_bytes_supported=4, - * which is the limit for CQHCI crypto. So all DUNs should be 32-bit. - */ - WARN_ON_ONCE(req->crypt_ctx->bc_dun[0] > U32_MAX); - - mrq->data_unit_num = req->crypt_ctx->bc_dun[0]; + mrq->crypto_ctx = req->crypt_ctx; + if (req->crypt_keyslot) + mrq->crypto_key_slot = blk_ksm_get_slot_idx(req->crypt_keyslot); } EXPORT_SYMBOL_GPL(mmc_crypto_prepare_req); diff --git a/drivers/mmc/host/cqhci-crypto.h b/drivers/mmc/host/cqhci-crypto.h index 60b58ee0e625..d7fb084f563b 100644 --- a/drivers/mmc/host/cqhci-crypto.h +++ b/drivers/mmc/host/cqhci-crypto.h @@ -22,12 +22,15 @@ int cqhci_crypto_init(struct cqhci_host *host); */ static inline u64 cqhci_crypto_prep_task_desc(struct mmc_request *mrq) { - if (!mrq->crypto_enabled) + if (!mrq->crypto_ctx) return 0; + /* We set max_dun_bytes_supported=4, so all DUNs should be 32-bit. */ + WARN_ON_ONCE(mrq->crypto_ctx->bc_dun[0] > U32_MAX); + return CQHCI_CRYPTO_ENABLE_BIT | CQHCI_CRYPTO_KEYSLOT(mrq->crypto_key_slot) | - mrq->data_unit_num; + mrq->crypto_ctx->bc_dun[0]; } #else /* CONFIG_MMC_CRYPTO */ diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index f86cae9aa1f4..9821dddb98fb 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -3,6 +3,7 @@ #define DRIVERS_PCI_H #include +#include /* Number of possible devfns: 0.0 to 1f.7 inclusive */ #define MAX_NR_DEVFNS 256 @@ -343,6 +344,11 @@ struct pci_sriov { u16 subsystem_device; /* VF subsystem device */ resource_size_t barsz[PCI_SRIOV_NUM_BARS]; /* VF BAR size */ bool drivers_autoprobe; /* Auto probing of VFs by driver */ + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /** diff --git a/drivers/powercap/Kconfig b/drivers/powercap/Kconfig index bc228725346b..20b4325c6161 100644 --- a/drivers/powercap/Kconfig +++ b/drivers/powercap/Kconfig @@ -43,4 +43,17 @@ config IDLE_INJECT CPUs for power capping. Idle period can be injected synchronously on a set of specified CPUs or alternatively on a per CPU basis. + +config DTPM + bool "Power capping for Dynamic Thermal Power Management" + help + This enables support for the power capping for the dynamic + thermal power management userspace engine. + +config DTPM_CPU + bool "Add CPU power capping based on the energy model" + depends on DTPM && ENERGY_MODEL + help + This enables support for CPU power limitation based on + energy model. endif diff --git a/drivers/powercap/Makefile b/drivers/powercap/Makefile index 7255c94ec61c..fabcf388a8d3 100644 --- a/drivers/powercap/Makefile +++ b/drivers/powercap/Makefile @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_DTPM) += dtpm.o +obj-$(CONFIG_DTPM_CPU) += dtpm_cpu.o obj-$(CONFIG_POWERCAP) += powercap_sys.o obj-$(CONFIG_INTEL_RAPL_CORE) += intel_rapl_common.o obj-$(CONFIG_INTEL_RAPL) += intel_rapl_msr.o diff --git a/drivers/powercap/dtpm.c b/drivers/powercap/dtpm.c new file mode 100644 index 000000000000..5a51cd34a7e8 --- /dev/null +++ b/drivers/powercap/dtpm.c @@ -0,0 +1,480 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2020 Linaro Limited + * + * Author: Daniel Lezcano + * + * The powercap based Dynamic Thermal Power Management framework + * provides to the userspace a consistent API to set the power limit + * on some devices. + * + * DTPM defines the functions to create a tree of constraints. Each + * parent node is a virtual description of the aggregation of the + * children. It propagates the constraints set at its level to its + * children and collect the children power information. The leaves of + * the tree are the real devices which have the ability to get their + * current power consumption and set their power limit. + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#define DTPM_POWER_LIMIT_FLAG 0 + +static const char *constraint_name[] = { + "Instantaneous", +}; + +static DEFINE_MUTEX(dtpm_lock); +static struct powercap_control_type *pct; +static struct dtpm *root; + +static int get_time_window_us(struct powercap_zone *pcz, int cid, u64 *window) +{ + return -ENOSYS; +} + +static int set_time_window_us(struct powercap_zone *pcz, int cid, u64 window) +{ + return -ENOSYS; +} + +static int get_max_power_range_uw(struct powercap_zone *pcz, u64 *max_power_uw) +{ + struct dtpm *dtpm = to_dtpm(pcz); + + mutex_lock(&dtpm_lock); + *max_power_uw = dtpm->power_max - dtpm->power_min; + mutex_unlock(&dtpm_lock); + + return 0; +} + +static int __get_power_uw(struct dtpm *dtpm, u64 *power_uw) +{ + struct dtpm *child; + u64 power; + int ret = 0; + + if (dtpm->ops) { + *power_uw = dtpm->ops->get_power_uw(dtpm); + return 0; + } + + *power_uw = 0; + + list_for_each_entry(child, &dtpm->children, sibling) { + ret = __get_power_uw(child, &power); + if (ret) + break; + *power_uw += power; + } + + return ret; +} + +static int get_power_uw(struct powercap_zone *pcz, u64 *power_uw) +{ + struct dtpm *dtpm = to_dtpm(pcz); + int ret; + + mutex_lock(&dtpm_lock); + ret = __get_power_uw(dtpm, power_uw); + mutex_unlock(&dtpm_lock); + + return ret; +} + +static void __dtpm_rebalance_weight(struct dtpm *dtpm) +{ + struct dtpm *child; + + list_for_each_entry(child, &dtpm->children, sibling) { + + pr_debug("Setting weight '%d' for '%s'\n", + child->weight, child->zone.name); + + child->weight = DIV64_U64_ROUND_CLOSEST( + child->power_max * 1024, dtpm->power_max); + + __dtpm_rebalance_weight(child); + } +} + +static void __dtpm_sub_power(struct dtpm *dtpm) +{ + struct dtpm *parent = dtpm->parent; + + while (parent) { + parent->power_min -= dtpm->power_min; + parent->power_max -= dtpm->power_max; + parent->power_limit -= dtpm->power_limit; + parent = parent->parent; + } + + __dtpm_rebalance_weight(root); +} + +static void __dtpm_add_power(struct dtpm *dtpm) +{ + struct dtpm *parent = dtpm->parent; + + while (parent) { + parent->power_min += dtpm->power_min; + parent->power_max += dtpm->power_max; + parent->power_limit += dtpm->power_limit; + parent = parent->parent; + } + + __dtpm_rebalance_weight(root); +} + +/** + * dtpm_update_power - Update the power on the dtpm + * @dtpm: a pointer to a dtpm structure to update + * @power_min: a u64 representing the new power_min value + * @power_max: a u64 representing the new power_max value + * + * Function to update the power values of the dtpm node specified in + * parameter. These new values will be propagated to the tree. + * + * Return: zero on success, -EINVAL if the values are inconsistent + */ +int dtpm_update_power(struct dtpm *dtpm, u64 power_min, u64 power_max) +{ + int ret = 0; + + mutex_lock(&dtpm_lock); + + if (power_min == dtpm->power_min && power_max == dtpm->power_max) + goto unlock; + + if (power_max < power_min) { + ret = -EINVAL; + goto unlock; + } + + __dtpm_sub_power(dtpm); + + dtpm->power_min = power_min; + dtpm->power_max = power_max; + if (!test_bit(DTPM_POWER_LIMIT_FLAG, &dtpm->flags)) + dtpm->power_limit = power_max; + + __dtpm_add_power(dtpm); + +unlock: + mutex_unlock(&dtpm_lock); + + return ret; +} + +/** + * dtpm_release_zone - Cleanup when the node is released + * @pcz: a pointer to a powercap_zone structure + * + * Do some housecleaning and update the weight on the tree. The + * release will be denied if the node has children. This function must + * be called by the specific release callback of the different + * backends. + * + * Return: 0 on success, -EBUSY if there are children + */ +int dtpm_release_zone(struct powercap_zone *pcz) +{ + struct dtpm *dtpm = to_dtpm(pcz); + struct dtpm *parent = dtpm->parent; + + mutex_lock(&dtpm_lock); + + if (!list_empty(&dtpm->children)) { + mutex_unlock(&dtpm_lock); + return -EBUSY; + } + + if (parent) + list_del(&dtpm->sibling); + + __dtpm_sub_power(dtpm); + + mutex_unlock(&dtpm_lock); + + if (dtpm->ops) + dtpm->ops->release(dtpm); + + kfree(dtpm); + + return 0; +} + +static int __get_power_limit_uw(struct dtpm *dtpm, int cid, u64 *power_limit) +{ + *power_limit = dtpm->power_limit; + return 0; +} + +static int get_power_limit_uw(struct powercap_zone *pcz, + int cid, u64 *power_limit) +{ + struct dtpm *dtpm = to_dtpm(pcz); + int ret; + + mutex_lock(&dtpm_lock); + ret = __get_power_limit_uw(dtpm, cid, power_limit); + mutex_unlock(&dtpm_lock); + + return ret; +} + +/* + * Set the power limit on the nodes, the power limit is distributed + * given the weight of the children. + * + * The dtpm node lock must be held when calling this function. + */ +static int __set_power_limit_uw(struct dtpm *dtpm, int cid, u64 power_limit) +{ + struct dtpm *child; + int ret = 0; + u64 power; + + /* + * A max power limitation means we remove the power limit, + * otherwise we set a constraint and flag the dtpm node. + */ + if (power_limit == dtpm->power_max) { + clear_bit(DTPM_POWER_LIMIT_FLAG, &dtpm->flags); + } else { + set_bit(DTPM_POWER_LIMIT_FLAG, &dtpm->flags); + } + + pr_debug("Setting power limit for '%s': %llu uW\n", + dtpm->zone.name, power_limit); + + /* + * Only leaves of the dtpm tree has ops to get/set the power + */ + if (dtpm->ops) { + dtpm->power_limit = dtpm->ops->set_power_uw(dtpm, power_limit); + } else { + dtpm->power_limit = 0; + + list_for_each_entry(child, &dtpm->children, sibling) { + + /* + * Integer division rounding will inevitably + * lead to a different min or max value when + * set several times. In order to restore the + * initial value, we force the child's min or + * max power every time if the constraint is + * at the boundaries. + */ + if (power_limit == dtpm->power_max) { + power = child->power_max; + } else if (power_limit == dtpm->power_min) { + power = child->power_min; + } else { + power = DIV_ROUND_CLOSEST_ULL( + power_limit * child->weight, 1024); + } + + pr_debug("Setting power limit for '%s': %llu uW\n", + child->zone.name, power); + + ret = __set_power_limit_uw(child, cid, power); + if (!ret) + ret = __get_power_limit_uw(child, cid, &power); + + if (ret) + break; + + dtpm->power_limit += power; + } + } + + return ret; +} + +static int set_power_limit_uw(struct powercap_zone *pcz, + int cid, u64 power_limit) +{ + struct dtpm *dtpm = to_dtpm(pcz); + int ret; + + mutex_lock(&dtpm_lock); + + /* + * Don't allow values outside of the power range previously + * set when initializing the power numbers. + */ + power_limit = clamp_val(power_limit, dtpm->power_min, dtpm->power_max); + + ret = __set_power_limit_uw(dtpm, cid, power_limit); + + pr_debug("%s: power limit: %llu uW, power max: %llu uW\n", + dtpm->zone.name, dtpm->power_limit, dtpm->power_max); + + mutex_unlock(&dtpm_lock); + + return ret; +} + +static const char *get_constraint_name(struct powercap_zone *pcz, int cid) +{ + return constraint_name[cid]; +} + +static int get_max_power_uw(struct powercap_zone *pcz, int id, u64 *max_power) +{ + struct dtpm *dtpm = to_dtpm(pcz); + + mutex_lock(&dtpm_lock); + *max_power = dtpm->power_max; + mutex_unlock(&dtpm_lock); + + return 0; +} + +static struct powercap_zone_constraint_ops constraint_ops = { + .set_power_limit_uw = set_power_limit_uw, + .get_power_limit_uw = get_power_limit_uw, + .set_time_window_us = set_time_window_us, + .get_time_window_us = get_time_window_us, + .get_max_power_uw = get_max_power_uw, + .get_name = get_constraint_name, +}; + +static struct powercap_zone_ops zone_ops = { + .get_max_power_range_uw = get_max_power_range_uw, + .get_power_uw = get_power_uw, + .release = dtpm_release_zone, +}; + +/** + * dtpm_alloc - Allocate and initialize a dtpm struct + * @name: a string specifying the name of the node + * + * Return: a struct dtpm pointer, NULL in case of error + */ +struct dtpm *dtpm_alloc(struct dtpm_ops *ops) +{ + struct dtpm *dtpm; + + dtpm = kzalloc(sizeof(*dtpm), GFP_KERNEL); + if (dtpm) { + INIT_LIST_HEAD(&dtpm->children); + INIT_LIST_HEAD(&dtpm->sibling); + dtpm->weight = 1024; + dtpm->ops = ops; + } + + return dtpm; +} + +/** + * dtpm_unregister - Unregister a dtpm node from the hierarchy tree + * @dtpm: a pointer to a dtpm structure corresponding to the node to be removed + * + * Call the underlying powercap unregister function. That will call + * the release callback of the powercap zone. + */ +void dtpm_unregister(struct dtpm *dtpm) +{ + powercap_unregister_zone(pct, &dtpm->zone); + + pr_info("Unregistered dtpm node '%s'\n", dtpm->zone.name); +} + +/** + * dtpm_register - Register a dtpm node in the hierarchy tree + * @name: a string specifying the name of the node + * @dtpm: a pointer to a dtpm structure corresponding to the new node + * @parent: a pointer to a dtpm structure corresponding to the parent node + * + * Create a dtpm node in the tree. If no parent is specified, the node + * is the root node of the hierarchy. If the root node already exists, + * then the registration will fail. The powercap controller must be + * initialized before calling this function. + * + * The dtpm structure must be initialized with the power numbers + * before calling this function. + * + * Return: zero on success, a negative value in case of error: + * -EAGAIN: the function is called before the framework is initialized. + * -EBUSY: the root node is already inserted + * -EINVAL: * there is no root node yet and @parent is specified + * * no all ops are defined + * * parent have ops which are reserved for leaves + * Other negative values are reported back from the powercap framework + */ +int dtpm_register(const char *name, struct dtpm *dtpm, struct dtpm *parent) +{ + struct powercap_zone *pcz; + + if (!pct) + return -EAGAIN; + + if (root && !parent) + return -EBUSY; + + if (!root && parent) + return -EINVAL; + + if (parent && parent->ops) + return -EINVAL; + + if (!dtpm) + return -EINVAL; + + if (dtpm->ops && !(dtpm->ops->set_power_uw && + dtpm->ops->get_power_uw && + dtpm->ops->release)) + return -EINVAL; + + pcz = powercap_register_zone(&dtpm->zone, pct, name, + parent ? &parent->zone : NULL, + &zone_ops, MAX_DTPM_CONSTRAINTS, + &constraint_ops); + if (IS_ERR(pcz)) + return PTR_ERR(pcz); + + mutex_lock(&dtpm_lock); + + if (parent) { + list_add_tail(&dtpm->sibling, &parent->children); + dtpm->parent = parent; + } else { + root = dtpm; + } + + __dtpm_add_power(dtpm); + + pr_info("Registered dtpm node '%s' / %llu-%llu uW, \n", + dtpm->zone.name, dtpm->power_min, dtpm->power_max); + + mutex_unlock(&dtpm_lock); + + return 0; +} + +static int __init dtpm_init(void) +{ + struct dtpm_descr **dtpm_descr; + + pct = powercap_register_control_type(NULL, "dtpm", NULL); + if (IS_ERR(pct)) { + pr_err("Failed to register control type\n"); + return PTR_ERR(pct); + } + + for_each_dtpm_table(dtpm_descr) + (*dtpm_descr)->init(*dtpm_descr); + + return 0; +} +late_initcall(dtpm_init); diff --git a/drivers/powercap/dtpm_cpu.c b/drivers/powercap/dtpm_cpu.c new file mode 100644 index 000000000000..51c366938acd --- /dev/null +++ b/drivers/powercap/dtpm_cpu.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2020 Linaro Limited + * + * Author: Daniel Lezcano + * + * The DTPM CPU is based on the energy model. It hooks the CPU in the + * DTPM tree which in turns update the power number by propagating the + * power number from the CPU energy model information to the parents. + * + * The association between the power and the performance state, allows + * to set the power of the CPU at the OPP granularity. + * + * The CPU hotplug is supported and the power numbers will be updated + * if a CPU is hot plugged / unplugged. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +static struct dtpm *__parent; + +static DEFINE_PER_CPU(struct dtpm *, dtpm_per_cpu); + +struct dtpm_cpu { + struct freq_qos_request qos_req; + int cpu; +}; + +/* + * When a new CPU is inserted at hotplug or boot time, add the power + * contribution and update the dtpm tree. + */ +static int power_add(struct dtpm *dtpm, struct em_perf_domain *em) +{ + u64 power_min, power_max; + + power_min = em->table[0].power; + power_min *= MICROWATT_PER_MILLIWATT; + power_min += dtpm->power_min; + + power_max = em->table[em->nr_perf_states - 1].power; + power_max *= MICROWATT_PER_MILLIWATT; + power_max += dtpm->power_max; + + return dtpm_update_power(dtpm, power_min, power_max); +} + +/* + * When a CPU is unplugged, remove its power contribution from the + * dtpm tree. + */ +static int power_sub(struct dtpm *dtpm, struct em_perf_domain *em) +{ + u64 power_min, power_max; + + power_min = em->table[0].power; + power_min *= MICROWATT_PER_MILLIWATT; + power_min = dtpm->power_min - power_min; + + power_max = em->table[em->nr_perf_states - 1].power; + power_max *= MICROWATT_PER_MILLIWATT; + power_max = dtpm->power_max - power_max; + + return dtpm_update_power(dtpm, power_min, power_max); +} + +static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit) +{ + struct dtpm_cpu *dtpm_cpu = dtpm->private; + struct em_perf_domain *pd; + struct cpumask cpus; + unsigned long freq; + u64 power; + int i, nr_cpus; + + pd = em_cpu_get(dtpm_cpu->cpu); + + cpumask_and(&cpus, cpu_online_mask, to_cpumask(pd->cpus)); + + nr_cpus = cpumask_weight(&cpus); + + for (i = 0; i < pd->nr_perf_states; i++) { + + power = pd->table[i].power * MICROWATT_PER_MILLIWATT * nr_cpus; + + if (power > power_limit) + break; + } + + freq = pd->table[i - 1].frequency; + + freq_qos_update_request(&dtpm_cpu->qos_req, freq); + + power_limit = pd->table[i - 1].power * + MICROWATT_PER_MILLIWATT * nr_cpus; + + return power_limit; +} + +static u64 get_pd_power_uw(struct dtpm *dtpm) +{ + struct dtpm_cpu *dtpm_cpu = dtpm->private; + struct em_perf_domain *pd; + struct cpumask cpus; + unsigned long freq; + int i, nr_cpus; + + pd = em_cpu_get(dtpm_cpu->cpu); + freq = cpufreq_quick_get(dtpm_cpu->cpu); + cpumask_and(&cpus, cpu_online_mask, to_cpumask(pd->cpus)); + nr_cpus = cpumask_weight(&cpus); + + for (i = 0; i < pd->nr_perf_states; i++) { + + if (pd->table[i].frequency < freq) + continue; + + return pd->table[i].power * + MICROWATT_PER_MILLIWATT * nr_cpus; + } + + return 0; +} + +static void pd_release(struct dtpm *dtpm) +{ + struct dtpm_cpu *dtpm_cpu = dtpm->private; + + if (freq_qos_request_active(&dtpm_cpu->qos_req)) + freq_qos_remove_request(&dtpm_cpu->qos_req); + + kfree(dtpm_cpu); +} + +static struct dtpm_ops dtpm_ops = { + .set_power_uw = set_pd_power_limit, + .get_power_uw = get_pd_power_uw, + .release = pd_release, +}; + +static int cpuhp_dtpm_cpu_offline(unsigned int cpu) +{ + struct cpufreq_policy *policy; + struct em_perf_domain *pd; + struct dtpm *dtpm; + + policy = cpufreq_cpu_get(cpu); + + if (!policy) + return 0; + + pd = em_cpu_get(cpu); + if (!pd) + return -EINVAL; + + dtpm = per_cpu(dtpm_per_cpu, cpu); + + power_sub(dtpm, pd); + + if (cpumask_weight(policy->cpus) != 1) + return 0; + + for_each_cpu(cpu, policy->related_cpus) + per_cpu(dtpm_per_cpu, cpu) = NULL; + + dtpm_unregister(dtpm); + + return 0; +} + +static int cpuhp_dtpm_cpu_online(unsigned int cpu) +{ + struct dtpm *dtpm; + struct dtpm_cpu *dtpm_cpu; + struct cpufreq_policy *policy; + struct em_perf_domain *pd; + char name[CPUFREQ_NAME_LEN]; + int ret = -ENOMEM; + + policy = cpufreq_cpu_get(cpu); + + if (!policy) + return 0; + + pd = em_cpu_get(cpu); + if (!pd) + return -EINVAL; + + dtpm = per_cpu(dtpm_per_cpu, cpu); + if (dtpm) + return power_add(dtpm, pd); + + dtpm = dtpm_alloc(&dtpm_ops); + if (!dtpm) + return -EINVAL; + + dtpm_cpu = kzalloc(sizeof(*dtpm_cpu), GFP_KERNEL); + if (!dtpm_cpu) + goto out_kfree_dtpm; + + dtpm->private = dtpm_cpu; + dtpm_cpu->cpu = cpu; + + for_each_cpu(cpu, policy->related_cpus) + per_cpu(dtpm_per_cpu, cpu) = dtpm; + + sprintf(name, "cpu%d", dtpm_cpu->cpu); + + ret = dtpm_register(name, dtpm, __parent); + if (ret) + goto out_kfree_dtpm_cpu; + + ret = power_add(dtpm, pd); + if (ret) + goto out_dtpm_unregister; + + ret = freq_qos_add_request(&policy->constraints, + &dtpm_cpu->qos_req, FREQ_QOS_MAX, + pd->table[pd->nr_perf_states - 1].frequency); + if (ret) + goto out_power_sub; + + return 0; + +out_power_sub: + power_sub(dtpm, pd); + +out_dtpm_unregister: + dtpm_unregister(dtpm); + dtpm_cpu = NULL; + dtpm = NULL; + +out_kfree_dtpm_cpu: + for_each_cpu(cpu, policy->related_cpus) + per_cpu(dtpm_per_cpu, cpu) = NULL; + kfree(dtpm_cpu); + +out_kfree_dtpm: + kfree(dtpm); + return ret; +} + +int dtpm_register_cpu(struct dtpm *parent) +{ + __parent = parent; + + return cpuhp_setup_state(CPUHP_AP_DTPM_CPU_ONLINE, + "dtpm_cpu:online", + cpuhp_dtpm_cpu_online, + cpuhp_dtpm_cpu_offline); +} diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h index b59136c4125b..c76f7c9e6b13 100644 --- a/drivers/scsi/sd.h +++ b/drivers/scsi/sd.h @@ -125,6 +125,9 @@ struct scsi_disk { unsigned urswrz : 1; unsigned security : 1; unsigned ignore_medium_access_errors : 1; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; #define to_scsi_disk(obj) container_of(obj,struct scsi_disk,dev) diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 27748aeb87cf..ea4e98a7363c 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -4809,6 +4809,8 @@ static int ufshcd_slave_configure(struct scsi_device *sdev) ufshcd_crypto_setup_rq_keyslot_manager(hba, q); + trace_android_vh_ufs_update_sdev(sdev); + return 0; } diff --git a/drivers/thermal/thermal_netlink.c b/drivers/thermal/thermal_netlink.c index 0bfe30233afe..c82dba5ee231 100644 --- a/drivers/thermal/thermal_netlink.c +++ b/drivers/thermal/thermal_netlink.c @@ -232,7 +232,7 @@ static int thermal_genl_send_event(enum thermal_genl_event event, void *hdr; int enable_thermal_genl = 1; - trace_android_vh_enable_thermal_genl_check(event, &enable_thermal_genl); + trace_android_vh_enable_thermal_genl_check(event, p->tz_id, &enable_thermal_genl); if (!enable_thermal_genl) return 0; diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 5e66518d9e7f..5b580b2979a1 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -783,8 +783,6 @@ static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) trace_dwc3_gadget_ep_disable(dep); - dwc3_remove_requests(dwc, dep); - /* make sure HW endpoint isn't stalled */ if (dep->flags & DWC3_EP_STALL) __dwc3_gadget_ep_set_halt(dep, 0, false); @@ -803,6 +801,8 @@ static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) dep->endpoint.desc = NULL; } + dwc3_remove_requests(dwc, dep); + return 0; } @@ -1617,7 +1617,7 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) { struct dwc3 *dwc = dep->dwc; - if (!dep->endpoint.desc || !dwc->pullups_connected) { + if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n", dep->name); return -ESHUTDOWN; @@ -2247,6 +2247,7 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) if (!is_on) { u32 count; + dwc->connected = false; /* * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a * Section 4.1.8 Table 4-7, it states that for a device-initiated @@ -2271,7 +2272,6 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) dwc->ev_buf->lpos = (dwc->ev_buf->lpos + count) % dwc->ev_buf->length; } - dwc->connected = false; } else { __dwc3_gadget_start(dwc); } @@ -3344,8 +3344,6 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) { u32 reg; - dwc->connected = true; - /* * WORKAROUND: DWC3 revisions <1.88a have an issue which * would cause a missing Disconnect Event if there's a @@ -3385,6 +3383,7 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) * transfers." */ dwc3_stop_active_transfers(dwc); + dwc->connected = true; reg = dwc3_readl(dwc->regs, DWC3_DCTL); reg &= ~DWC3_DCTL_TSTCTRL_MASK; diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index 9f8daa0239cf..dbc1104fe343 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -129,6 +129,8 @@ struct gadget_config_name { struct list_head list; }; +#define USB_MAX_STRING_WITH_NULL_LEN (USB_MAX_STRING_LEN+1) + static int usb_string_copy(const char *s, char **s_copy) { int ret; @@ -138,12 +140,16 @@ static int usb_string_copy(const char *s, char **s_copy) if (ret > USB_MAX_STRING_LEN) return -EOVERFLOW; - str = kstrdup(s, GFP_KERNEL); - if (!str) - return -ENOMEM; + if (copy) { + str = copy; + } else { + str = kmalloc(USB_MAX_STRING_WITH_NULL_LEN, GFP_KERNEL); + if (!str) + return -ENOMEM; + } + strcpy(str, s); if (str[ret - 1] == '\n') str[ret - 1] = '\0'; - kfree(copy); *s_copy = str; return 0; } diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index a8b890a0dd73..9d72017f5b93 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -17,6 +17,7 @@ #include #include #include +#include /* Code sharing between pci-quirks and xhci hcd */ #include "xhci-ext-caps.h" @@ -809,6 +810,9 @@ struct xhci_command { struct completion *completion; union xhci_trb *command_trb; struct list_head cmd_list; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /* drop context bitmasks */ @@ -1533,6 +1537,8 @@ struct xhci_segment { void *bounce_buf; unsigned int bounce_offs; unsigned int bounce_len; + + ANDROID_KABI_RESERVE(1); }; enum xhci_cancelled_td_status { @@ -1622,6 +1628,9 @@ struct xhci_ring { enum xhci_ring_type type; bool last_td_was_short; struct radix_tree_root *trb_address_map; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct xhci_erst_entry { @@ -1639,6 +1648,8 @@ struct xhci_erst { dma_addr_t erst_dma_addr; /* Num entries the ERST can contain */ unsigned int erst_size; + + ANDROID_KABI_RESERVE(1); }; struct xhci_scratchpad { @@ -1916,6 +1927,12 @@ struct xhci_hcd { struct list_head regset_list; void *dbc; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); + /* platform-specific data -- must come last */ unsigned long priv[] __aligned(sizeof(s64)); }; diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index 027afd7dfdce..372a5fb0baf4 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "tcpci.h" @@ -24,14 +25,10 @@ #define AUTO_DISCHARGE_PD_HEADROOM_MV 850 #define AUTO_DISCHARGE_PPS_HEADROOM_MV 1250 -#define tcpc_presenting_cc1_rd(reg) \ +#define tcpc_presenting_rd(reg, cc) \ (!(TCPC_ROLE_CTRL_DRP & (reg)) && \ - (((reg) & (TCPC_ROLE_CTRL_CC1_MASK << TCPC_ROLE_CTRL_CC1_SHIFT)) == \ - (TCPC_ROLE_CTRL_CC_RD << TCPC_ROLE_CTRL_CC1_SHIFT))) -#define tcpc_presenting_cc2_rd(reg) \ - (!(TCPC_ROLE_CTRL_DRP & (reg)) && \ - (((reg) & (TCPC_ROLE_CTRL_CC2_MASK << TCPC_ROLE_CTRL_CC2_SHIFT)) == \ - (TCPC_ROLE_CTRL_CC_RD << TCPC_ROLE_CTRL_CC2_SHIFT))) + (((reg) & (TCPC_ROLE_CTRL_## cc ##_MASK << TCPC_ROLE_CTRL_## cc ##_SHIFT)) == \ + (TCPC_ROLE_CTRL_CC_RD << TCPC_ROLE_CTRL_## cc ##_SHIFT))) struct tcpci { struct device *dev; @@ -126,14 +123,17 @@ static int tcpci_start_toggling(struct tcpc_dev *tcpc, int ret; struct tcpci *tcpci = tcpc_to_tcpci(tcpc); unsigned int reg = TCPC_ROLE_CTRL_DRP; + int override_toggling = 0; if (port_type != TYPEC_PORT_DRP) return -EOPNOTSUPP; /* Handle vendor drp toggling */ if (tcpci->data->start_drp_toggling) { + trace_android_vh_typec_tcpci_override_toggling(tcpci, tcpci->data, + &override_toggling); ret = tcpci->data->start_drp_toggling(tcpci, tcpci->data, cc); - if (ret < 0) + if (ret < 0 || override_toggling) return ret; } @@ -201,11 +201,11 @@ static int tcpci_get_cc(struct tcpc_dev *tcpc, *cc1 = tcpci_to_typec_cc((reg >> TCPC_CC_STATUS_CC1_SHIFT) & TCPC_CC_STATUS_CC1_MASK, reg & TCPC_CC_STATUS_TERM || - tcpc_presenting_cc1_rd(role_control)); + tcpc_presenting_rd(role_control, CC1)); *cc2 = tcpci_to_typec_cc((reg >> TCPC_CC_STATUS_CC2_SHIFT) & TCPC_CC_STATUS_CC2_MASK, reg & TCPC_CC_STATUS_TERM || - tcpc_presenting_cc2_rd(role_control)); + tcpc_presenting_rd(role_control, CC2)); return 0; } @@ -417,7 +417,11 @@ static int tcpci_get_vbus(struct tcpc_dev *tcpc) { struct tcpci *tcpci = tcpc_to_tcpci(tcpc); unsigned int reg; - int ret; + int ret, vbus, bypass = 0; + + trace_android_vh_typec_tcpci_get_vbus(tcpci, tcpci->data, &vbus, &bypass); + if (bypass) + return vbus; ret = regmap_read(tcpci->regmap, TCPC_POWER_STATUS, ®); if (ret < 0) @@ -426,6 +430,15 @@ static int tcpci_get_vbus(struct tcpc_dev *tcpc) return !!(reg & TCPC_POWER_STATUS_VBUS_PRES); } +static int tcpci_check_contaminant(struct tcpc_dev *tcpc) +{ + struct tcpci *tcpci = tcpc_to_tcpci(tcpc); + int ret = 0; + + trace_android_vh_typec_tcpci_check_contaminant(tcpci, tcpci->data, &ret); + return ret; +} + static bool tcpci_is_vbus_vsafe0v(struct tcpc_dev *tcpc) { struct tcpci *tcpci = tcpc_to_tcpci(tcpc); @@ -744,6 +757,7 @@ struct tcpci *tcpci_register_port(struct device *dev, struct tcpci_data *data) tcpci->tcpc.enable_frs = tcpci_enable_frs; tcpci->tcpc.frs_sourcing_vbus = tcpci_frs_sourcing_vbus; tcpci->tcpc.set_partner_usb_comm_capable = tcpci_set_partner_usb_comm_capable; + tcpci->tcpc.check_contaminant = tcpci_check_contaminant; if (tcpci->data->auto_discharge_disconnect) { tcpci->tcpc.enable_auto_vbus_discharge = tcpci_enable_auto_vbus_discharge; diff --git a/drivers/usb/typec/tcpm/tcpci.h b/drivers/usb/typec/tcpm/tcpci.h index 57b6e24e0a0c..2be7a77d400e 100644 --- a/drivers/usb/typec/tcpm/tcpci.h +++ b/drivers/usb/typec/tcpm/tcpci.h @@ -47,7 +47,10 @@ #define TCPC_TCPC_CTRL 0x19 #define TCPC_TCPC_CTRL_ORIENTATION BIT(0) +#define PLUG_ORNT_CC1 0 +#define PLUG_ORNT_CC2 1 #define TCPC_TCPC_CTRL_BIST_TM BIT(1) +#define TCPC_TCPC_CTRL_EN_LK4CONN_ALRT BIT(6) #define TCPC_EXTENDED_STATUS 0x20 #define TCPC_EXTENDED_STATUS_VSAFE0V BIT(0) @@ -74,21 +77,28 @@ #define TCPC_POWER_CTRL_VCONN_ENABLE BIT(0) #define TCPC_POWER_CTRL_BLEED_DISCHARGE BIT(3) #define TCPC_POWER_CTRL_AUTO_DISCHARGE BIT(4) +#define TCPC_DIS_VOLT_ALRM BIT(5) +#define TCPC_POWER_CTRL_VBUS_VOLT_MON BIT(6) #define TCPC_FAST_ROLE_SWAP_EN BIT(7) #define TCPC_CC_STATUS 0x1d #define TCPC_CC_STATUS_TOGGLING BIT(5) #define TCPC_CC_STATUS_TERM BIT(4) +#define TCPC_CC_STATUS_TERM_RP 0 +#define TCPC_CC_STATUS_TERM_RD 1 +#define TCPC_CC_STATE_SRC_OPEN 0 #define TCPC_CC_STATUS_CC2_SHIFT 2 #define TCPC_CC_STATUS_CC2_MASK 0x3 #define TCPC_CC_STATUS_CC1_SHIFT 0 #define TCPC_CC_STATUS_CC1_MASK 0x3 #define TCPC_POWER_STATUS 0x1e +#define TCPC_POWER_STATUS_DBG_ACC_CON BIT(7) #define TCPC_POWER_STATUS_UNINIT BIT(6) #define TCPC_POWER_STATUS_SOURCING_VBUS BIT(4) #define TCPC_POWER_STATUS_VBUS_DET BIT(3) #define TCPC_POWER_STATUS_VBUS_PRES BIT(2) +#define TCPC_POWER_STATUS_SINKING_VBUS BIT(0) #define TCPC_FAULT_STATUS 0x1f @@ -121,6 +131,10 @@ #define TCPC_RX_DETECT 0x2f #define TCPC_RX_DETECT_HARD_RESET BIT(5) #define TCPC_RX_DETECT_SOP BIT(0) +#define TCPC_RX_DETECT_SOP1 BIT(1) +#define TCPC_RX_DETECT_SOP2 BIT(2) +#define TCPC_RX_DETECT_DBG1 BIT(3) +#define TCPC_RX_DETECT_DBG2 BIT(4) #define TCPC_RX_BYTE_CNT 0x30 #define TCPC_RX_BUF_FRAME_TYPE 0x31 @@ -139,6 +153,8 @@ #define TCPC_TX_DATA 0x54 /* through 0x6f */ #define TCPC_VBUS_VOLTAGE 0x70 +#define TCPC_VBUS_VOLTAGE_MASK 0x3ff +#define TCPC_VBUS_VOLTAGE_LSB_MV 25 #define TCPC_VBUS_SINK_DISCONNECT_THRESH 0x72 #define TCPC_VBUS_SINK_DISCONNECT_THRESH_LSB_MV 25 #define TCPC_VBUS_SINK_DISCONNECT_THRESH_MAX 0x3ff diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 283c9694f100..49caf4cad123 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -962,6 +962,7 @@ static int tcpm_set_current_limit(struct tcpm_port *port, u32 max_ma, u32 mv) port->supply_voltage = mv; port->current_limit = max_ma; + power_supply_changed(port->psy); if (port->tcpc->set_current_limit) ret = port->tcpc->set_current_limit(port->tcpc, max_ma, mv); @@ -2948,6 +2949,7 @@ static int tcpm_pd_select_pdo(struct tcpm_port *port, int *sink_pdo, port->pps_data.supported = false; port->usb_type = POWER_SUPPLY_USB_TYPE_PD; + power_supply_changed(port->psy); /* * Select the source PDO providing the most power which has a @@ -2972,6 +2974,7 @@ static int tcpm_pd_select_pdo(struct tcpm_port *port, int *sink_pdo, port->pps_data.supported = true; port->usb_type = POWER_SUPPLY_USB_TYPE_PD_PPS; + power_supply_changed(port->psy); } continue; default: @@ -3129,6 +3132,7 @@ static unsigned int tcpm_pd_select_pps_apdo(struct tcpm_port *port) port->pps_data.out_volt)); port->pps_data.op_curr = min(port->pps_data.max_curr, port->pps_data.op_curr); + power_supply_changed(port->psy); } return src_pdo; @@ -3364,6 +3368,7 @@ static int tcpm_set_charge(struct tcpm_port *port, bool charge) return ret; } port->vbus_charge = charge; + power_supply_changed(port->psy); return 0; } @@ -3548,6 +3553,7 @@ static void tcpm_reset_port(struct tcpm_port *port) port->try_src_count = 0; port->try_snk_count = 0; port->usb_type = POWER_SUPPLY_USB_TYPE_C; + power_supply_changed(port->psy); port->nr_sink_caps = 0; port->sink_cap_done = false; if (port->tcpc->enable_frs) @@ -3683,8 +3689,8 @@ EXPORT_SYMBOL_GPL(tcpm_is_toggling); static void tcpm_check_send_discover(struct tcpm_port *port) { - if (port->data_role == TYPEC_HOST && port->send_discover && - port->pd_capable) + if ((port->data_role == TYPEC_HOST || port->negotiated_rev > PD_REV20) && + port->send_discover && port->pd_capable) tcpm_send_vdm(port, USB_SID_PD, CMD_DISCOVER_IDENT, NULL, 0); port->send_discover = false; } @@ -5274,7 +5280,7 @@ static void tcpm_enable_frs_work(struct kthread_work *work) goto unlock; /* Send when the state machine is idle */ - if (port->state != SNK_READY || port->vdm_state != VDM_STATE_DONE || port->send_discover) + if (port->state != SNK_READY || port->vdm_sm_running || port->send_discover) goto resched; port->upcoming_state = GET_SINK_CAP; @@ -6003,7 +6009,6 @@ static int tcpm_psy_get_prop(struct power_supply *psy, ret = -EINVAL; break; } - return ret; } @@ -6055,6 +6060,7 @@ static int tcpm_psy_set_prop(struct power_supply *psy, ret = -EINVAL; break; } + power_supply_changed(port->psy); return ret; } @@ -6208,6 +6214,7 @@ struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc) err = devm_tcpm_psy_register(port); if (err) goto out_role_sw_put; + power_supply_changed(port->psy); port->typec_port = typec_register_port(port->dev, &port->typec_caps); if (IS_ERR(port->typec_port)) { diff --git a/drivers/usb/typec/tps6598x.c b/drivers/usb/typec/tps6598x.c index dbd10b3f2d65..30bfc314b743 100644 --- a/drivers/usb/typec/tps6598x.c +++ b/drivers/usb/typec/tps6598x.c @@ -62,7 +62,6 @@ enum { struct tps6598x_rx_identity_reg { u8 status; struct usb_pd_identity identity; - u32 vdo[3]; } __packed; /* Standard Task return codes */ diff --git a/fs/buffer.c b/fs/buffer.c index 85fc97361b45..0afef9982b82 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1265,6 +1265,14 @@ static void bh_lru_install(struct buffer_head *bh) int i; check_irqs_on(); + /* + * buffer_head in bh_lru could increase refcount of the page + * until it will be invalidated. It causes page migraion failure. + * Skip putting upcoming bh into bh_lru until migration is done. + */ + if (lru_cache_disabled()) + return; + bh_lru_lock(); b = this_cpu_ptr(&bh_lrus); @@ -1410,7 +1418,7 @@ EXPORT_SYMBOL(__bread_gfp); * This doesn't race because it runs in each cpu either in irq * or with preempt disabled. */ -static void invalidate_bh_lru(void *arg) +void invalidate_bh_lru(void *arg) { struct bh_lru *b = &get_cpu_var(bh_lrus); int i; @@ -1422,7 +1430,7 @@ static void invalidate_bh_lru(void *arg) put_cpu_var(bh_lrus); } -static bool has_bh_in_lru(int cpu, void *dummy) +bool has_bh_in_lru(int cpu, void *dummy) { struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu); int i; diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 77fa342de38f..3c9d797dbdd6 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -1353,6 +1353,7 @@ unlock_continue: if (fio.compr_blocks) f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false); f2fs_i_compr_blocks_update(inode, cc->nr_cpages, true); + add_compr_block_stat(inode, cc->nr_cpages); set_inode_flag(cc->inode, FI_APPEND_WRITE); if (cc->cluster_idx == 0) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 0aef7c21931a..74158ce55319 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1909,7 +1909,6 @@ next: } if (size) { - flags |= FIEMAP_EXTENT_MERGED; if (IS_ENCRYPTED(inode)) flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 52b7c7d0a351..7e6977173d28 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1623,6 +1623,11 @@ struct f2fs_sb_info { #ifdef CONFIG_F2FS_FS_COMPRESSION struct kmem_cache *page_array_slab; /* page array entry */ unsigned int page_array_slab_size; /* default page array slab size */ + + /* For runtime compression statistics */ + u64 compr_written_block; + u64 compr_saved_block; + u32 compr_new_inode; #endif }; @@ -3954,6 +3959,18 @@ int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi); void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi); int __init f2fs_init_compress_cache(void); void f2fs_destroy_compress_cache(void); +#define inc_compr_inode_stat(inode) \ + do { \ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \ + sbi->compr_new_inode++; \ + } while (0) +#define add_compr_block_stat(inode, blocks) \ + do { \ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \ + int diff = F2FS_I(inode)->i_cluster_size - blocks; \ + sbi->compr_written_block += blocks; \ + sbi->compr_saved_block += diff; \ + } while (0) #else static inline bool f2fs_is_compressed_page(struct page *page) { return false; } static inline bool f2fs_is_compress_backend_ready(struct inode *inode) @@ -3982,6 +3999,7 @@ static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { } static inline int __init f2fs_init_compress_cache(void) { return 0; } static inline void f2fs_destroy_compress_cache(void) { } +#define inc_compr_inode_stat(inode) do { } while (0) #endif static inline void set_compress_context(struct inode *inode) @@ -4005,6 +4023,7 @@ static inline void set_compress_context(struct inode *inode) F2FS_I(inode)->i_flags |= F2FS_COMPR_FL; set_inode_flag(inode, FI_COMPRESSED_FILE); stat_inc_compr_inode(inode); + inc_compr_inode_stat(inode); f2fs_mark_inode_dirty_sync(inode, true); } diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index e38a7f6921dd..a166eebbe019 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "f2fs.h" #include "segment.h" @@ -282,6 +283,17 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a, return len; } +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (!strcmp(a->attr.name, "compr_written_block")) + return sysfs_emit(buf, "%llu\n", sbi->compr_written_block); + + if (!strcmp(a->attr.name, "compr_saved_block")) + return sysfs_emit(buf, "%llu\n", sbi->compr_saved_block); + + if (!strcmp(a->attr.name, "compr_new_inode")) + return sysfs_emit(buf, "%u\n", sbi->compr_new_inode); +#endif + ui = (unsigned int *)(ptr + a->offset); return sprintf(buf, "%u\n", *ui); @@ -458,6 +470,24 @@ out: return count; } +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (!strcmp(a->attr.name, "compr_written_block") || + !strcmp(a->attr.name, "compr_saved_block")) { + if (t != 0) + return -EINVAL; + sbi->compr_written_block = 0; + sbi->compr_saved_block = 0; + return count; + } + + if (!strcmp(a->attr.name, "compr_new_inode")) { + if (t != 0) + return -EINVAL; + sbi->compr_new_inode = 0; + return count; + } +#endif + *ui = (unsigned int)t; return count; @@ -668,6 +698,9 @@ F2FS_FEATURE_RO_ATTR(sb_checksum, FEAT_SB_CHECKSUM); F2FS_FEATURE_RO_ATTR(casefold, FEAT_CASEFOLD); #ifdef CONFIG_F2FS_FS_COMPRESSION F2FS_FEATURE_RO_ATTR(compression, FEAT_COMPRESSION); +F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, compr_written_block, compr_written_block); +F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, compr_saved_block, compr_saved_block); +F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, compr_new_inode, compr_new_inode); #endif #define ATTR_LIST(name) (&f2fs_attr_##name.attr) @@ -730,6 +763,11 @@ static struct attribute *f2fs_attrs[] = { ATTR_LIST(moved_blocks_foreground), ATTR_LIST(moved_blocks_background), ATTR_LIST(avg_vblocks), +#endif +#ifdef CONFIG_F2FS_FS_COMPRESSION + ATTR_LIST(compr_written_block), + ATTR_LIST(compr_saved_block), + ATTR_LIST(compr_new_inode), #endif NULL, }; diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index 887a5532e449..208c9c4a83d7 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -18,7 +18,7 @@ #endif #include #include "internal.h" - +#include void __attribute__((weak)) arch_report_meminfo(struct seq_file *m) { } @@ -145,6 +145,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) show_val_kb(m, "CmaFree: ", global_zone_page_state(NR_FREE_CMA_PAGES)); #endif + trace_android_vh_meminfo_proc_show(m); hugetlb_report_meminfo(m); diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index f247e4ad8cce..ce62b3840e3f 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -319,6 +319,16 @@ #define THERMAL_TABLE(name) #endif +#ifdef CONFIG_DTPM +#define DTPM_TABLE() \ + . = ALIGN(8); \ + __dtpm_table = .; \ + KEEP(*(__dtpm_table)) \ + __dtpm_table_end = .; +#else +#define DTPM_TABLE() +#endif + #define KERNEL_DTB() \ STRUCT_ALIGN(); \ __dtb_start = .; \ @@ -753,6 +763,7 @@ ACPI_PROBE_TABLE(irqchip) \ ACPI_PROBE_TABLE(timer) \ THERMAL_TABLE(governor) \ + DTPM_TABLE() \ EARLYCON_TABLE() \ LSM_TABLE() \ EARLY_LSM_TABLE() \ @@ -1019,7 +1030,8 @@ * keep any .init_array.* sections. * https://bugs.llvm.org/show_bug.cgi?id=46478 */ -#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) +#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) || \ + defined(CONFIG_CFI_CLANG) # ifdef CONFIG_CONSTRUCTORS # define SANITIZER_DISCARDS \ *(.eh_frame) diff --git a/include/linux/android_kabi.h b/include/linux/android_kabi.h new file mode 100644 index 000000000000..efa52953fe35 --- /dev/null +++ b/include/linux/android_kabi.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * android_kabi.h - Android kernel abi abstraction header + * + * Copyright (C) 2020 Google, Inc. + * + * Heavily influenced by rh_kabi.h which came from the RHEL/CENTOS kernel and + * was: + * Copyright (c) 2014 Don Zickus + * Copyright (c) 2015-2018 Jiri Benc + * Copyright (c) 2015 Sabrina Dubroca, Hannes Frederic Sowa + * Copyright (c) 2016-2018 Prarit Bhargava + * Copyright (c) 2017 Paolo Abeni, Larry Woodman + * + * These macros are to be used to try to help alleviate future kernel abi + * changes that will occur as LTS and other kernel patches are merged into the + * tree during a period in which the kernel abi is wishing to not be disturbed. + * + * There are two times these macros should be used: + * - Before the kernel abi is "frozen" + * Padding can be added to various kernel structures that have in the past + * been known to change over time. That will give "room" in the structure + * that can then be used when fields are added so that the structure size + * will not change. + * + * - After the kernel abi is "frozen" + * If a structure's field is changed to a type that is identical in size to + * the previous type, it can be changed with a union macro + * If a field is added to a structure, the padding fields can be used to add + * the new field in a "safe" way. + */ +#ifndef _ANDROID_KABI_H +#define _ANDROID_KABI_H + +#include + +/* + * Worker macros, don't use these, use the ones without a leading '_' + */ + +#define __ANDROID_KABI_CHECK_SIZE_ALIGN(_orig, _new) \ + union { \ + _Static_assert(sizeof(struct{_new;}) <= sizeof(struct{_orig;}), \ + __FILE__ ":" __stringify(__LINE__) ": " \ + __stringify(_new) \ + " is larger than " \ + __stringify(_orig) ); \ + _Static_assert(__alignof__(struct{_new;}) <= __alignof__(struct{_orig;}), \ + __FILE__ ":" __stringify(__LINE__) ": " \ + __stringify(_orig) \ + " is not aligned the same as " \ + __stringify(_new) ); \ + } + +#ifdef __GENKSYMS__ + +#define _ANDROID_KABI_REPLACE(_orig, _new) _orig + +#else + +#define _ANDROID_KABI_REPLACE(_orig, _new) \ + union { \ + _new; \ + struct { \ + _orig; \ + } __UNIQUE_ID(android_kabi_hide); \ + __ANDROID_KABI_CHECK_SIZE_ALIGN(_orig, _new); \ + } + +#endif /* __GENKSYMS__ */ + +#define _ANDROID_KABI_RESERVE(n) u64 android_kabi_reserved##n + + +/* + * Macros to use _before_ the ABI is frozen + */ + +/* + * ANDROID_KABI_RESERVE + * Reserve some "padding" in a structure for potential future use. + * This normally placed at the end of a structure. + * number: the "number" of the padding variable in the structure. Start with + * 1 and go up. + */ +#define ANDROID_KABI_RESERVE(number) _ANDROID_KABI_RESERVE(number) + + +/* + * Macros to use _after_ the ABI is frozen + */ + +/* + * ANDROID_KABI_USE(number, _new) + * Use a previous padding entry that was defined with ANDROID_KABI_RESERVE + * number: the previous "number" of the padding variable + * _new: the variable to use now instead of the padding variable + */ +#define ANDROID_KABI_USE(number, _new) \ + _ANDROID_KABI_REPLACE(_ANDROID_KABI_RESERVE(number), _new) + +/* + * ANDROID_KABI_USE2(number, _new1, _new2) + * Use a previous padding entry that was defined with ANDROID_KABI_RESERVE for + * two new variables that fit into 64 bits. This is good for when you do not + * want to "burn" a 64bit padding variable for a smaller variable size if not + * needed. + */ +#define ANDROID_KABI_USE2(number, _new1, _new2) \ + _ANDROID_KABI_REPLACE(_ANDROID_KABI_RESERVE(number), struct{ _new1; _new2; }) + + +#endif /* _ANDROID_KABI_H */ diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index e47f86a147a6..a42b751da268 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -9,6 +9,7 @@ #include #include #include +#include struct bio_set; struct bio; @@ -261,6 +262,9 @@ struct bio { struct bio_set *bi_pool; + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + /* * We can inline a number of vecs at the end of the bio, to avoid * double allocations for a small number of bio_vecs. This member diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 6b47f94378c5..05998b5947a2 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -194,6 +194,8 @@ void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size, struct buffer_head *__bread_gfp(struct block_device *, sector_t block, unsigned size, gfp_t gfp); void invalidate_bh_lrus(void); +void invalidate_bh_lru(void *arg); +bool has_bh_in_lru(int cpu, void *dummy); struct buffer_head *alloc_buffer_head(gfp_t gfp_flags); void free_buffer_head(struct buffer_head * bh); void unlock_buffer(struct buffer_head *bh); @@ -406,6 +408,8 @@ static inline int inode_has_buffers(struct inode *inode) { return 0; } static inline void invalidate_inode_buffers(struct inode *inode) {} static inline int remove_inode_buffers(struct inode *inode) { return 1; } static inline int sync_mapping_buffers(struct address_space *mapping) { return 0; } +static inline void invalidate_bh_lru(void *arg) {} +static inline bool has_bh_in_lru(int cpu, void *dummy) { return 0; } #define buffer_heads_over_limit 0 #endif /* CONFIG_BLOCK */ diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index bc56287a1ed1..72fd8db62342 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -192,6 +192,7 @@ enum cpuhp_state { CPUHP_AP_ONLINE_DYN_END = CPUHP_AP_ONLINE_DYN + 30, CPUHP_AP_X86_HPET_ONLINE, CPUHP_AP_X86_KVM_CLK_ONLINE, + CPUHP_AP_DTPM_CPU_ONLINE, CPUHP_AP_ACTIVE, CPUHP_ONLINE, }; diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index a3ff9280f7aa..84c14dbcdb66 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -8,6 +8,7 @@ #include #include +#include struct cma; @@ -69,6 +70,11 @@ struct dma_map_ops { u64 (*get_required_mask)(struct device *dev); size_t (*max_mapping_size)(struct device *dev); unsigned long (*get_merge_boundary)(struct device *dev); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #ifdef CONFIG_DMA_OPS diff --git a/include/linux/dtpm.h b/include/linux/dtpm.h new file mode 100644 index 000000000000..e80a332e3d8a --- /dev/null +++ b/include/linux/dtpm.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Linaro Ltd + * + * Author: Daniel Lezcano + */ +#ifndef ___DTPM_H__ +#define ___DTPM_H__ + +#include + +#define MAX_DTPM_DESCR 8 +#define MAX_DTPM_CONSTRAINTS 1 + +struct dtpm { + struct powercap_zone zone; + struct dtpm *parent; + struct list_head sibling; + struct list_head children; + struct dtpm_ops *ops; + unsigned long flags; + u64 power_limit; + u64 power_max; + u64 power_min; + int weight; + void *private; +}; + +struct dtpm_ops { + u64 (*set_power_uw)(struct dtpm *, u64); + u64 (*get_power_uw)(struct dtpm *); + void (*release)(struct dtpm *); +}; + +struct dtpm_descr; + +typedef int (*dtpm_init_t)(struct dtpm_descr *); + +struct dtpm_descr { + struct dtpm *parent; + const char *name; + dtpm_init_t init; +}; + +/* Init section thermal table */ +extern struct dtpm_descr *__dtpm_table[]; +extern struct dtpm_descr *__dtpm_table_end[]; + +#define DTPM_TABLE_ENTRY(name) \ + static typeof(name) *__dtpm_table_entry_##name \ + __used __section("__dtpm_table") = &name + +#define DTPM_DECLARE(name) DTPM_TABLE_ENTRY(name) + +#define for_each_dtpm_table(__dtpm) \ + for (__dtpm = __dtpm_table; \ + __dtpm < __dtpm_table_end; \ + __dtpm++) + +static inline struct dtpm *to_dtpm(struct powercap_zone *zone) +{ + return container_of(zone, struct dtpm, zone); +} + +int dtpm_update_power(struct dtpm *dtpm, u64 power_min, u64 power_max); + +int dtpm_release_zone(struct powercap_zone *pcz); + +struct dtpm *dtpm_alloc(struct dtpm_ops *ops); + +void dtpm_unregister(struct dtpm *dtpm); + +int dtpm_register(const char *name, struct dtpm *dtpm, struct dtpm *parent); + +int dtpm_register_cpu(struct dtpm *parent); + +#endif diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 6408b446051f..b7b747c4d6e9 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -15,6 +15,7 @@ #include #include +#include #include #ifdef CONFIG_COMPAT @@ -509,6 +510,11 @@ struct ethtool_ops { const struct ethtool_tunable *, void *); int (*set_phy_tunable)(struct net_device *, const struct ethtool_tunable *, const void *); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; int ethtool_check_ops(const struct ethtool_ops *ops); diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 03da3f603d30..8935b1e63ada 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #define dev_to_disk(device) container_of((device), struct gendisk, part0.__dev) @@ -73,6 +74,11 @@ struct hd_struct { int make_it_fail; #endif struct rcu_work rcu_work; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /** @@ -162,6 +168,9 @@ struct blk_integrity { unsigned char tuple_size; unsigned char interval_exp; unsigned char tag_size; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct gendisk { @@ -208,6 +217,12 @@ struct gendisk { int node_id; struct badblocks *bb; struct lockdep_map lockdep_map; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); + }; #if IS_REACHABLE(CONFIG_CDROM) diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 107cedd7019a..38a8388f598f 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -20,6 +20,7 @@ #include #include #include +#include struct hrtimer_clock_base; struct hrtimer_cpu_base; @@ -124,6 +125,8 @@ struct hrtimer { u8 is_rel; u8 is_soft; u8 is_hard; + + ANDROID_KABI_RESERVE(1); }; /** diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 5bd3cac4df9c..8a6add3abb3f 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -9,6 +9,7 @@ #include #include #include +#include struct address_space; struct fiemap_extent_info; @@ -141,6 +142,9 @@ struct iomap_ops { */ int (*iomap_end)(struct inode *inode, loff_t pos, loff_t length, ssize_t written, unsigned flags, struct iomap *iomap); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /* diff --git a/include/linux/ioport.h b/include/linux/ioport.h index 5135d4b86cd6..d36041c1cd42 100644 --- a/include/linux/ioport.h +++ b/include/linux/ioport.h @@ -13,6 +13,7 @@ #include #include #include +#include /* * Resources are tree-like, allowing * nesting etc.. @@ -24,6 +25,11 @@ struct resource { unsigned long flags; unsigned long desc; struct resource *parent, *sibling, *child; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /* diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index c4f241f1f59b..a7c5c7f4d97f 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -3,6 +3,7 @@ #define _IPV6_H #include +#include #define ipv6_optlen(p) (((p)->hdrlen+1) << 3) #define ipv6_authlen(p) (((p)->hdrlen+2) << 2) @@ -78,6 +79,11 @@ struct ipv6_devconf { __s32 rpl_seg_enabled; struct ctl_table_header *sysctl_header; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; struct ipv6_params { diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index ea5a337e0f8b..d6208a426525 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -35,6 +35,7 @@ #include #include #include +#include struct device_node; struct irq_domain; @@ -178,6 +179,11 @@ struct irq_domain { struct dentry *debugfs_file; #endif + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); + /* reverse map data. The linear map gets appended to the irq_domain */ irq_hw_number_t hwirq_max; unsigned int revmap_direct_max_irq; diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 89f6a4214a70..c8529e9fa8de 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -16,6 +16,7 @@ #include #include #include +#include struct file; struct dentry; @@ -175,6 +176,11 @@ struct kernfs_syscall_ops { const char *new_name); int (*show_path)(struct seq_file *sf, struct kernfs_node *kn, struct kernfs_root *root); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; struct kernfs_root { @@ -268,6 +274,9 @@ struct kernfs_ops { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lock_class_key lockdep_key; #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /* diff --git a/include/linux/kobject.h b/include/linux/kobject.h index ea30529fba08..e6d8e1b8ab0a 100644 --- a/include/linux/kobject.h +++ b/include/linux/kobject.h @@ -27,6 +27,7 @@ #include #include #include +#include #define UEVENT_HELPER_PATH_LEN 256 #define UEVENT_NUM_ENVP 64 /* number of env pointers */ @@ -77,6 +78,11 @@ struct kobject { unsigned int state_add_uevent_sent:1; unsigned int state_remove_uevent_sent:1; unsigned int uevent_suppress:1; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; extern __printf(2, 3) @@ -143,6 +149,11 @@ struct kobj_type { const struct kobj_ns_type_operations *(*child_ns_type)(struct kobject *kobj); const void *(*namespace)(struct kobject *kobj); void (*get_ownership)(struct kobject *kobj, kuid_t *uid, kgid_t *gid); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; struct kobj_uevent_env { @@ -194,6 +205,11 @@ struct kset { spinlock_t list_lock; struct kobject kobj; const struct kset_uevent_ops *uevent_ops; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); } __randomize_layout; extern void kset_init(struct kset *kset); diff --git a/include/linux/migrate.h b/include/linux/migrate.h index fb2281b36953..8512469ff6fb 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -45,8 +45,6 @@ extern struct page *alloc_migration_target(struct page *page, unsigned long priv extern int isolate_movable_page(struct page *page, isolate_mode_t mode); extern void putback_movable_page(struct page *page); -extern int migrate_prep(void); -extern int migrate_prep_local(void); extern void migrate_page_states(struct page *newpage, struct page *page); extern void migrate_page_copy(struct page *newpage, struct page *page); extern int migrate_huge_page_move_mapping(struct address_space *mapping, @@ -66,9 +64,6 @@ static inline struct page *alloc_migration_target(struct page *page, static inline int isolate_movable_page(struct page *page, isolate_mode_t mode) { return -EBUSY; } -static inline int migrate_prep(void) { return -ENOSYS; } -static inline int migrate_prep_local(void) { return -ENOSYS; } - static inline void migrate_page_states(struct page *newpage, struct page *page) { } diff --git a/include/linux/mm.h b/include/linux/mm.h index 151d4f2fbb49..68e65aebf392 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -32,6 +32,7 @@ #include #include #include +#include struct mempolicy; struct anon_vma; @@ -638,6 +639,11 @@ struct vm_operations_struct { */ struct page *(*find_special_page)(struct vm_area_struct *vma, unsigned long addr); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; static inline void INIT_VMA(struct vm_area_struct *vma) diff --git a/include/linux/mmc/core.h b/include/linux/mmc/core.h index ab19245e9945..71101d1ec825 100644 --- a/include/linux/mmc/core.h +++ b/include/linux/mmc/core.h @@ -164,9 +164,8 @@ struct mmc_request { int tag; #ifdef CONFIG_MMC_CRYPTO - bool crypto_enabled; + const struct bio_crypt_ctx *crypto_ctx; int crypto_key_slot; - u32 data_unit_num; #endif }; diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index b8200782dede..a474b773718b 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h @@ -8,6 +8,7 @@ #include #include #include +#include struct mmu_notifier_subscriptions; struct mmu_notifier; @@ -215,6 +216,11 @@ struct mmu_notifier_ops { */ struct mmu_notifier *(*alloc_notifier)(struct mm_struct *mm); void (*free_notifier)(struct mmu_notifier *subscription); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /* @@ -234,6 +240,9 @@ struct mmu_notifier { struct mm_struct *mm; struct rcu_head rcu; unsigned int users; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /** diff --git a/include/linux/net.h b/include/linux/net.h index 0dcd51feef02..f5569faf930b 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -23,6 +23,7 @@ #include #include #include +#include #include @@ -203,6 +204,11 @@ struct proto_ops { int (*sendmsg_locked)(struct sock *sk, struct msghdr *msg, size_t size); int (*set_rcvlowat)(struct sock *sk, int val); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #define DECLARE_SOCKADDR(type, dst, src) \ diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 8753e98a8d58..0b8653232d0d 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -48,6 +48,7 @@ #include #include #include +#include struct netpoll_info; struct device; @@ -278,6 +279,9 @@ struct header_ops { const unsigned char *haddr); bool (*validate)(const char *ll_header, unsigned int len); __be16 (*parse_protocol)(const struct sk_buff *skb); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /* These flag bits are private to the generic network queueing @@ -347,6 +351,11 @@ struct napi_struct { struct list_head dev_list; struct hlist_node napi_hash_node; unsigned int napi_id; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; enum { @@ -621,6 +630,11 @@ struct netdev_queue { #ifdef CONFIG_BQL struct dql dql; #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); } ____cacheline_aligned_in_smp; extern int sysctl_fb_tunnels_only_for_init_net; @@ -744,6 +758,11 @@ struct netdev_rx_queue { #ifdef CONFIG_XDP_SOCKETS struct xsk_buff_pool *pool; #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); } ____cacheline_aligned_in_smp; /* @@ -924,6 +943,11 @@ struct xfrmdev_ops { bool (*xdo_dev_offload_ok) (struct sk_buff *skb, struct xfrm_state *x); void (*xdo_dev_state_advance_esn) (struct xfrm_state *x); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #endif @@ -1487,6 +1511,15 @@ struct net_device_ops { int (*ndo_tunnel_ctl)(struct net_device *dev, struct ip_tunnel_parm *p, int cmd); struct net_device * (*ndo_get_peer_dev)(struct net_device *dev); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); + ANDROID_KABI_RESERVE(5); + ANDROID_KABI_RESERVE(6); + ANDROID_KABI_RESERVE(7); + ANDROID_KABI_RESERVE(8); }; /** @@ -2164,6 +2197,15 @@ struct net_device { /* protected by rtnl_lock */ struct bpf_xdp_entity xdp_state[__MAX_XDP_MODE]; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); + ANDROID_KABI_RESERVE(5); + ANDROID_KABI_RESERVE(6); + ANDROID_KABI_RESERVE(7); + ANDROID_KABI_RESERVE(8); }; #define to_net_dev(d) container_of(d, struct net_device, dev) @@ -2513,6 +2555,11 @@ struct packet_type { struct sock *sk); void *af_packet_priv; struct list_head list; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; struct offload_callbacks { diff --git a/include/linux/pci.h b/include/linux/pci.h index 081bbc1eb00a..cc772efa9713 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -41,6 +41,7 @@ #include #include +#include #define PCI_STATUS_ERROR_BITS (PCI_STATUS_DETECTED_PARITY | \ PCI_STATUS_SIG_SYSTEM_ERROR | \ @@ -500,6 +501,11 @@ struct pci_dev { char *driver_override; /* Driver name to force a match */ unsigned long priv_flags; /* Private flags for the PCI driver */ + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; static inline struct pci_dev *pci_physfn(struct pci_dev *dev) @@ -636,6 +642,11 @@ struct pci_bus { struct bin_attribute *legacy_io; /* Legacy I/O for this bus */ struct bin_attribute *legacy_mem; /* Legacy mem */ unsigned int is_added:1; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #define to_pci_bus(n) container_of(n, struct pci_bus, dev) @@ -869,6 +880,11 @@ struct pci_driver { const struct attribute_group **groups; struct device_driver driver; struct pci_dynids dynids; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #define to_pci_driver(drv) container_of(drv, struct pci_driver, driver) diff --git a/include/linux/quota.h b/include/linux/quota.h index 27aab84fcbaa..3088ac929e56 100644 --- a/include/linux/quota.h +++ b/include/linux/quota.h @@ -316,6 +316,9 @@ struct quota_format_ops { int (*commit_dqblk)(struct dquot *dquot); /* Write structure for one user */ int (*release_dqblk)(struct dquot *dquot); /* Called when last reference to dquot is being dropped */ int (*get_next_id)(struct super_block *sb, struct kqid *qid); /* Get next ID with existing structure in the quota file */ + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /* Operations working with dquots */ @@ -335,6 +338,9 @@ struct dquot_operations { int (*get_inode_usage) (struct inode *, qsize_t *); /* Get next ID with active quota structure */ int (*get_next_id) (struct super_block *sb, struct kqid *qid); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct path; @@ -438,6 +444,9 @@ struct quotactl_ops { int (*set_dqblk)(struct super_block *, struct kqid, struct qc_dqblk *); int (*get_state)(struct super_block *, struct qc_state *); int (*rm_xquota)(struct super_block *, unsigned int); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct quota_format_type { diff --git a/include/linux/sched.h b/include/linux/sched.h index f40e26b230e0..a4d6cb3c12ca 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1358,7 +1358,7 @@ struct task_struct { struct callback_head mce_kill_me; #endif ANDROID_VENDOR_DATA_ARRAY(1, 64); - ANDROID_OEM_DATA_ARRAY(1, 2); + ANDROID_OEM_DATA_ARRAY(1, 6); /* * New fields for task_struct should be added above here, so that diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index a828cf99c521..e68a8d7fb523 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -40,6 +40,7 @@ #if IS_ENABLED(CONFIG_NF_CONNTRACK) #include #endif +#include /* The interface for checksum offload between the stack and networking drivers * is as follows... @@ -908,6 +909,9 @@ struct sk_buff { __u32 headers_end[0]; /* public: */ + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + /* These elements must be at the end, see alloc_skb() for details. */ sk_buff_data_t tail; sk_buff_data_t end; diff --git a/include/linux/swap.h b/include/linux/swap.h index 7aebbd1f17d2..865ed8c2c939 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -341,6 +341,9 @@ extern void lru_cache_add(struct page *); extern void lru_add_page_tail(struct page *page, struct page *page_tail, struct lruvec *lruvec, struct list_head *head); extern void mark_page_accessed(struct page *); +extern void lru_cache_disable(void); +extern void lru_cache_enable(void); +extern bool lru_cache_disabled(void); extern void lru_add_drain(void); extern void lru_add_drain_cpu(int cpu); extern void lru_add_drain_cpu_zone(struct zone *zone); diff --git a/include/linux/timer.h b/include/linux/timer.h index d10bc7e73b41..a01c23824b88 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -7,6 +7,7 @@ #include #include #include +#include struct timer_list { /* @@ -21,6 +22,9 @@ struct timer_list { #ifdef CONFIG_LOCKDEP struct lockdep_map lockdep_map; #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; #ifdef CONFIG_LOCKDEP diff --git a/include/linux/usb/gadget.h b/include/linux/usb/gadget.h index ee04ef214ce8..ea02847923bf 100644 --- a/include/linux/usb/gadget.h +++ b/include/linux/usb/gadget.h @@ -26,6 +26,7 @@ #include #include #include +#include #define UDC_TRACE_STR_MAX 512 @@ -444,6 +445,11 @@ struct usb_gadget { unsigned connected:1; unsigned lpm_capable:1; int irq; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #define work_to_gadget(w) (container_of((w), struct usb_gadget, work)) diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h index 6ef1c7109fc4..2330b911ee47 100644 --- a/include/linux/user_namespace.h +++ b/include/linux/user_namespace.h @@ -10,6 +10,7 @@ #include #include #include +#include #define UID_GID_MAP_MAX_BASE_EXTENTS 5 #define UID_GID_MAP_MAX_EXTENTS 340 @@ -87,6 +88,9 @@ struct user_namespace { #endif struct ucounts *ucounts; int ucount_max[UCOUNT_COUNTS]; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); } __randomize_layout; struct ucounts { diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h index 439379ca9ffa..7a44164f147c 100644 --- a/include/net/netfilter/nf_conntrack.h +++ b/include/net/netfilter/nf_conntrack.h @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -105,6 +106,9 @@ struct nf_conn { /* Storage reserved for other modules, must be the last member */ union nf_conntrack_proto proto; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; static inline struct nf_conn * diff --git a/include/net/tls.h b/include/net/tls.h index 2bdd802212fe..690b76cbd2a0 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -292,6 +293,12 @@ struct tlsdev_ops { int (*tls_dev_resync)(struct net_device *netdev, struct sock *sk, u32 seq, u8 *rcd_sn, enum tls_offload_ctx_dir direction); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); + }; enum tls_offload_sync_type { diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index e7af6bdd0bf5..bcd7f1fbb8cd 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -677,14 +677,6 @@ DECLARE_TRACE(sched_update_nr_running_tp, TP_PROTO(struct rq *rq, int change), TP_ARGS(rq, change)); -DECLARE_TRACE(sugov_util_update_tp, - TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max_cap, unsigned int flags), - TP_ARGS(cpu, util, max_cap, flags)); - -DECLARE_TRACE(sugov_next_freq_tp, - TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max, unsigned int freq), - TP_ARGS(cpu, util, max, freq)); - #endif /* _TRACE_SCHED_H */ /* This part must be outside protection */ diff --git a/include/trace/hooks/avc.h b/include/trace/hooks/avc.h new file mode 100644 index 000000000000..2c76e02fc3c5 --- /dev/null +++ b/include/trace/hooks/avc.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM avc + +#define TRACE_INCLUDE_PATH trace/hooks +#if !defined(_TRACE_HOOK_AVC_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_AVC_H +#include +#include +/* + * Following tracepoints are not exported in tracefs and provide a + * mechanism for vendor modules to hook and extend functionality + */ +struct avc_node; +DECLARE_HOOK(android_vh_selinux_avc_insert, + TP_PROTO(const struct avc_node *node), + TP_ARGS(node)); + +DECLARE_HOOK(android_vh_selinux_avc_node_delete, + TP_PROTO(const struct avc_node *node), + TP_ARGS(node)); + +DECLARE_HOOK(android_vh_selinux_avc_node_replace, + TP_PROTO(const struct avc_node *old, const struct avc_node *new), + TP_ARGS(old, new)); + +DECLARE_HOOK(android_vh_selinux_avc_lookup, + TP_PROTO(const struct avc_node *node, u32 ssid, u32 tsid, u16 tclass), + TP_ARGS(node, ssid, tsid, tclass)); + +#endif /* _TRACE_HOOK_AVC_H */ +/* This part must be outside protection */ +#include diff --git a/include/trace/hooks/creds.h b/include/trace/hooks/creds.h new file mode 100644 index 000000000000..dd877e3a1f3e --- /dev/null +++ b/include/trace/hooks/creds.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM creds + +#define TRACE_INCLUDE_PATH trace/hooks +#if !defined(_TRACE_HOOK_CREDS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_CREDS_H +#include +#include +/* + * Following tracepoints are not exported in tracefs and provide a + * mechanism for vendor modules to hook and extend functionality + */ +struct cred; +struct task_struct; +DECLARE_HOOK(android_vh_commit_creds, + TP_PROTO(const struct task_struct *task, const struct cred *new), + TP_ARGS(task, new)); + +DECLARE_HOOK(android_vh_exit_creds, + TP_PROTO(const struct task_struct *task, const struct cred *cred), + TP_ARGS(task, cred)); + +DECLARE_HOOK(android_vh_override_creds, + TP_PROTO(const struct task_struct *task, const struct cred *new), + TP_ARGS(task, new)); + +DECLARE_HOOK(android_vh_revert_creds, + TP_PROTO(const struct task_struct *task, const struct cred *old), + TP_ARGS(task, old)); + +#endif /* _TRACE_HOOK_CREDS_H */ +/* This part must be outside protection */ +#include diff --git a/include/trace/hooks/memory.h b/include/trace/hooks/memory.h new file mode 100644 index 000000000000..4b3f77a72d82 --- /dev/null +++ b/include/trace/hooks/memory.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM memory + +#define TRACE_INCLUDE_PATH trace/hooks +#if !defined(_TRACE_HOOK_MEMORY_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_MEMORY_H +#include +#include +/* + * Following tracepoints are not exported in tracefs and provide a + * mechanism for vendor modules to hook and extend functionality + */ +DECLARE_HOOK(android_vh_set_memory_x, + TP_PROTO(unsigned long addr, int nr_pages), + TP_ARGS(addr, nr_pages)); + +DECLARE_HOOK(android_vh_set_memory_nx, + TP_PROTO(unsigned long addr, int nr_pages), + TP_ARGS(addr, nr_pages)); + +DECLARE_HOOK(android_vh_set_memory_ro, + TP_PROTO(unsigned long addr, int nr_pages), + TP_ARGS(addr, nr_pages)); + +DECLARE_HOOK(android_vh_set_memory_rw, + TP_PROTO(unsigned long addr, int nr_pages), + TP_ARGS(addr, nr_pages)); + +#endif /* _TRACE_HOOK_MEMORY_H */ +/* This part must be outside protection */ +#include diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index d0639cfdebd2..2fcc968e659e 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -32,6 +32,25 @@ DECLARE_HOOK(android_vh_pagecache_get_page, TP_PROTO(struct address_space *mapping, pgoff_t index, int fgp_flags, gfp_t gfp_mask, struct page *page), TP_ARGS(mapping, index, fgp_flags, gfp_mask, page)); +DECLARE_HOOK(android_vh_meminfo_proc_show, + TP_PROTO(struct seq_file *m), + TP_ARGS(m)); +DECLARE_HOOK(android_vh_exit_mm, + TP_PROTO(struct mm_struct *mm), + TP_ARGS(mm)); +DECLARE_HOOK(android_vh_show_mem, + TP_PROTO(unsigned int filter, nodemask_t *nodemask), + TP_ARGS(filter, nodemask)); +DECLARE_HOOK(android_vh_alloc_pages_slowpath, + TP_PROTO(gfp_t gfp_mask, unsigned int order, unsigned long delta), + TP_ARGS(gfp_mask, order, delta)); +DECLARE_HOOK(android_vh_print_slabinfo_header, + TP_PROTO(struct seq_file *m), + TP_ARGS(m)); +struct slabinfo; +DECLARE_HOOK(android_vh_cache_show, + TP_PROTO(struct seq_file *m, struct slabinfo *sinfo, struct kmem_cache *s), + TP_ARGS(m, sinfo, s)); /* macro versions of hooks are no longer required */ #endif /* _TRACE_HOOK_MM_H */ diff --git a/include/trace/hooks/module.h b/include/trace/hooks/module.h new file mode 100644 index 000000000000..281cb0d37c12 --- /dev/null +++ b/include/trace/hooks/module.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM module + +#define TRACE_INCLUDE_PATH trace/hooks +#if !defined(_TRACE_HOOK_MODULE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_MODULE_H +#include +#include +/* + * Following tracepoints are not exported in tracefs and provide a + * mechanism for vendor modules to hook and extend functionality + */ +struct module; +DECLARE_HOOK(android_vh_set_module_permit_before_init, + TP_PROTO(const struct module *mod), + TP_ARGS(mod)); + +DECLARE_HOOK(android_vh_set_module_permit_after_init, + TP_PROTO(const struct module *mod), + TP_ARGS(mod)); + +#endif /* _TRACE_HOOK_MODULE_H */ +/* This part must be outside protection */ +#include diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h index 2358afcc9bdb..aed5b8f6fd0e 100644 --- a/include/trace/hooks/sched.h +++ b/include/trace/hooks/sched.h @@ -113,20 +113,10 @@ DECLARE_RESTRICTED_HOOK(android_rvh_find_energy_efficient_cpu, TP_PROTO(struct task_struct *p, int prev_cpu, int sync, int *new_cpu), TP_ARGS(p, prev_cpu, sync, new_cpu), 1); -struct sched_attr; -DECLARE_HOOK(android_vh_set_sugov_sched_attr, - TP_PROTO(struct sched_attr *attr), - TP_ARGS(attr)); - DECLARE_RESTRICTED_HOOK(android_rvh_set_iowait, TP_PROTO(struct task_struct *p, int *should_iowait_boost), TP_ARGS(p, should_iowait_boost), 1); -struct sugov_policy; -DECLARE_RESTRICTED_HOOK(android_rvh_set_sugov_update, - TP_PROTO(struct sugov_policy *sg_policy, unsigned int next_freq, bool *should_update), - TP_ARGS(sg_policy, next_freq, should_update), 1); - DECLARE_RESTRICTED_HOOK(android_rvh_cpu_overutilized, TP_PROTO(int cpu, int *overutilized), TP_ARGS(cpu, overutilized), 1); @@ -190,8 +180,8 @@ DECLARE_RESTRICTED_HOOK(android_rvh_account_irq, struct sched_entity; DECLARE_RESTRICTED_HOOK(android_rvh_place_entity, - TP_PROTO(struct sched_entity *se, u64 *vruntime), - TP_ARGS(se, vruntime), 1); + TP_PROTO(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial, u64 vruntime), + TP_ARGS(cfs_rq, se, initial, vruntime), 1); DECLARE_RESTRICTED_HOOK(android_rvh_build_perf_domains, TP_PROTO(bool *eas_check), @@ -270,10 +260,27 @@ DECLARE_HOOK(android_vh_set_wake_flags, enum uclamp_id; struct uclamp_se; -DECLARE_RESTRICTED_HOOK(android_rvh_uclamp_eff_value, +DECLARE_RESTRICTED_HOOK(android_rvh_uclamp_eff_get, TP_PROTO(struct task_struct *p, enum uclamp_id clamp_id, - struct uclamp_se *uclamp_default, unsigned long *ret), - TP_ARGS(p, clamp_id, uclamp_default, ret), 1); + struct uclamp_se *uclamp_max, struct uclamp_se *uclamp_eff, int *ret), + TP_ARGS(p, clamp_id, uclamp_max, uclamp_eff, ret), 1); + +DECLARE_HOOK(android_vh_build_sched_domains, + TP_PROTO(bool has_asym), + TP_ARGS(has_asym)); +DECLARE_RESTRICTED_HOOK(android_rvh_check_preempt_tick, + TP_PROTO(struct task_struct *p, unsigned long *ideal_runtime, bool *skip_preempt), + TP_ARGS(p, ideal_runtime, skip_preempt), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_check_preempt_wakeup_ignore, + TP_PROTO(struct task_struct *p, bool *ignore), + TP_ARGS(p, ignore), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_replace_next_task_fair, + TP_PROTO(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple), + TP_ARGS(rq, p, se, repick, simple), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_util_est_update, + TP_PROTO(struct cfs_rq *cfs_rq, struct task_struct *p, bool task_sleep, int *ret), + TP_ARGS(cfs_rq, p, task_sleep, ret), 1); /* macro versions of hooks are no longer required */ diff --git a/include/trace/hooks/thermal.h b/include/trace/hooks/thermal.h index 0e47e0af9284..ae9b7ee652fb 100644 --- a/include/trace/hooks/thermal.h +++ b/include/trace/hooks/thermal.h @@ -11,8 +11,8 @@ #include DECLARE_HOOK(android_vh_enable_thermal_genl_check, - TP_PROTO(int event, int *enable_thermal_genl), - TP_ARGS(event, enable_thermal_genl)); + TP_PROTO(int event, int tz_id, int *enable_thermal_genl), + TP_ARGS(event, tz_id, enable_thermal_genl)); #endif /* _TRACE_HOOK_THERMAL_H */ /* This part must be outside protection */ diff --git a/include/trace/hooks/traps.h b/include/trace/hooks/traps.h new file mode 100644 index 000000000000..350a14369acc --- /dev/null +++ b/include/trace/hooks/traps.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM traps +#define TRACE_INCLUDE_PATH trace/hooks + +#if !defined(_TRACE_HOOK_TRAPS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_TRAPS_H +#include +#include +/* + * Following tracepoints are not exported in tracefs and provide a + * mechanism for vendor modules to hook and extend functionality + */ +struct pt_regs; +DECLARE_RESTRICTED_HOOK(android_rvh_do_undefinstr, + TP_PROTO(struct pt_regs *regs, bool user), + TP_ARGS(regs, user), + TP_CONDITION(!user)); + +DECLARE_RESTRICTED_HOOK(android_rvh_bad_mode, + TP_PROTO(struct pt_regs *regs, unsigned int esr, int reason), + TP_ARGS(regs, reason, esr), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_arm64_serror_panic, + TP_PROTO(struct pt_regs *regs, unsigned int esr), + TP_ARGS(regs, esr), 1); + +/* macro versions of hooks are no longer required */ + +#endif /* _TRACE_HOOK_TRAPS_H */ +/* This part must be outside protection */ +#include diff --git a/include/trace/hooks/typec.h b/include/trace/hooks/typec.h new file mode 100644 index 000000000000..127a16b440c1 --- /dev/null +++ b/include/trace/hooks/typec.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM typec +#define TRACE_INCLUDE_PATH trace/hooks +#if !defined(_TRACE_HOOK_TYPEC_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_TYPEC_H +#include +#include +/* + * Following tracepoints are not exported in tracefs and provide a + * mechanism for vendor modules to hook and extend functionality + */ +struct tcpci; +struct tcpci_data; + +DECLARE_HOOK(android_vh_typec_tcpci_override_toggling, + TP_PROTO(struct tcpci *tcpci, struct tcpci_data *data, int *override_toggling), + TP_ARGS(tcpci, data, override_toggling)); + +DECLARE_HOOK(android_vh_typec_tcpci_check_contaminant, + TP_PROTO(struct tcpci *tcpci, struct tcpci_data *data, int *ret), + TP_ARGS(tcpci, data, ret)); + +/* + * This hook is for addressing hardware anomalies where TCPC_POWER_STATUS_VBUS_PRES bit can return 0 + * even before falling below sSinkDisconnect threshold. + * Handler has to set bypass to override the value that would otherwise be returned by this + * function. + * Handler can set vbus or clear vbus to indicate vbus present or absent + */ +DECLARE_HOOK(android_vh_typec_tcpci_get_vbus, + TP_PROTO(struct tcpci *tcpci, struct tcpci_data *data, int *vbus, int *bypass), + TP_ARGS(tcpci, data, vbus, bypass)); + +#endif /* _TRACE_HOOK_UFSHCD_H */ +/* This part must be outside protection */ +#include diff --git a/include/trace/hooks/ufshcd.h b/include/trace/hooks/ufshcd.h index e2b755271457..6e37044eb264 100644 --- a/include/trace/hooks/ufshcd.h +++ b/include/trace/hooks/ufshcd.h @@ -50,6 +50,11 @@ DECLARE_HOOK(android_vh_ufs_check_int_errors, TP_PROTO(struct ufs_hba *hba, bool queue_eh_work), TP_ARGS(hba, queue_eh_work)); +struct scsi_device; +DECLARE_HOOK(android_vh_ufs_update_sdev, + TP_PROTO(struct scsi_device *sdev), + TP_ARGS(sdev)); + #endif /* _TRACE_HOOK_UFSHCD_H */ /* This part must be outside protection */ #include diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 4c3b543bb33b..94d832e571d9 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -10,6 +10,7 @@ #include #include #include +#include enum bpf_struct_ops_state { BPF_STRUCT_OPS_STATE_INIT, @@ -448,7 +449,9 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, bpf_map_inc(map); set_memory_ro((long)st_map->image, 1); + trace_android_vh_set_memory_ro((unsigned long)st_map->image, 1); set_memory_x((long)st_map->image, 1); + trace_android_vh_set_memory_x((unsigned long)st_map->image, 1); err = st_ops->reg(kdata); if (likely(!err)) { /* Pair with smp_load_acquire() during lookup_elem(). @@ -532,6 +535,8 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) if (st_map->progs) bpf_struct_ops_map_put_progs(st_map); bpf_map_area_free(st_map->progs); + trace_android_vh_set_memory_rw((unsigned long)st_map->image, 1); + trace_android_vh_set_memory_nx((unsigned long)st_map->image, 1); bpf_jit_free_exec(st_map->image); bpf_map_area_free(st_map->uvalue); bpf_map_area_free(st_map); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 55454d2278b1..a9f8bb65420e 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -34,6 +34,8 @@ #include #include +#include + /* Registers */ #define BPF_R0 regs[BPF_REG_0] #define BPF_R1 regs[BPF_REG_1] @@ -897,6 +899,8 @@ void bpf_jit_binary_free(struct bpf_binary_header *hdr) { u32 pages = hdr->pages; + trace_android_vh_set_memory_rw((unsigned long)hdr, pages); + trace_android_vh_set_memory_nx((unsigned long)hdr, pages); bpf_jit_free_exec(hdr); bpf_jit_uncharge_modmem(pages); } @@ -2294,6 +2298,7 @@ DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key); EXPORT_SYMBOL(bpf_stats_enabled_key); /* All definitions of tracepoints related to BPF. */ +#undef TRACE_INCLUDE_PATH #define CREATE_TRACE_POINTS #include diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 35c5887d82ff..e73fdf278fb5 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -9,6 +9,7 @@ #include #include #include +#include /* dummy _ops. The verifier will operate on target program's ops. */ const struct bpf_verifier_ops bpf_extension_verifier_ops = { @@ -38,6 +39,7 @@ void *bpf_jit_alloc_exec_page(void) * everytime new program is attached or detached. */ set_memory_x((long)image, 1); + trace_android_vh_set_memory_x((unsigned long)image, 1); return image; } @@ -374,6 +376,7 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) * for tasks to get out of trampoline code before freeing it. */ synchronize_rcu_tasks(); + trace_android_vh_set_memory_nx((unsigned long)tr->image, 1); bpf_jit_free_exec(tr->image); hlist_del(&tr->hlist); kfree(tr); diff --git a/kernel/cpu.c b/kernel/cpu.c index 62cb4fb7a867..135037717b0b 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -1005,7 +1005,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); int prev_state, ret = 0; - if (num_active_cpus() == 1) + if (num_active_cpus() == 1 && cpu_active(cpu)) return -EBUSY; if (!cpu_present(cpu)) diff --git a/kernel/cred.c b/kernel/cred.c index 421b1149c651..64bd9efa1c27 100644 --- a/kernel/cred.c +++ b/kernel/cred.c @@ -17,6 +17,8 @@ #include #include +#include + #if 0 #define kdebug(FMT, ...) \ printk("[%-5.5s%5u] " FMT "\n", \ @@ -178,6 +180,7 @@ void exit_creds(struct task_struct *tsk) key_put(tsk->cached_requested_key); tsk->cached_requested_key = NULL; #endif + trace_android_vh_exit_creds(tsk, cred); } /** @@ -489,6 +492,7 @@ int commit_creds(struct cred *new) atomic_inc(&new->user->processes); rcu_assign_pointer(task->real_cred, new); rcu_assign_pointer(task->cred, new); + trace_android_vh_commit_creds(task, new); if (new->user != old->user) atomic_dec(&old->user->processes); alter_cred_subscribers(old, -2); @@ -566,6 +570,7 @@ const struct cred *override_creds(const struct cred *new) get_new_cred((struct cred *)new); alter_cred_subscribers(new, 1); rcu_assign_pointer(current->cred, new); + trace_android_vh_override_creds(current, new); alter_cred_subscribers(old, -1); kdebug("override_creds() = %p{%d,%d}", old, @@ -594,6 +599,7 @@ void revert_creds(const struct cred *old) validate_creds(override); alter_cred_subscribers(old, 1); rcu_assign_pointer(current->cred, old); + trace_android_vh_revert_creds(current, old); alter_cred_subscribers(override, -1); put_cred(override); } diff --git a/kernel/exit.c b/kernel/exit.c index d13d67fc5f4e..f61ac326daab 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -68,6 +68,7 @@ #include #include #include +#include static void __unhash_process(struct task_struct *p, bool group_dead) { @@ -484,6 +485,7 @@ static void exit_mm(void) enter_lazy_tlb(mm, current); task_unlock(current); mm_update_next_owner(mm); + trace_android_vh_exit_mm(mm); mmput(mm); if (test_thread_flag(TIF_MEMDIE)) exit_oom_victim(); diff --git a/kernel/module.c b/kernel/module.c index 972ef24295a4..a0b99cd4cef7 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -63,6 +63,10 @@ #define CREATE_TRACE_POINTS #include +#undef CREATE_TRACE_POINTS +#include +#include + #ifndef ARCH_SHF_SMALL #define ARCH_SHF_SMALL 0 #endif @@ -2257,6 +2261,10 @@ static void free_module(struct module *mod) /* This may be empty, but that's OK */ module_arch_freeing_init(mod); + trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base, + (mod->init_layout.size)>>PAGE_SHIFT); + trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base, + (mod->init_layout.size)>>PAGE_SHIFT); module_memfree(mod->init_layout.base); kfree(mod->args); percpu_modfree(mod); @@ -2265,6 +2273,10 @@ static void free_module(struct module *mod) lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size); /* Finally, free the core (containing the module structure) */ + trace_android_vh_set_memory_rw((unsigned long)mod->core_layout.base, + (mod->core_layout.size)>>PAGE_SHIFT); + trace_android_vh_set_memory_nx((unsigned long)mod->core_layout.base, + (mod->core_layout.size)>>PAGE_SHIFT); module_memfree(mod->core_layout.base); } @@ -3537,7 +3549,15 @@ static void module_deallocate(struct module *mod, struct load_info *info) { percpu_modfree(mod); module_arch_freeing_init(mod); + trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base, + (mod->init_layout.size)>>PAGE_SHIFT); + trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base, + (mod->init_layout.size)>>PAGE_SHIFT); module_memfree(mod->init_layout.base); + trace_android_vh_set_memory_rw((unsigned long)mod->core_layout.base, + (mod->core_layout.size)>>PAGE_SHIFT); + trace_android_vh_set_memory_nx((unsigned long)mod->core_layout.base, + (mod->core_layout.size)>>PAGE_SHIFT); module_memfree(mod->core_layout.base); } @@ -3695,8 +3715,13 @@ static noinline int do_init_module(struct module *mod) rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms); #endif module_enable_ro(mod, true); + trace_android_vh_set_module_permit_after_init(mod); mod_tree_remove_init(mod); module_arch_freeing_init(mod); + trace_android_vh_set_memory_rw((unsigned long)mod->init_layout.base, + (mod->init_layout.size)>>PAGE_SHIFT); + trace_android_vh_set_memory_nx((unsigned long)mod->init_layout.base, + (mod->init_layout.size)>>PAGE_SHIFT); mod->init_layout.base = NULL; mod->init_layout.size = 0; mod->init_layout.ro_size = 0; @@ -3803,6 +3828,7 @@ static int complete_formation(struct module *mod, struct load_info *info) module_enable_ro(mod, false); module_enable_nx(mod); module_enable_x(mod); + trace_android_vh_set_module_permit_before_init(mod); /* Mark state as coming so strong_try_module_get() ignores us, * but kallsyms etc. can see us. */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 73da84989397..c9c0735198b1 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -45,8 +45,6 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); EXPORT_TRACEPOINT_SYMBOL_GPL(sched_switch); -EXPORT_TRACEPOINT_SYMBOL_GPL(sugov_next_freq_tp); -EXPORT_TRACEPOINT_SYMBOL_GPL(sugov_util_update_tp); DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); EXPORT_SYMBOL_GPL(runqueues); @@ -1130,6 +1128,12 @@ uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id) { struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id); struct uclamp_se uc_max = uclamp_default[clamp_id]; + struct uclamp_se uc_eff; + int ret = 0; + + trace_android_rvh_uclamp_eff_get(p, clamp_id, &uc_max, &uc_eff, &ret); + if (ret) + return uc_eff; /* System default restrictions always apply */ if (unlikely(uc_req.value > uc_max.value)) @@ -1142,12 +1146,6 @@ unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id) { struct uclamp_se uc_eff; - unsigned long ret = ULONG_MAX; - - trace_android_rvh_uclamp_eff_value(p, clamp_id, &uclamp_default[clamp_id], &ret); - if (ret != ULONG_MAX) - return ret; - /* Task currently refcounted: use back-annotated (effective) value */ if (p->uclamp[clamp_id].active) return (unsigned long)p->uclamp[clamp_id].value; diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index ac5b1d4eb830..caa526ba16ea 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -103,7 +103,6 @@ static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time) static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time, unsigned int next_freq) { - bool should_update = true; if (!sg_policy->need_freq_update) { if (sg_policy->next_freq == next_freq) return false; @@ -111,10 +110,6 @@ static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time, sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS); } - trace_android_rvh_set_sugov_update(sg_policy, next_freq, &should_update); - if (!should_update) - return false; - sg_policy->next_freq = next_freq; sg_policy->last_freq_update_time = time; @@ -176,8 +171,6 @@ static unsigned int get_next_freq(struct sugov_policy *sg_policy, else freq = map_util_freq(util, freq, max); - trace_sugov_next_freq_tp(policy->cpu, util, max, freq); - if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update) return sg_policy->next_freq; @@ -469,9 +462,6 @@ static void sugov_update_single(struct update_util_data *hook, u64 time, util = sugov_get_util(sg_cpu); max = sg_cpu->max; - - trace_sugov_util_update_tp(sg_cpu->cpu, util, max, flags); - util = sugov_iowait_apply(sg_cpu, time, util, max); next_f = get_next_freq(sg_policy, util, max); /* @@ -528,7 +518,6 @@ sugov_update_shared(struct update_util_data *hook, u64 time, unsigned int flags) { struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); struct sugov_policy *sg_policy = sg_cpu->sg_policy; - unsigned long util = sugov_get_util(sg_cpu); unsigned int next_f; raw_spin_lock(&sg_policy->update_lock); @@ -541,8 +530,6 @@ sugov_update_shared(struct update_util_data *hook, u64 time, unsigned int flags) if (sugov_should_update_freq(sg_policy, time)) { next_f = sugov_next_freq_shared(sg_cpu, time); - trace_sugov_util_update_tp(sg_cpu->cpu, util, sg_cpu->max, flags); - if (sg_policy->policy->fast_switch_enabled) sugov_fast_switch(sg_policy, time, next_f); else @@ -681,7 +668,6 @@ static int sugov_kthread_create(struct sugov_policy *sg_policy) if (policy->fast_switch_enabled) return 0; - trace_android_vh_set_sugov_sched_attr(&attr); kthread_init_work(&sg_policy->work, sugov_work); kthread_init_worker(&sg_policy->worker); thread = kthread_create(kthread_worker_fn, &sg_policy->worker, diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 17eec77acbfe..9ed56b5d7efb 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3971,6 +3971,11 @@ static inline void util_est_update(struct cfs_rq *cfs_rq, { long last_ewma_diff; struct util_est ue; + int ret = 0; + + trace_android_rvh_util_est_update(cfs_rq, p, task_sleep, &ret); + if (ret) + return; if (!sched_feat(UTIL_EST)) return; @@ -4148,11 +4153,11 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) thresh >>= 1; vruntime -= thresh; - trace_android_rvh_place_entity(se, &vruntime); } /* ensure we never gain time by being placed backwards. */ se->vruntime = max_vruntime(se->vruntime, vruntime); + trace_android_rvh_place_entity(cfs_rq, se, initial, vruntime); } static void check_enqueue_throttle(struct cfs_rq *cfs_rq); @@ -4376,9 +4381,13 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) unsigned long ideal_runtime, delta_exec; struct sched_entity *se; s64 delta; + bool skip_preempt = false; ideal_runtime = sched_slice(cfs_rq, curr); delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; + trace_android_rvh_check_preempt_tick(current, &ideal_runtime, &skip_preempt); + if (skip_preempt) + return; if (delta_exec > ideal_runtime) { resched_curr(rq_of(cfs_rq)); /* @@ -7018,9 +7027,13 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ int scale = cfs_rq->nr_running >= sched_nr_latency; int next_buddy_marked = 0; bool preempt = false; + bool ignore = false; if (unlikely(se == pse)) return; + trace_android_rvh_check_preempt_wakeup_ignore(curr, &ignore); + if (ignore) + return; /* * This is possible from callers such as attach_tasks(), in which we @@ -7104,6 +7117,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf struct sched_entity *se; struct task_struct *p; int new_tasks; + bool repick = false; again: if (!sched_fair_runnable(rq)) @@ -7157,6 +7171,7 @@ again: } while (cfs_rq); p = task_of(se); + trace_android_rvh_replace_next_task_fair(rq, &p, &se, &repick, false); /* * Since we haven't yet done put_prev_entity and if the selected task @@ -7190,6 +7205,10 @@ simple: if (prev) put_prev_task(rq, prev); + trace_android_rvh_replace_next_task_fair(rq, &p, &se, &repick, true); + if (repick) + goto done; + do { se = pick_next_entity(cfs_rq, NULL); set_next_entity(cfs_rq, se); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 8e475521f033..b9a654bba1b7 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -2063,6 +2063,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att pr_info("root domain span: %*pbl (max cpu_capacity = %lu)\n", cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity); } + trace_android_vh_build_sched_domains(has_asym); ret = 0; error: diff --git a/lib/plist.c b/lib/plist.c index 0d86ed7a76ac..4b2334f1b6f4 100644 --- a/lib/plist.c +++ b/lib/plist.c @@ -102,6 +102,7 @@ ins_node: plist_check_head(head); } +EXPORT_SYMBOL_GPL(plist_add); /** * plist_del - Remove a @node from plist. @@ -131,6 +132,7 @@ void plist_del(struct plist_node *node, struct plist_head *head) plist_check_head(head); } +EXPORT_SYMBOL_GPL(plist_del); /** * plist_requeue - Requeue @node at end of same-prio entries. @@ -171,6 +173,7 @@ void plist_requeue(struct plist_node *node, struct plist_head *head) plist_check_head(head); } +EXPORT_SYMBOL_GPL(plist_requeue); #ifdef CONFIG_DEBUG_PLIST #include diff --git a/lib/show_mem.c b/lib/show_mem.c index 1c26c14ffbb9..663e312814cb 100644 --- a/lib/show_mem.c +++ b/lib/show_mem.c @@ -7,7 +7,7 @@ #include #include - +#include void show_mem(unsigned int filter, nodemask_t *nodemask) { pg_data_t *pgdat; @@ -41,4 +41,5 @@ void show_mem(unsigned int filter, nodemask_t *nodemask) #ifdef CONFIG_MEMORY_FAILURE printk("%lu pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages)); #endif + trace_android_vh_show_mem(filter, nodemask); } diff --git a/mm/Makefile b/mm/Makefile index eb0993adb49e..51e855ae8795 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -57,6 +57,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ # Give 'page_alloc' its own module-parameter namespace page-alloc-y := page_alloc.o page-alloc-$(CONFIG_SHUFFLE_PAGE_ALLOCATOR) += shuffle.o +CFLAGS_page_alloc.o += -DDYNAMIC_DEBUG_MODULE obj-y += page-alloc.o obj-y += init-mm.o diff --git a/mm/compaction.c b/mm/compaction.c index dba424447473..34d99d7d1a6a 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -2275,7 +2275,8 @@ compact_zone(struct compact_control *cc, struct capture_control *capc) trace_mm_compaction_begin(start_pfn, cc->migrate_pfn, cc->free_pfn, end_pfn, sync); - migrate_prep_local(); + /* lru_add_drain_all could be expensive with involving other CPUs */ + lru_add_drain(); while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) { int err; diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index b414debbf2ab..c4f1d5f3548d 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1504,6 +1504,7 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages) !IS_ALIGNED(start_pfn | nr_pages, PAGES_PER_SECTION))) return -EINVAL; + lru_cache_disable(); mem_hotplug_begin(); /* @@ -1562,7 +1563,6 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages) } cond_resched(); - lru_add_drain_all(); ret = scan_movable_pages(pfn, end_pfn, &pfn); if (!ret) { @@ -1647,6 +1647,8 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages) memory_notify(MEM_OFFLINE, &arg); remove_pfn_range_from_zone(zone, start_pfn, nr_pages); mem_hotplug_done(); + lru_cache_enable(); + return 0; failed_removal_isolated: @@ -1659,6 +1661,7 @@ failed_removal: reason); /* pushback to free area */ mem_hotplug_done(); + lru_cache_enable(); return ret; } diff --git a/mm/mempolicy.c b/mm/mempolicy.c index c627bac1b374..6bdfc21b54d6 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -1124,12 +1124,10 @@ int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from, const nodemask_t *to, int flags) { int busy = 0; - int err; + int err = 0; nodemask_t tmp; - err = migrate_prep(); - if (err) - return err; + lru_add_drain_all(); mmap_read_lock(mm); @@ -1328,9 +1326,7 @@ static long do_mbind(unsigned long start, unsigned long len, if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) { - err = migrate_prep(); - if (err) - goto mpol_out; + lru_cache_disable(); } { NODEMASK_SCRATCH(scratch); @@ -1378,6 +1374,8 @@ up_out: mmap_write_unlock(mm); mpol_out: mpol_put(new); + if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) + lru_cache_enable(); return err; } diff --git a/mm/migrate.c b/mm/migrate.c index a99747d3f917..2275cf150cc8 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -57,32 +57,6 @@ #include "internal.h" -/* - * migrate_prep() needs to be called before we start compiling a list of pages - * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is - * undesirable, use migrate_prep_local() - */ -int migrate_prep(void) -{ - /* - * Clear the LRU lists so pages can be isolated. - * Note that pages may be moved off the LRU after we have - * drained them. Those pages will fail to migrate like other - * pages that may be busy. - */ - lru_add_drain_all(); - - return 0; -} - -/* Do the necessary work of migrate_prep but not if it involves other CPUs */ -int migrate_prep_local(void) -{ - lru_add_drain(); - - return 0; -} - int isolate_movable_page(struct page *page, isolate_mode_t mode) { struct address_space *mapping; @@ -1723,7 +1697,7 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, int start, i; int err = 0, err1; - migrate_prep(); + lru_cache_disable(); for (i = start = 0; i < nr_pages; i++) { const void __user *p; @@ -1792,6 +1766,7 @@ out_flush: if (err >= 0) err = err1; out: + lru_cache_enable(); return err; } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 500e157d651d..43e2257cd28a 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4692,7 +4692,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, int no_progress_loops; unsigned int cpuset_mems_cookie; int reserve_flags; - + unsigned long alloc_start = jiffies; /* * We also sanity check to catch abuse of atomic reserves being used by * callers that are not in atomic context. @@ -4934,6 +4934,7 @@ fail: warn_alloc(gfp_mask, ac->nodemask, "page allocation failure: order:%u", order); got_pg: + trace_android_vh_alloc_pages_slowpath(gfp_mask, order, alloc_start); return page; } @@ -8472,6 +8473,27 @@ static unsigned long pfn_max_align_up(unsigned long pfn) pageblock_nr_pages)); } +#if defined(CONFIG_DYNAMIC_DEBUG) || \ + (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) +/* Usage: See admin-guide/dynamic-debug-howto.rst */ +static void alloc_contig_dump_pages(struct list_head *page_list) +{ + DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, "migrate failure"); + + if (DYNAMIC_DEBUG_BRANCH(descriptor)) { + struct page *page; + + dump_stack(); + list_for_each_entry(page, page_list, lru) + dump_page(page, "migration failure"); + } +} +#else +static inline void alloc_contig_dump_pages(struct list_head *page_list) +{ +} +#endif + /* [start, end) must belong to a single zone. */ static int __alloc_contig_migrate_range(struct compact_control *cc, unsigned long start, unsigned long end) @@ -8490,7 +8512,7 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, if (cc->alloc_contig && cc->mode == MIGRATE_ASYNC) max_tries = 1; - migrate_prep(); + lru_add_drain_all(); while (pfn < end || !list_empty(&cc->migratepages)) { if (fatal_signal_pending(current)) { @@ -8519,6 +8541,7 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE); } if (ret < 0) { + alloc_contig_dump_pages(&cc->migratepages); putback_movable_pages(&cc->migratepages); return ret; } diff --git a/mm/slab_common.c b/mm/slab_common.c index 5a0eed95b2c0..dbe4ef8be9ba 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -27,7 +27,8 @@ #define CREATE_TRACE_POINTS #include - +#undef CREATE_TRACE_POINTS +#include #include "internal.h" #include "slab.h" @@ -933,6 +934,7 @@ static void print_slabinfo_header(struct seq_file *m) seq_puts(m, " : globalstat "); seq_puts(m, " : cpustat "); #endif + trace_android_vh_print_slabinfo_header(m); seq_putc(m, '\n'); } @@ -968,6 +970,7 @@ static void cache_show(struct kmem_cache *s, struct seq_file *m) seq_printf(m, " : slabdata %6lu %6lu %6lu", sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail); slabinfo_show_stats(m, s); + trace_android_vh_cache_show(m, &sinfo, s); seq_putc(m, '\n'); } diff --git a/mm/swap.c b/mm/swap.c index d160a07830a7..75327e54b92b 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -36,6 +36,7 @@ #include #include #include +#include #include "internal.h" @@ -257,6 +258,18 @@ static void pagevec_move_tail(struct pagevec *pvec) __count_vm_events(PGROTATED, pgmoved); } +/* return true if pagevec needs to drain */ +static bool pagevec_add_and_need_flush(struct pagevec *pvec, struct page *page) +{ + bool ret = false; + + if (!pagevec_add(pvec, page) || PageCompound(page) || + lru_cache_disabled()) + ret = true; + + return ret; +} + /* * Writeback is about to end against a page which has been marked for immediate * reclaim. If it still appears to be reclaimable, move it to the tail of the @@ -272,7 +285,7 @@ void rotate_reclaimable_page(struct page *page) get_page(page); local_lock_irqsave(&lru_rotate.lock, flags); pvec = this_cpu_ptr(&lru_rotate.pvec); - if (!pagevec_add(pvec, page) || PageCompound(page)) + if (pagevec_add_and_need_flush(pvec, page)) pagevec_move_tail(pvec); local_unlock_irqrestore(&lru_rotate.lock, flags); } @@ -357,7 +370,7 @@ static void activate_page(struct page *page) local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.activate_page); get_page(page); - if (!pagevec_add(pvec, page) || PageCompound(page)) + if (pagevec_add_and_need_flush(pvec, page)) pagevec_lru_move_fn(pvec, __activate_page, NULL); local_unlock(&lru_pvecs.lock); } @@ -469,7 +482,7 @@ void lru_cache_add(struct page *page) get_page(page); local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_add); - if (!pagevec_add(pvec, page) || PageCompound(page)) + if (pagevec_add_and_need_flush(pvec, page)) __pagevec_lru_add(pvec); local_unlock(&lru_pvecs.lock); } @@ -653,6 +666,7 @@ void lru_add_drain_cpu(int cpu) pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); activate_page_drain(cpu); + invalidate_bh_lru(NULL); } /** @@ -678,7 +692,7 @@ void deactivate_file_page(struct page *page) local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate_file); - if (!pagevec_add(pvec, page) || PageCompound(page)) + if (pagevec_add_and_need_flush(pvec, page)) pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); local_unlock(&lru_pvecs.lock); } @@ -700,7 +714,7 @@ void deactivate_page(struct page *page) local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate); get_page(page); - if (!pagevec_add(pvec, page) || PageCompound(page)) + if (pagevec_add_and_need_flush(pvec, page)) pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); local_unlock(&lru_pvecs.lock); } @@ -722,7 +736,7 @@ void mark_page_lazyfree(struct page *page) local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_lazyfree); get_page(page); - if (!pagevec_add(pvec, page) || PageCompound(page)) + if (pagevec_add_and_need_flush(pvec, page)) pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); local_unlock(&lru_pvecs.lock); } @@ -759,7 +773,7 @@ static void lru_add_drain_per_cpu(struct work_struct *dummy) * Calling this function with cpu hotplug locks held can actually lead * to obscure indirect dependencies via WQ context. */ -void lru_add_drain_all(void) +static void __lru_add_drain_all(bool force_all_cpus) { /* * lru_drain_gen - Global pages generation number @@ -804,7 +818,7 @@ void lru_add_drain_all(void) * (C) Exit the draining operation if a newer generation, from another * lru_add_drain_all(), was already scheduled for draining. Check (A). */ - if (unlikely(this_gen != lru_drain_gen)) + if (unlikely(this_gen != lru_drain_gen && !force_all_cpus)) goto done; /* @@ -834,12 +848,14 @@ void lru_add_drain_all(void) for_each_online_cpu(cpu) { struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); - if (pagevec_count(&per_cpu(lru_pvecs.lru_add, cpu)) || + if (force_all_cpus || + pagevec_count(&per_cpu(lru_pvecs.lru_add, cpu)) || data_race(pagevec_count(&per_cpu(lru_rotate.pvec, cpu))) || pagevec_count(&per_cpu(lru_pvecs.lru_deactivate_file, cpu)) || pagevec_count(&per_cpu(lru_pvecs.lru_deactivate, cpu)) || pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree, cpu)) || - need_activate_page_drain(cpu)) { + need_activate_page_drain(cpu) || + has_bh_in_lru(cpu, NULL)) { INIT_WORK(work, lru_add_drain_per_cpu); queue_work_on(cpu, mm_percpu_wq, work); __cpumask_set_cpu(cpu, &has_work); @@ -852,6 +868,11 @@ void lru_add_drain_all(void) done: mutex_unlock(&lock); } + +void lru_add_drain_all(void) +{ + __lru_add_drain_all(false); +} #else void lru_add_drain_all(void) { @@ -859,6 +880,44 @@ void lru_add_drain_all(void) } #endif /* CONFIG_SMP */ +static atomic_t lru_disable_count = ATOMIC_INIT(0); + +bool lru_cache_disabled(void) +{ + return atomic_read(&lru_disable_count); +} + +void lru_cache_enable(void) +{ + atomic_dec(&lru_disable_count); +} + +/* + * lru_cache_disable() needs to be called before we start compiling + * a list of pages to be migrated using isolate_lru_page(). + * It drains pages on LRU cache and then disable on all cpus until + * lru_cache_enable is called. + * + * Must be paired with a call to lru_cache_enable(). + */ +void lru_cache_disable(void) +{ + atomic_inc(&lru_disable_count); +#ifdef CONFIG_SMP + /* + * lru_add_drain_all in the force mode will schedule draining on + * all online CPUs so any calls of lru_cache_disabled wrapped by + * local_lock or preemption disabled would be ordered by that. + * The atomic operation doesn't need to have stronger ordering + * requirements because that is enforeced by the scheduling + * guarantees. + */ + __lru_add_drain_all(true); +#else + lru_add_drain(); +#endif +} + /** * release_pages - batched put_page() * @pages: array of pages to release diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 3c05827608b6..a6719fd24d20 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -44,6 +44,9 @@ #define avc_cache_stats_incr(field) do {} while (0) #endif +#undef CREATE_TRACE_POINTS +#include + struct avc_entry { u32 ssid; u32 tsid; @@ -440,6 +443,7 @@ static void avc_node_free(struct rcu_head *rhead) static void avc_node_delete(struct selinux_avc *avc, struct avc_node *node) { + trace_android_vh_selinux_avc_node_delete(node); hlist_del_rcu(&node->list); call_rcu(&node->rhead, avc_node_free); atomic_dec(&avc->avc_cache.active_nodes); @@ -456,6 +460,7 @@ static void avc_node_kill(struct selinux_avc *avc, struct avc_node *node) static void avc_node_replace(struct selinux_avc *avc, struct avc_node *new, struct avc_node *old) { + trace_android_vh_selinux_avc_node_replace(old, new); hlist_replace_rcu(&old->list, &new->list); call_rcu(&old->rhead, avc_node_free); atomic_dec(&avc->avc_cache.active_nodes); @@ -564,8 +569,10 @@ static struct avc_node *avc_lookup(struct selinux_avc *avc, avc_cache_stats_incr(lookups); node = avc_search_node(avc, ssid, tsid, tclass); - if (node) + if (node) { + trace_android_vh_selinux_avc_lookup(node, ssid, tsid, tclass); return node; + } avc_cache_stats_incr(misses); return NULL; @@ -649,6 +656,7 @@ static struct avc_node *avc_insert(struct selinux_avc *avc, } } hlist_add_head_rcu(&node->list, head); + trace_android_vh_selinux_avc_insert(node); found: spin_unlock_irqrestore(lock, flag); return node; diff --git a/sound/usb/card.h b/sound/usb/card.h index ce4f213d5cf5..e3a59455f71c 100644 --- a/sound/usb/card.h +++ b/sound/usb/card.h @@ -2,6 +2,8 @@ #ifndef __USBAUDIO_CARD_H #define __USBAUDIO_CARD_H +#include + #define MAX_NR_RATES 1024 #define MAX_PACKS 6 /* per URB */ #define MAX_PACKS_HS (MAX_PACKS * 8) /* in high speed mode */ @@ -112,6 +114,11 @@ struct snd_usb_endpoint { spinlock_t lock; struct list_head list; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; struct media_ctl;