diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index 8a2e377b0dde..1a9b6ac22c38 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -648,3 +648,21 @@ Description: SPURR ticks for cpuX when it was idle. This sysfs interface exposes the number of SPURR ticks for cpuX when it was idle. + +What: /sys/devices/system/cpu/cpuX/mte_tcf_preferred +Date: July 2021 +Contact: Linux ARM Kernel Mailing list +Description: Preferred MTE tag checking mode + + When a user program specifies more than one MTE tag checking + mode, this sysfs node is used to specify which mode should + be preferred when running on that CPU. Possible values: + + ================ ============================================== + "sync" Prefer synchronous mode + "async" Prefer asynchronous mode + ================ ============================================== + + Changes to this sysfs node may not take effect immediately. + + See also: Documentation/arm64/memory-tagging-extension.rst diff --git a/Documentation/ABI/testing/sysfs-kernel-dmabuf-buffers b/Documentation/ABI/testing/sysfs-kernel-dmabuf-buffers index 6f7c65209f07..122185c6a88c 100644 --- a/Documentation/ABI/testing/sysfs-kernel-dmabuf-buffers +++ b/Documentation/ABI/testing/sysfs-kernel-dmabuf-buffers @@ -22,31 +22,3 @@ KernelVersion: v5.12 Contact: Hridya Valsaraju Description: This file is read-only and specifies the size of the DMA-BUF in bytes. - -What: /sys/kernel/dmabuf/buffers//attachments -Date: January 2021 -KernelVersion: v5.12 -Contact: Hridya Valsaraju -Description: This directory will contain subdirectories representing every - attachment of the DMA-BUF. - -What: /sys/kernel/dmabuf/buffers//attachments/ -Date: January 2021 -KernelVersion: v5.12 -Contact: Hridya Valsaraju -Description: This directory will contain information on the attaching device - and the number of current distinct device mappings. - -What: /sys/kernel/dmabuf/buffers//attachments//device -Date: January 2021 -KernelVersion: v5.12 -Contact: Hridya Valsaraju -Description: This file is read-only and is a symlink to the attaching devices's - sysfs entry. - -What: /sys/kernel/dmabuf/buffers//attachments//map_counter -Date: January 2021 -KernelVersion: v5.12 -Contact: Hridya Valsaraju -Description: This file is read-only and contains a map_counter indicating the - number of distinct device mappings of the attachment. diff --git a/Documentation/arm64/memory-tagging-extension.rst b/Documentation/arm64/memory-tagging-extension.rst index b540178a93f8..7b99c8f428eb 100644 --- a/Documentation/arm64/memory-tagging-extension.rst +++ b/Documentation/arm64/memory-tagging-extension.rst @@ -77,14 +77,20 @@ configurable behaviours: address is unknown). The user can select the above modes, per thread, using the -``prctl(PR_SET_TAGGED_ADDR_CTRL, flags, 0, 0, 0)`` system call where -``flags`` contain one of the following values in the ``PR_MTE_TCF_MASK`` +``prctl(PR_SET_TAGGED_ADDR_CTRL, flags, 0, 0, 0)`` system call where ``flags`` +contains any number of the following values in the ``PR_MTE_TCF_MASK`` bit-field: -- ``PR_MTE_TCF_NONE`` - *Ignore* tag check faults +- ``PR_MTE_TCF_NONE``  - *Ignore* tag check faults + (ignored if combined with other options) - ``PR_MTE_TCF_SYNC`` - *Synchronous* tag check fault mode - ``PR_MTE_TCF_ASYNC`` - *Asynchronous* tag check fault mode +If no modes are specified, tag check faults are ignored. If a single +mode is specified, the program will run in that mode. If multiple +modes are specified, the mode is selected as described in the "Per-CPU +preferred tag checking modes" section below. + The current tag check fault mode can be read using the ``prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)`` system call. @@ -120,13 +126,39 @@ in the ``PR_MTE_TAG_MASK`` bit-field. interface provides an include mask. An include mask of ``0`` (exclusion mask ``0xffff``) results in the CPU always generating tag ``0``. +Per-CPU preferred tag checking mode +----------------------------------- + +On some CPUs the performance of MTE in stricter tag checking modes +is similar to that of less strict tag checking modes. This makes it +worthwhile to enable stricter checks on those CPUs when a less strict +checking mode is requested, in order to gain the error detection +benefits of the stricter checks without the performance downsides. To +support this scenario, a privileged user may configure a stricter +tag checking mode as the CPU's preferred tag checking mode. + +The preferred tag checking mode for each CPU is controlled by +``/sys/devices/system/cpu/cpu/mte_tcf_preferred``, to which a +privileged user may write the value ``async`` or ``sync``. The default +preferred mode for each CPU is ``async``. + +To allow a program to potentially run in the CPU's preferred tag +checking mode, the user program may set multiple tag check fault mode +bits in the ``flags`` argument to the ``prctl(PR_SET_TAGGED_ADDR_CTRL, +flags, 0, 0, 0)`` system call. If the CPU's preferred tag checking +mode is in the task's set of provided tag checking modes (this will +always be the case at present because the kernel only supports two +tag checking modes, but future kernels may support more modes), that +mode will be selected. Otherwise, one of the modes in the task's mode +set will be selected in a currently unspecified manner. + Initial process state --------------------- On ``execve()``, the new process has the following configuration: - ``PR_TAGGED_ADDR_ENABLE`` set to 0 (disabled) -- Tag checking mode set to ``PR_MTE_TCF_NONE`` +- No tag checking modes are selected (tag check faults ignored) - ``PR_MTE_TAG_MASK`` set to 0 (all tags excluded) - ``PSTATE.TCO`` set to 0 - ``PROT_MTE`` not set on any of the initial memory maps @@ -251,11 +283,13 @@ Example of correct usage return EXIT_FAILURE; /* - * Enable the tagged address ABI, synchronous MTE tag check faults and - * allow all non-zero tags in the randomly generated set. + * Enable the tagged address ABI, synchronous or asynchronous MTE + * tag check faults (based on per-CPU preference) and allow all + * non-zero tags in the randomly generated set. */ if (prctl(PR_SET_TAGGED_ADDR_CTRL, - PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT), + PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC | + (0xfffe << PR_MTE_TAG_SHIFT), 0, 0, 0)) { perror("prctl() failed"); return EXIT_FAILURE; diff --git a/Documentation/arm64/pointer-authentication.rst b/Documentation/arm64/pointer-authentication.rst index 30b2ab06526b..f127666ea3a8 100644 --- a/Documentation/arm64/pointer-authentication.rst +++ b/Documentation/arm64/pointer-authentication.rst @@ -107,3 +107,37 @@ filter out the Pointer Authentication system key registers from KVM_GET/SET_REG_* ioctls and mask those features from cpufeature ID register. Any attempt to use the Pointer Authentication instructions will result in an UNDEFINED exception being injected into the guest. + + +Enabling and disabling keys +--------------------------- + +The prctl PR_PAC_SET_ENABLED_KEYS allows the user program to control which +PAC keys are enabled in a particular task. It takes two arguments, the +first being a bitmask of PR_PAC_APIAKEY, PR_PAC_APIBKEY, PR_PAC_APDAKEY +and PR_PAC_APDBKEY specifying which keys shall be affected by this prctl, +and the second being a bitmask of the same bits specifying whether the key +should be enabled or disabled. For example:: + + prctl(PR_PAC_SET_ENABLED_KEYS, + PR_PAC_APIAKEY | PR_PAC_APIBKEY | PR_PAC_APDAKEY | PR_PAC_APDBKEY, + PR_PAC_APIBKEY, 0, 0); + +disables all keys except the IB key. + +The main reason why this is useful is to enable a userspace ABI that uses PAC +instructions to sign and authenticate function pointers and other pointers +exposed outside of the function, while still allowing binaries conforming to +the ABI to interoperate with legacy binaries that do not sign or authenticate +pointers. + +The idea is that a dynamic loader or early startup code would issue this +prctl very early after establishing that a process may load legacy binaries, +but before executing any PAC instructions. + +For compatibility with previous kernel versions, processes start up with IA, +IB, DA and DB enabled, and are reset to this state on exec(). Processes created +via fork() and clone() inherit the key enabled state from the calling process. + +It is recommended to avoid disabling the IA key, as this has higher performance +overhead than disabling any of the other keys. diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml index 70a60d06019b..ea1e52bc57cd 100755 --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -1,16 +1,16 @@ - - - - + + + + - - + + - + @@ -26,62 +26,62 @@ - - - - + + + + - - - - + + + + - - - + + + - - - + + + - - - - - - + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + @@ -91,25 +91,25 @@ - + - + - + - - + + - + - - - - - + + + + + @@ -118,8 +118,8 @@ - - + + @@ -127,204 +127,204 @@ - + - - + + - - + + - - - - - - + + + + + + - - - + + + - - - - - - + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - + + + + + - - - + + + - + - - - - - - - - + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - + - - + + - + - - - + + + - + - - - - - + + + + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - - - - - + + + + + + - + - - - + + + - - + + - + - - - - + + + + - + - - + + - - - - + + + + - + @@ -333,32 +333,32 @@ - + - + - - + + - + - + - - - - + + + + - - + + @@ -373,16 +373,18 @@ - - + + - + - + - + + + @@ -402,15 +404,15 @@ - + - + - - + + - + @@ -419,40 +421,40 @@ - - + + - - - - - + + + + + - + - - - - + + + + - - - - - + + + + + - + - - + + - - - - + + + + @@ -461,18 +463,18 @@ - - + + - - - - - - - - - + + + + + + + + + @@ -501,23 +503,23 @@ - - - - - + + + + + - + - - + + @@ -526,29 +528,29 @@ - + - - - - + + + + - - - - + + + + - + - - - - + + + + - + - - + + @@ -561,46 +563,46 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + @@ -613,259 +615,259 @@ - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - - - - + + + + - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - + + + - - - - - - - - - + + + + + + + + + - + - - - - - - - - + + + + + + + + - - - - - - + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - + - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - + - - + + - + - + - - - - - + + + + + - - + + - - - - - - - - - + + + + + + + + + - - - - - - - + + + + + + + - + - + @@ -880,496 +882,498 @@ - - - - - - - - + + + + + + + + - - - - + + + + - + - - + + - - + + - - - - + + + + - - - + + + - + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1387,59 +1391,61 @@ - - - - - - - - - + + + + + + + + + - + - - - - - + + + + + - + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + + @@ -1453,196 +1459,197 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - + - - - - + + + + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -1651,341 +1658,342 @@ - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - + + + + - - - - + + + + - + - - - - - + + + + + - - + + - + - - - - + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - + - + - - - - - - - - - - + + + + + + + + + + - - - - - - + + + + + + - - - - - - - - + + + + + + + + - - - - - - + + + + + + - - + + - - - - - + + + + + - - - - - - - + + + + + + + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - - - - - - + + + + + + - + - + - - + + - + - + - + - + - - - - - - - - - - - + + + + + + + + + + + @@ -2002,53 +2010,53 @@ - - - + + + - - - - - - - - - - + + + + + + + + + + - - - - + + + + - - - - + + + + - - + + - + - - - - - - + + + + + + - - - + + + @@ -2056,68 +2064,68 @@ - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - - - - + + + + + - + - - - - - + + + + + @@ -2126,81 +2134,95 @@ - - - - - - + + + + + + - - + + - - + + - - + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + @@ -2214,239 +2236,239 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - - - - - - - - - - + + + + + + + + + + - + - + - - + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - - + + + + - + - + - - - + + + - - - - - + + + + + - - + + - + - + - - - - + + + + - - + + - - + + - - - - + + + + - - - - - - + + + + + + @@ -2454,26 +2476,26 @@ - - - - - - - - - - + + + + + + + + + + - - - + + + - + @@ -2493,27 +2515,27 @@ - - - - - - + + + + + + - - - - - + + + + + - - + + - - - - + + + + @@ -2532,34 +2554,36 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - + + + + + @@ -2568,32 +2592,32 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -2621,105 +2645,105 @@ - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + @@ -2729,63 +2753,63 @@ - + - - - - - - - - - + + + + + + + + + - + - - + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - + - + - - + + @@ -2795,451 +2819,451 @@ - - - - + + + + - + - - - - + + + + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - + - - + + - + @@ -3250,53 +3274,53 @@ - - - - + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - + + + + + + + - - - + + + @@ -3335,154 +3359,154 @@ - + - - - - - - + + + + + + - + - - - - - - - + + + + + + + - + - + - + - + - - + + - - + + - + - - + + - - - - - + + + + + - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + - - + + - + - - - - + + + + - + - + - - + + @@ -3492,81 +3516,81 @@ - - - - - + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - + - - + + - + - + - + @@ -3575,172 +3599,172 @@ - + - - - - - - - - - - - + + + + + + + + + + + - - - - + + + + - - - - - + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - + + + + - + - - - + + + - + - + @@ -3749,312 +3773,312 @@ - - - - - - + + + + + + - - - - - + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - + + + + + - - - - - - + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - + @@ -4069,7 +4093,7 @@ - + @@ -4091,14 +4115,14 @@ - - - - - + + + + + - - + + @@ -4106,47 +4130,47 @@ - - - + + + - + - + - - - - - - - - + + + + + + + + - - - - - - - - - - + + + + + + + + + + - + - - - + + + - + @@ -4157,22 +4181,22 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -4181,592 +4205,591 @@ - - - + + + - - - - - - + + + + + + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - + + + + + - + - + - - - - + + + + - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - - + + - - + + - - + + - - - - - - - - - - - + + + + + + + + + + + - + - - - - - - + + + + + + - - - - - + + + + + - - + + - - - + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - - - - - + + + + + + + - - - - - + + + + + - + - - + + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - + + - - + + - + - - - - - - + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + - + - + - + - - - + + + - - + + - - + + - - - + + + @@ -4781,82 +4804,82 @@ - - - - - - - - - - - + + + + + + + + + + + - + - + - - - + + + - + - - - - - - + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - + + + + - - - - - - - - - + + + + + + + + + - - + + - - + + @@ -5018,6 +5041,8 @@ + + @@ -5080,22 +5105,22 @@ - + - - + + - - - - - - + + + + + + - + @@ -5103,11 +5128,11 @@ - + - - - + + + @@ -5118,7 +5143,7 @@ - + @@ -5127,67 +5152,67 @@ - + - + - - - - + + + + - + - - - + + + - - + + - + - - - + + + - + - + - - + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -5195,20 +5220,20 @@ - + - + - - + + - - - + + + @@ -5227,17 +5252,17 @@ - - - - + + + + - + - - - + + + @@ -5275,20 +5300,28 @@ - + + + + + + + + - + - + - + - + + @@ -5309,39 +5342,42 @@ - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + + + + @@ -5366,78 +5402,81 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - + + - + - - - - - - - - + + - + + + + + + + + + + + + + + + + + + + @@ -5544,21 +5583,21 @@ - + - + - + - + - + - + @@ -5687,14 +5726,14 @@ - + - + - + @@ -5708,20 +5747,20 @@ - + - + - + - + - + - + @@ -5860,7 +5899,7 @@ - + @@ -5926,6 +5965,7 @@ + @@ -5984,7 +6024,7 @@ - + @@ -6008,30 +6048,33 @@ - + - + - + - + - + - + - + - + - + + + + @@ -6315,6 +6358,7 @@ + @@ -6322,7 +6366,7 @@ - + @@ -6334,36 +6378,39 @@ - + - + - + - + - + - + - + - + - + - + + + + - + @@ -6388,10 +6435,10 @@ - + - + @@ -6409,15 +6456,15 @@ - + - + - + - + @@ -6442,6 +6489,14 @@ + + + + + + + + @@ -6454,7 +6509,7 @@ - + @@ -6564,77 +6619,100 @@ + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -7087,26 +7165,26 @@ - + - + - + - + - + - + - + @@ -7152,7 +7230,7 @@ - + @@ -7161,7 +7239,7 @@ - + @@ -7213,40 +7291,43 @@ - + - + - + - + - + - + - + - + - + - + - - + + - - + + + + + @@ -7256,7 +7337,24 @@ - + + + + + + + + + + + + + + + + + + @@ -7266,112 +7364,112 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -7430,6 +7528,7 @@ + @@ -7530,7 +7629,7 @@ - + @@ -7617,12 +7716,12 @@ - + - + - + @@ -7862,39 +7961,39 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -7913,7 +8012,7 @@ - + @@ -7951,14 +8050,14 @@ - + - + @@ -7972,7 +8071,7 @@ - + @@ -8083,12 +8182,12 @@ - + - + - + @@ -8109,12 +8208,12 @@ - + - + - + @@ -8130,24 +8229,27 @@ - + - + - + - + - + - + - + + + + @@ -8243,13 +8345,13 @@ - + - + - + @@ -8336,171 +8438,174 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -8513,12 +8618,13 @@ + - + @@ -8558,10 +8664,10 @@ - + - + @@ -8578,7 +8684,7 @@ - + @@ -8643,33 +8749,36 @@ - + - + - + - + - + - + - + - + - + - + + + + @@ -8805,6 +8914,7 @@ + @@ -9061,48 +9171,51 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -9266,58 +9379,58 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -9326,7 +9439,7 @@ - + @@ -9429,7 +9542,7 @@ - + @@ -9438,21 +9551,24 @@ - + - + - - + + - - + + - - + + - - + + + + + @@ -9535,180 +9651,192 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -9766,7 +9894,7 @@ - + @@ -9918,48 +10046,51 @@ - + - + - + - + - - - - + - + - - - - - - - - + + - + - - + + - - + + - - + + - - + + + + + + + + + + + + + + @@ -10060,7 +10191,7 @@ - + @@ -10205,6 +10336,23 @@ + + + + + + + + + + + + + + + + + @@ -10246,54 +10394,66 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -10687,6 +10847,14 @@ + + + + + + + + @@ -10708,30 +10876,30 @@ - + - + - + - + - + - + - + - + - + @@ -10827,7 +10995,158 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -10898,7 +11217,7 @@ - + @@ -10926,19 +11245,19 @@ - + - + - + - + - + @@ -11031,18 +11350,18 @@ - + - + - + - + - + @@ -11080,7 +11399,7 @@ - + @@ -11099,47 +11418,50 @@ - + - + - + - + - + - + - - + + - - + + - - - - - - - - + + - + - - + + - - + + + + + + + + + + + @@ -11200,90 +11522,93 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -11323,7 +11648,7 @@ - + @@ -11607,42 +11932,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -11921,66 +12249,66 @@ - + - + - + - + - + - - + + - - + + - - + + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -12012,48 +12340,51 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -12121,24 +12452,24 @@ - + - + - + - + - + - + - + @@ -12260,6 +12591,18 @@ + + + + + + + + + + + + @@ -12275,42 +12618,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -12429,21 +12772,21 @@ - + - + - + - + - + - + @@ -12469,18 +12812,18 @@ - + - + - + - + - + @@ -12729,60 +13072,60 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -12902,132 +13245,138 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -13127,51 +13476,51 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -13207,18 +13556,21 @@ - + - + - + - + - + + + + @@ -13268,45 +13620,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -13316,28 +13668,31 @@ - + - + - + - + - + - + + + + @@ -13400,18 +13755,18 @@ - + - + - + - + - + @@ -13442,27 +13797,30 @@ - + - + - + - + - + - + - + - + + + + @@ -13699,23 +14057,31 @@ + + + + + + + + - + - + - + - + - + @@ -13743,12 +14109,18 @@ - + - + - + + + + + + + @@ -13915,31 +14287,34 @@ - + - + - + - + - + - + - + - + + + + @@ -14069,6 +14444,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -14159,180 +14563,192 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -14439,7 +14855,7 @@ - + @@ -14491,16 +14907,16 @@ - + - + - + - + @@ -14567,6 +14983,7 @@ + @@ -14585,51 +15002,54 @@ - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - + - + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -14713,7 +15133,7 @@ - + @@ -14774,6 +15194,18 @@ + + + + + + + + + + + + @@ -15144,6 +15576,7 @@ + @@ -15461,39 +15894,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -15541,21 +15977,24 @@ - + - + - + - + - + - + + + + @@ -15592,6 +16031,12 @@ + + + + + + @@ -15735,7 +16180,7 @@ - + @@ -15745,13 +16190,13 @@ - + - + - + @@ -15860,53 +16305,56 @@ - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - + - + - + @@ -15919,36 +16367,36 @@ - + - + - + - + - + - + - + - + - + - + @@ -15981,7 +16429,7 @@ - + @@ -16106,36 +16554,39 @@ - + - + - - - - - - - - - - - - - - - - - - - + - + + + + - + + + + + + + + + + + + + + + + + + + @@ -16176,15 +16627,15 @@ - + - + - + - + @@ -16251,11 +16702,11 @@ - + - + @@ -16266,15 +16717,18 @@ - + - + - + - + + + + @@ -16291,9 +16745,11 @@ + + @@ -16344,138 +16800,141 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -16539,12 +16998,12 @@ - + - + - + @@ -16558,12 +17017,12 @@ - + - + - + @@ -16584,12 +17043,12 @@ - + - + - + @@ -16627,7 +17086,7 @@ - + @@ -16637,17 +17096,17 @@ - + - + - + - + @@ -16712,9 +17171,9 @@ - + - + @@ -16891,39 +17350,50 @@ - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + @@ -17127,23 +17597,23 @@ - + - + - + - + - + - + @@ -17283,9 +17753,9 @@ - + - + @@ -17320,66 +17790,75 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + @@ -17450,7 +17929,7 @@ - + @@ -17685,7 +18164,11 @@ - + + + + + @@ -17716,7 +18199,7 @@ - + @@ -17735,31 +18218,31 @@ - + - + - + - + - + - + - + - + - + @@ -17819,7 +18302,7 @@ - + @@ -17881,234 +18364,234 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -18361,13 +18844,14 @@ + - + @@ -18549,14 +19033,6 @@ - - - - - - - - @@ -18720,7 +19196,7 @@ - + @@ -18783,45 +19259,51 @@ - + - + - - - - - - - - - - - - - + - + - + - + - + - + - + + + + + + + - + + + + + + + + + + + + + @@ -18865,15 +19347,15 @@ - + - + - + - - + + @@ -18930,63 +19412,66 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + @@ -19009,7 +19494,7 @@ - + @@ -19028,7 +19513,7 @@ - + @@ -19048,6 +19533,9 @@ + + + @@ -19130,9 +19618,9 @@ - + - + @@ -19313,21 +19801,24 @@ - + - + - + - + - + - + + + + @@ -19408,21 +19899,24 @@ - + - + - + - + - + - + + + + @@ -19759,195 +20253,208 @@ - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - - + + + + + + + + + + + + + + @@ -20009,7 +20516,7 @@ - + @@ -20049,6 +20556,9 @@ + + + @@ -20109,7 +20619,7 @@ - + @@ -20142,28 +20652,28 @@ - + - + - + - + - + - + - + @@ -20173,46 +20683,46 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -20229,12 +20739,12 @@ - + - + - + @@ -20578,12 +21088,12 @@ - + - + - + @@ -20626,18 +21136,18 @@ - + - + - + - + - + @@ -20868,18 +21378,18 @@ - + - + - + - + - + @@ -20973,7 +21483,7 @@ - + @@ -21215,13 +21725,13 @@ - + - + - + @@ -21344,30 +21854,30 @@ - + - + - + - + - + - + - + - + - + @@ -21401,150 +21911,162 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + @@ -21564,6 +22086,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -21670,81 +22227,93 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -21759,32 +22328,32 @@ - + - + - + - + - + - + - + - + - + @@ -21849,6 +22418,20 @@ + + + + + + + + + + + + + + @@ -21859,15 +22442,15 @@ - + - + - + - + @@ -21913,18 +22496,18 @@ - + - + - + - + - + @@ -22094,99 +22677,105 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - + - - + + @@ -22203,58 +22792,58 @@ - - + + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - + - + - - + + - - + + + + + + + + @@ -22313,51 +22902,54 @@ - + - + - + - + - + - + - + - + - - - - - + + - + - + - - + + - - + + - - + + + + + - + + + + @@ -22383,6 +22975,9 @@ + + + @@ -22460,15 +23055,15 @@ - + - + - + - + @@ -22545,6 +23140,7 @@ + @@ -22711,7 +23307,7 @@ - + @@ -22763,44 +23359,44 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -22833,11 +23429,17 @@ - + - - + + + + + + + + @@ -22976,11 +23578,11 @@ - + - + @@ -23028,45 +23630,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -23172,7 +23777,7 @@ - + @@ -23215,99 +23820,99 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -23508,7 +24113,7 @@ - + @@ -23621,21 +24226,21 @@ - + - + - + - + - + - + @@ -23783,25 +24388,27 @@ - + + + + + + + + + - + - + - - - - - - - + @@ -23880,6 +24487,11 @@ + + + + + @@ -23900,22 +24512,23 @@ + - + - + - + - + - + - + @@ -23936,15 +24549,15 @@ - + - + - + - + @@ -24035,18 +24648,18 @@ - + - + - + - + - + @@ -24131,6 +24744,10 @@ + + + + @@ -24144,6 +24761,7 @@ + @@ -24176,39 +24794,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -24234,72 +24855,84 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -24433,26 +25066,25 @@ - + - + - + - - + - + - + @@ -24501,28 +25133,28 @@ - + - + - + - + - + - + - - + + - + @@ -24547,22 +25179,22 @@ - + - + - + - + - + - + @@ -24601,61 +25233,61 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -24708,9 +25340,6 @@ - - - @@ -24785,15 +25414,15 @@ - + - + - + - + @@ -24836,37 +25465,37 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -24908,34 +25537,34 @@ - + - + - + - + - + - + - + - + - + - + @@ -24947,43 +25576,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -25478,18 +26107,18 @@ - + - + - + - + - + @@ -25520,39 +26149,42 @@ - + - + - + - + - - - - - - - - - - - - - - - - - - - - + + - + + + + + + + + + + + + + + + + + + + + + + @@ -25600,32 +26232,32 @@ - + - + - + - + - + - + - + - + - + @@ -25663,7 +26295,7 @@ - + @@ -25688,7 +26320,7 @@ - + @@ -25782,15 +26414,15 @@ - + - + - + - + @@ -25866,6 +26498,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -25949,7 +26619,7 @@ - + @@ -26183,149 +26853,155 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - + + - - - - + - + - - + + - - + + - - - - - - - - - - - - - - + + - + + + + + + + - - - - + - + + + + - + + + + + + + - + - + + + + - + - - + + - - + + + + + + + + + + + - + - + - + - + - + @@ -26367,93 +27043,96 @@ - + - + - + - + - + - + - + - - - - + - - - - + - + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - + + - + + + + + + + + + + + + + + + + + + + - + - - + + - - + + - - + + - - + + - - + + + + + + + + @@ -26578,6 +27257,7 @@ + @@ -26774,75 +27454,78 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -26874,12 +27557,12 @@ - + - + - + @@ -26947,12 +27630,12 @@ - + - + - + @@ -26984,15 +27667,15 @@ - + - + - + - + @@ -27019,6 +27702,7 @@ + @@ -27174,116 +27858,122 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + @@ -27302,96 +27992,102 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -27421,7 +28117,7 @@ - + @@ -27557,27 +28253,27 @@ - + - + - + - + - + - + - + - + @@ -27623,62 +28319,62 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -27700,94 +28396,94 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -27885,7 +28581,7 @@ - + @@ -27978,24 +28674,24 @@ - + - + - + - + - + - + - + @@ -28046,6 +28742,17 @@ + + + + + + + + + + + @@ -28203,7 +28910,7 @@ - + @@ -28213,19 +28920,19 @@ - + - + - + - + - + @@ -28473,6 +29180,7 @@ + @@ -28602,7 +29310,7 @@ - + @@ -28657,21 +29365,24 @@ + + + - + - + - + - + @@ -28719,7 +29430,7 @@ - + @@ -28870,15 +29581,15 @@ - + - + - + - + @@ -28974,7 +29685,7 @@ - + @@ -29113,7 +29824,7 @@ - + @@ -29231,177 +29942,180 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -29518,7 +30232,7 @@ - + @@ -29579,24 +30293,24 @@ - + - + - + - + - + - + - + @@ -29699,7 +30413,7 @@ - + @@ -29772,6 +30486,14 @@ + + + + + + + + @@ -29862,30 +30584,33 @@ - + - + - + - + - + - + - + - + - + + + + @@ -29948,69 +30673,72 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -30040,65 +30768,65 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -30272,44 +31000,44 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -30321,10 +31049,10 @@ - + - + @@ -30357,9 +31085,9 @@ - + - + @@ -30414,45 +31142,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -30498,12 +31229,12 @@ - + - + - + @@ -30606,9 +31337,9 @@ - + - + @@ -30957,22 +31688,22 @@ - - + + - + - + - + - + - + @@ -31162,12 +31893,12 @@ - + - + - + @@ -31201,18 +31932,18 @@ - + - + - + - + - + @@ -31266,6 +31997,10 @@ + + + + @@ -31340,7 +32075,7 @@ - + @@ -31381,6 +32116,7 @@ + @@ -31429,7 +32165,7 @@ - + @@ -31439,147 +32175,153 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - + - + - + + + + - - - - + - + - + - - + + + + + + + + + + + + + + @@ -31857,27 +32599,27 @@ - + - + - + - + - + - + - + - + @@ -31887,27 +32629,33 @@ - + - + - + - + - + - + - + - + + + + + + + @@ -31985,39 +32733,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -32038,7 +32789,7 @@ - + @@ -32224,7 +32975,7 @@ - + @@ -32504,6 +33255,7 @@ + @@ -32544,6 +33296,9 @@ + + + @@ -32605,6 +33360,11 @@ + + + + + @@ -32622,7 +33382,7 @@ - + @@ -32647,55 +33407,55 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -32836,15 +33596,15 @@ - + - + - + - + @@ -32936,7 +33696,7 @@ - + @@ -32982,274 +33742,274 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -33274,7 +34034,7 @@ - + @@ -33401,6 +34161,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -33521,7 +34301,7 @@ - + @@ -33627,7 +34407,7 @@ - + @@ -33640,58 +34420,58 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -33971,17 +34751,17 @@ - + - + - + - + @@ -34135,6 +34915,7 @@ + @@ -34226,46 +35007,46 @@ - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + @@ -34301,12 +35082,12 @@ - + - + - + @@ -34361,12 +35142,12 @@ - + - + - + @@ -34444,7 +35225,7 @@ - + @@ -34464,6 +35245,7 @@ + @@ -34525,7 +35307,7 @@ - + @@ -34637,26 +35419,26 @@ - + - + - + - + - + - + - + @@ -34701,36 +35483,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -34779,47 +35561,50 @@ - - - - + - + - - + + + + + - + - + + + + - + - + - + - + - - + + - + - + - + @@ -35128,123 +35913,123 @@ - + - - - - - - - + - + - + + + + - - - - - - - - - - + - + + + + - + + + + - + - - + + - - + + - - + + - - + + - - + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + @@ -35252,6 +36037,14 @@ + + + + + + + + @@ -35276,15 +36069,15 @@ - + - + - + - + @@ -35325,38 +36118,38 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -35377,15 +36170,15 @@ - + - + - + - + @@ -35402,7 +36195,7 @@ - + @@ -35433,22 +36226,22 @@ - + - + - + - + - + - + @@ -35751,7 +36544,7 @@ - + @@ -35833,15 +36626,15 @@ - + - + - + - + @@ -35899,7 +36692,7 @@ - + @@ -36145,18 +36938,18 @@ - + - + - + - + - + @@ -36176,6 +36969,14 @@ + + + + + + + + @@ -36370,36 +37171,39 @@ - + - + - + - + - + - + - + - + - + - + - + + + + @@ -36454,7 +37258,7 @@ - + @@ -36464,14 +37268,18 @@ - + - + + + + + @@ -36590,7 +37398,7 @@ - + @@ -36675,81 +37483,81 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36758,42 +37566,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36948,126 +37756,132 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - + - + - + + + + + + + + + + + + + @@ -37080,33 +37894,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -37141,6 +37955,14 @@ + + + + + + + + @@ -37162,12 +37984,12 @@ - + - + - + @@ -37328,7 +38150,7 @@ - + @@ -37677,82 +38499,85 @@ - + - + - + - + - + - + - + - + - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -37836,18 +38661,24 @@ - + - + - - + + - - + + - - + + + + + + + + @@ -37921,7 +38752,7 @@ - + @@ -37967,22 +38798,22 @@ - + - + - + - + - + - + @@ -38037,27 +38868,27 @@ - + - + - + - + - + - + - + - + @@ -38108,111 +38939,114 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -38257,21 +39091,21 @@ - + - + - + - + - + - + @@ -38309,7 +39143,7 @@ - + @@ -38469,16 +39303,16 @@ - + - + - + - + @@ -38589,70 +39423,70 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -38697,39 +39531,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -38817,7 +39654,7 @@ - + @@ -38836,7 +39673,7 @@ - + @@ -38870,12 +39707,12 @@ - + - + - + @@ -38890,12 +39727,12 @@ - + - + - + @@ -39060,7 +39897,7 @@ - + @@ -39104,51 +39941,51 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39163,60 +40000,60 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39298,24 +40135,24 @@ - + - + - + - + - + - + - + @@ -39336,15 +40173,15 @@ - + - + - + - + @@ -39413,117 +40250,123 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - + - + - + + + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -39608,69 +40451,81 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -39753,7 +40608,7 @@ - + @@ -39771,7 +40626,7 @@ - + @@ -39880,12 +40735,12 @@ - + - + - + @@ -39943,15 +40798,15 @@ - + - + - + - + @@ -40284,7 +41139,7 @@ - + @@ -40484,45 +41339,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -40700,6 +41558,7 @@ + @@ -40856,9 +41715,9 @@ - + - + @@ -41163,24 +42022,27 @@ - + - + - + - + - + - + - + + + + @@ -41231,6 +42093,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -41399,96 +42305,99 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -41513,6 +42422,7 @@ + @@ -41522,24 +42432,24 @@ - + - + - + - + - + - + - + @@ -41552,21 +42462,21 @@ - + - + - + - + - + - + @@ -41674,7 +42584,7 @@ - + @@ -41742,6 +42652,19 @@ + + + + + + + + + + + + + @@ -41819,12 +42742,12 @@ - + - + - + @@ -42134,7 +43057,7 @@ - + @@ -42159,16 +43082,16 @@ - + - + - + - + @@ -42201,7 +43124,7 @@ - + @@ -42357,18 +43280,18 @@ - + - + - + - + - + @@ -42380,138 +43303,162 @@ - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - + - + + + + - + - + - - - - + - + - + - + + + + - + - + - - - - - - - + - - - - + - + - + + + + + + + - + + + + - + - - - - + - + + + + + + + + + + + + + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -42524,22 +43471,22 @@ - + - + - + - + - + - + @@ -42551,76 +43498,88 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + - + @@ -42672,115 +43631,115 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -42880,21 +43839,24 @@ - + - + - + - + - + - + + + + @@ -43013,7 +43975,7 @@ - + @@ -43073,79 +44035,82 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + + + + - + - + - + @@ -43170,249 +44135,252 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - + - - - - + - + - - - - - + + - + - - + + + + + - + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -43499,6 +44467,9 @@ + + + @@ -43636,9 +44607,9 @@ - + - + @@ -43734,7 +44705,7 @@ - + @@ -43770,15 +44741,15 @@ - + - + - + - + @@ -43819,7 +44790,7 @@ - + @@ -44104,45 +45075,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -44187,23 +45158,37 @@ - + - + - + - + - + + + + + + + + + + + + + + + @@ -44213,33 +45198,39 @@ - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -44364,38 +45355,38 @@ - + - + - + - - + + - - + + - - - - - - - - - - - + + - + + + + + + + + + + - + - + @@ -44566,7 +45557,7 @@ - + @@ -44606,40 +45597,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -44683,7 +45674,7 @@ - + @@ -44770,114 +45761,114 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -44891,17 +45882,17 @@ - + - + - + - + @@ -45050,12 +46041,12 @@ - + - + - + @@ -45127,45 +46118,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -45267,9 +46261,9 @@ - + - + @@ -45423,7 +46417,7 @@ - + @@ -45490,32 +46484,32 @@ - + - + - + - + - + - + - + - + - + @@ -45567,46 +46561,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -45614,15 +46611,15 @@ - + - + - + - + @@ -45746,7 +46743,7 @@ - + @@ -45992,114 +46989,117 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - + @@ -46125,24 +47125,27 @@ - + - + - + - + - + - + - + + + + @@ -46200,24 +47203,24 @@ - + - + - + - + - + - + - + @@ -46268,24 +47271,24 @@ - + - + - + - + - + - + - + @@ -46314,27 +47317,27 @@ - + - + - + - + - + - - + + - - + + - - + + @@ -46397,21 +47400,21 @@ - + - + - + - + - + - + @@ -46471,69 +47474,75 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + + + + @@ -46557,7 +47566,7 @@ - + @@ -46567,16 +47576,16 @@ - + - + - + - + @@ -46601,162 +47610,162 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -47004,7 +48013,7 @@ - + @@ -47130,57 +48139,63 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -47516,72 +48531,75 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + @@ -47739,18 +48757,18 @@ - + - + - + - + - + @@ -47837,18 +48855,18 @@ - + - + - + - + - + @@ -47996,7 +49014,7 @@ - + @@ -48113,7 +49131,7 @@ - + @@ -48135,13 +49153,13 @@ - + - + - + @@ -48252,6 +49270,9 @@ + + + @@ -48310,7 +49331,7 @@ - + @@ -48341,7 +49362,7 @@ - + @@ -48475,20 +49496,23 @@ - + - + - + - + - + + + + @@ -48543,7 +49567,7 @@ - + @@ -48571,25 +49595,25 @@ - + - + - + - + - + - + - + @@ -48685,48 +49709,48 @@ - + - + - + - + - + - + - + - + - + - + - - - - - - - + - + - - + + + + + + + + @@ -48761,21 +49785,24 @@ - + - + - + - + - + - + + + + @@ -48860,59 +49887,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48962,7 +49989,7 @@ - + @@ -49044,31 +50071,31 @@ - + - + - + - + - + - + - + - + - + @@ -49147,9 +50174,9 @@ - + - + @@ -49167,7 +50194,7 @@ - + @@ -49224,6 +50251,11 @@ + + + + + @@ -49307,21 +50339,21 @@ - + - + - + - + - + - + @@ -49351,6 +50383,14 @@ + + + + + + + + @@ -49418,30 +50458,33 @@ - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + + + + @@ -49521,50 +50564,50 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -49577,6 +50620,7 @@ + @@ -49599,12 +50643,12 @@ - + - + - + @@ -49617,12 +50661,12 @@ - + - + - + @@ -49631,188 +50675,188 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -50143,15 +51187,15 @@ - + - + - + - + @@ -50210,6 +51254,7 @@ + @@ -50313,6 +51358,17 @@ + + + + + + + + + + + @@ -50350,42 +51406,48 @@ - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + @@ -50527,15 +51589,15 @@ - + - - - - + - + + + + @@ -50572,159 +51634,171 @@ - + - + - - + + - - + + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - + - + + + + - + + + + - + + + + + + + - - - - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -50769,12 +51843,12 @@ - + - + - + @@ -50810,24 +51884,27 @@ - + - - - - + - + - + - + - + + + + + + + @@ -50980,7 +52057,7 @@ - + @@ -51050,19 +52127,19 @@ - + - + - + - + - + @@ -51137,6 +52214,17 @@ + + + + + + + + + + + @@ -51164,7 +52252,7 @@ - + @@ -51186,16 +52274,16 @@ - + - + - + - + @@ -51237,180 +52325,195 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -51437,72 +52540,75 @@ - + - - - - - - - + - + - + - - - - + - + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - - - + - + + + + + + + + + + + + + + + + @@ -51542,7 +52648,7 @@ - + @@ -51638,96 +52744,96 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - + - - + + - - + + - - + + - - + + @@ -51745,12 +52851,12 @@ - + - + - + @@ -51760,24 +52866,24 @@ - + - + - + - + - + - + - + @@ -51785,13 +52891,13 @@ - + - + - + @@ -51812,30 +52918,30 @@ - + - + - + - + - + - + - + - + - + @@ -51900,7 +53006,7 @@ - + @@ -51934,10 +53040,10 @@ - + - + @@ -52010,7 +53116,7 @@ - + @@ -52056,25 +53162,25 @@ - + - + - + - + - + - + - + @@ -52115,12 +53221,12 @@ - + - + - + @@ -52136,18 +53242,18 @@ - + - + - + - + - + @@ -52200,51 +53306,54 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -52258,39 +53367,39 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -52400,372 +53509,390 @@ - + - - - - - - - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - - - - + + - + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - + - + + + + + + + + + + - - - - - - - + - + - - - - + - - - - + - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - - - - - - - - - - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + - - - - - - - - - - + - + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + + + + - + + + + + + + + + + + + + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + @@ -52799,6 +53926,7 @@ + @@ -52896,7 +54024,7 @@ - + @@ -53023,7 +54151,7 @@ - + @@ -53069,6 +54197,20 @@ + + + + + + + + + + + + + + @@ -53077,99 +54219,102 @@ - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + + + + - + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + - + - - + + + + + + + + + + + + + + + + + @@ -53250,7 +54395,7 @@ - + @@ -53363,18 +54508,18 @@ - + - + - + - + - + @@ -53601,51 +54746,51 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -53668,36 +54813,39 @@ - + - + - + - + - + - + - + - + - + - + - + + + + @@ -53724,234 +54872,255 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + + + + @@ -54060,6 +55229,11 @@ + + + + + @@ -54128,15 +55302,15 @@ - + - + - + - + @@ -54149,132 +55323,138 @@ - + - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + @@ -54297,23 +55477,23 @@ - + - + - + - + - + - + @@ -54652,12 +55832,12 @@ - + - + - + @@ -54730,90 +55910,93 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + @@ -55056,6 +56239,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -55140,12 +56346,12 @@ - + - + - + @@ -55167,84 +56373,96 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -55356,7 +56574,7 @@ - + @@ -55369,37 +56587,37 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -55823,27 +57041,27 @@ - + - + - + - + - + - + - + - + @@ -56061,34 +57279,37 @@ + + + - + - + - + - + - + - + - + @@ -56175,107 +57396,110 @@ - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - + + - + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + @@ -56298,35 +57522,35 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -56348,33 +57572,36 @@ - + - + - + - + - + - + - + - + - + - + + + + @@ -56411,15 +57638,18 @@ - + - + - + - + + + + @@ -56666,7 +57896,7 @@ - + @@ -56762,7 +57992,7 @@ - + @@ -57093,7 +58323,7 @@ - + @@ -57169,7 +58399,7 @@ - + @@ -57177,7 +58407,7 @@ - + @@ -57223,22 +58453,22 @@ - + - + - + - + - + - + @@ -57302,7 +58532,7 @@ - + @@ -57318,79 +58548,79 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -57461,14 +58691,14 @@ - + - + - + - + @@ -58011,12 +59241,12 @@ - + - + - + @@ -58060,6 +59290,20 @@ + + + + + + + + + + + + + + @@ -58081,12 +59325,12 @@ - + - + - + @@ -58180,6 +59424,10 @@ + + + + @@ -58427,24 +59675,24 @@ - + - - - - - - - + - + - + + + + - + + + + @@ -58543,54 +59791,57 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -58655,7 +59906,7 @@ - + @@ -58668,340 +59919,340 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -59161,39 +60412,39 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -59280,117 +60531,123 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -59494,6 +60751,7 @@ + @@ -59632,6 +60890,7 @@ + @@ -59641,15 +60900,15 @@ - + - + - + - + @@ -59775,6 +61034,11 @@ + + + + + @@ -59826,33 +61090,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -60025,7 +61289,7 @@ - + @@ -60036,21 +61300,24 @@ - + - + - - + + - - + + - - + + - - + + + + + @@ -60065,7 +61332,7 @@ - + @@ -60142,33 +61409,36 @@ - + - - - - + - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + @@ -60218,12 +61488,12 @@ - + - + - + @@ -60338,6 +61608,14 @@ + + + + + + + + @@ -60357,69 +61635,72 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -60429,15 +61710,15 @@ - + - + - + - + @@ -60540,7 +61821,7 @@ - + @@ -60668,7 +61949,7 @@ - + @@ -60708,22 +61989,22 @@ - + - + - + - + - + - + @@ -60734,30 +62015,33 @@ - + - + - + - + - + - + - + - + - + + + + @@ -60858,15 +62142,19 @@ - + + - + - - + + - - + + + + + @@ -60968,83 +62256,89 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -61100,6 +62394,7 @@ + @@ -61194,6 +62489,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -61286,18 +62601,18 @@ - + - + - + - + - + @@ -61408,30 +62723,36 @@ - + - + - + - + - + - + - + - + - + + + + + + + @@ -61532,24 +62853,24 @@ - + - + - + - + - + - + - + @@ -61599,6 +62920,7 @@ + @@ -61610,18 +62932,18 @@ - + - + - + - + - + @@ -61635,39 +62957,39 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -61862,24 +63184,24 @@ - + - + - + - + - + - + - + @@ -62039,87 +63361,90 @@ - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + @@ -62278,7 +63603,7 @@ - + @@ -62495,9 +63820,9 @@ - + - + @@ -62811,7 +64136,20 @@ - + + + + + + + + + + + + + + @@ -62840,24 +64178,24 @@ - + - + - + - + - + - + - + @@ -62891,7 +64229,10 @@ - + + + + @@ -62954,12 +64295,12 @@ - + - + - + @@ -63078,72 +64419,78 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -63169,36 +64516,36 @@ - + - + - + - + - + - + - + - + - + - + @@ -63211,75 +64558,78 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -63618,7 +64968,7 @@ - + @@ -63635,18 +64985,18 @@ - + - + - + - + - + @@ -63664,6 +65014,7 @@ + @@ -63842,237 +65193,240 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - + - + - + + + + + + + + + + - + + + + + + + + + + + + + - + + + + + + + + + + + + + - + - - - - - + + - + - - + + - - + + - - + + - - - - - + + - + + + + - + - + - - - - + - + - + + + + + + + + + + + + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + + + + - + + + + + + + + + + + + + + + + + + + - + - + - + - + - - - - + - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -64405,7 +65759,7 @@ - + @@ -64424,36 +65778,36 @@ - + - + - + - + - + - + - + - + - + - + @@ -64475,78 +65829,78 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + @@ -64554,16 +65908,16 @@ - + - + - + - + @@ -64582,6 +65936,9 @@ + + + @@ -64630,6 +65987,14 @@ + + + + + + + + @@ -64731,33 +66096,36 @@ - + - + - + - + - + - + - + - + - + - + + + + @@ -64997,12 +66365,12 @@ - + - + - + @@ -65031,7 +66399,7 @@ - + @@ -65081,36 +66449,36 @@ - + - + - + - + - + - + - + - + - + - + @@ -65210,6 +66578,7 @@ + @@ -65376,32 +66745,32 @@ - + - + - + - + - + - + - + - + @@ -65506,7 +66875,7 @@ - + @@ -65519,28 +66888,28 @@ - + - + - + - + - + - + - + - + @@ -65548,7 +66917,7 @@ - + @@ -65662,39 +67031,39 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -65706,7 +67075,7 @@ - + @@ -65725,308 +67094,311 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -66060,6 +67432,9 @@ + + + @@ -66088,56 +67463,56 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -66164,12 +67539,12 @@ - + - + - + @@ -66332,6 +67707,7 @@ + @@ -66496,39 +67872,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -66540,21 +67919,21 @@ - + - + - + - + - + - + @@ -66598,14 +67977,14 @@ - + - + - + @@ -66730,6 +68109,15 @@ + + + + + + + + + @@ -66808,12 +68196,12 @@ - + - + - + @@ -66922,7 +68310,7 @@ - + @@ -66946,7 +68334,7 @@ - + @@ -66994,87 +68382,93 @@ - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - - - + - + + + + - + - + - + - - + + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -67083,7 +68477,7 @@ - + @@ -67214,12 +68608,12 @@ - + - + - + @@ -67537,15 +68931,15 @@ - + - + - - + + - - + + @@ -67883,7 +69277,7 @@ - + @@ -68023,266 +69417,292 @@ - + - + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + @@ -68496,21 +69916,21 @@ - + - + - + - + - + - + @@ -68531,7 +69951,7 @@ - + @@ -68553,22 +69973,22 @@ - + - + - + - + - + - + @@ -68615,7 +70035,7 @@ - + @@ -68767,6 +70187,9 @@ + + + @@ -68820,33 +70243,36 @@ - + - + - + - + - + - + - + - + - + - + + + + @@ -68860,39 +70286,42 @@ - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + @@ -69008,9 +70437,9 @@ - + - + @@ -69043,7 +70472,7 @@ - + @@ -69059,13 +70488,13 @@ - + - + - + @@ -69124,36 +70553,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -69172,27 +70601,27 @@ - + - + - + - + - + - + - + - + @@ -69220,6 +70649,10 @@ + + + + @@ -69264,15 +70697,15 @@ - + - + - + - + @@ -69311,6 +70744,20 @@ + + + + + + + + + + + + + + @@ -69321,29 +70768,29 @@ - + - + - + - + - + - + - + - + @@ -69377,15 +70824,15 @@ - + - + - + - + @@ -69429,51 +70876,54 @@ - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + @@ -69485,7 +70935,7 @@ - + @@ -69526,136 +70976,136 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -69760,81 +71210,108 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -69851,15 +71328,15 @@ - + - + - + - + @@ -69931,9 +71408,9 @@ - + - + @@ -69956,39 +71433,40 @@ - + - + - + - + - + - + - + - + - + - + - + + @@ -70178,7 +71656,7 @@ - + @@ -70195,18 +71673,18 @@ - + - + - + - + - + @@ -70256,6 +71734,14 @@ + + + + + + + + @@ -70279,18 +71765,18 @@ - + - + - + - + - + @@ -70396,12 +71882,12 @@ - + - + - + @@ -70616,48 +72102,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -70911,99 +72397,102 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - + + - + - + - + + + + - + - - + + - - + + - - - - - - - - - - - - - - + + - + + + + - + - - + + + + + + + + + + + + + + + + + @@ -71213,15 +72702,15 @@ - + - + - + - + @@ -71301,68 +72790,68 @@ - + - - - - - - - + - + - + - + + + + - + - + - + - + - - - - - - - - + + - + - + - + - - - - + - - - - + + + + + + + + + + + + + + + + @@ -71399,186 +72888,192 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -71651,7 +73146,7 @@ - + @@ -71805,200 +73300,209 @@ - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - + + - - - - + - + - + - - - - - - - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - + - + - + - + @@ -72015,6 +73519,11 @@ + + + + + @@ -72038,18 +73547,21 @@ - + - + - + - + - + + + + @@ -72535,41 +74047,41 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -72609,6 +74121,10 @@ + + + + @@ -72629,7 +74145,7 @@ - + @@ -72699,25 +74215,37 @@ + + + + + + + + + + + + - + - + - + - + - + - + - + @@ -72759,30 +74287,30 @@ - + - + - + - + - + - + - + - + - + @@ -72917,6 +74445,7 @@ + @@ -73142,6 +74671,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -73279,39 +74861,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -73335,72 +74920,81 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + @@ -73455,51 +75049,51 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -73514,48 +75108,48 @@ - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -73674,7 +75268,7 @@ - + @@ -73759,12 +75353,12 @@ - + - + - + @@ -74009,206 +75603,39 @@ - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -74372,12 +75799,12 @@ - + - + - + @@ -74430,45 +75857,45 @@ - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - + @@ -74816,47 +76243,22 @@ - + + - - - - - - - + - - - - + - + - - - - - - - - - - - - - - - - - - + - + @@ -74899,7 +76301,7 @@ - + @@ -74989,115 +76391,121 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -75210,30 +76618,33 @@ - + - + - + - + - + - + - + - + - + + + + @@ -75243,7 +76654,7 @@ - + @@ -75343,73 +76754,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -75418,52 +76829,52 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -75499,12 +76910,12 @@ - + - + - + @@ -75672,6 +77083,9 @@ + + + @@ -75749,7 +77163,7 @@ - + @@ -75823,15 +77237,15 @@ - + - + - + - + @@ -75859,12 +77273,12 @@ - + - + - + @@ -76137,7 +77551,7 @@ - + @@ -76171,10 +77585,10 @@ - + - + @@ -76206,12 +77620,12 @@ - + - + - + @@ -76250,18 +77664,18 @@ - + - + - + - + - + @@ -76375,42 +77789,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -76474,7 +77891,7 @@ - + @@ -76538,7 +77955,7 @@ - + @@ -76558,6 +77975,7 @@ + @@ -76573,201 +77991,213 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - - - + - + + + + - + - + - + - + - - + + - - + + - - + + - - - - - - - - + + - + - - + + + + + + + + + + + + + + + + + + + + + + + @@ -76776,7 +78206,7 @@ - + @@ -76979,15 +78409,18 @@ - + - + - + + + + @@ -77315,57 +78748,72 @@ - + - + + + + - + - + - + - + - + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + @@ -77390,12 +78838,188 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -77416,279 +79040,291 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - + + - + - - + + - - - - - - - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -77873,24 +79509,24 @@ - + - + - + - + - + - + - + @@ -77907,42 +79543,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -78048,30 +79687,30 @@ - + - + - + - + - + - + - + - + - + @@ -78100,72 +79739,72 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -78195,51 +79834,54 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -78272,7 +79914,7 @@ - + @@ -78387,46 +80029,46 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -78445,24 +80087,24 @@ - + - + - + - + - + - + - + @@ -79043,45 +80685,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -79129,7 +80771,7 @@ - + @@ -79256,8 +80898,11 @@ + + + - + @@ -79267,12 +80912,12 @@ - + - + - + @@ -79307,21 +80952,24 @@ - + - + - + - + - + - - + + + + + @@ -79366,14 +81014,14 @@ - + - + @@ -79426,12 +81074,12 @@ - + - + - + @@ -79469,15 +81117,15 @@ - + - + - + - + @@ -79497,27 +81145,28 @@ - + + - + - + - + - + - + - + - + @@ -79545,63 +81194,78 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + @@ -79676,16 +81340,16 @@ - + - + - + @@ -79716,7 +81380,7 @@ - + @@ -79822,56 +81486,59 @@ - + - + - + - + - + - + - + - + - + + + + - + - + - + - + - + - + - + - + - + @@ -79999,61 +81666,67 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -80091,7 +81764,7 @@ - + @@ -80158,12 +81831,12 @@ - + - + - + @@ -80257,26 +81930,26 @@ - + - + - + - + - + - + - + @@ -80314,7 +81987,7 @@ - + @@ -81019,36 +82692,39 @@ - + - + - + - + - + - + - + - + - + - + - + + + + @@ -81069,699 +82745,711 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - + + - + - - - - - + + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - - + + - - + + - - - - - - - - - - - - - - - - - + + - + + + + + + + - + + + + - + - - + + - - - - - - - - - - - - - - - - - - - - + + - + - - + + + + + - - - - + - + + + + - + - - + + + + + + + + + + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -82053,7 +83741,7 @@ - + @@ -82095,10 +83783,19 @@ - + + + + + + + + + + @@ -82126,6 +83823,9 @@ + + + @@ -82441,12 +84141,12 @@ - + - + - + @@ -82539,83 +84239,83 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -82800,15 +84500,15 @@ - + - + - + - + @@ -83085,15 +84785,15 @@ - + - + - + - + @@ -83136,18 +84836,18 @@ - + - + - + - + - + @@ -83339,18 +85039,18 @@ - + - + - + - + - + @@ -83476,7 +85176,7 @@ - + @@ -83486,204 +85186,216 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - + + - - - - + - + + + + - + - - + + - - + + - - + + - - + + - - + + - - - - - + + - + - + - - + + - - + + - - + + - - - - - - - - + + - + - + - - + + - - + + + + + + + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + @@ -83755,7 +85467,7 @@ - + @@ -84021,14 +85733,15 @@ - + - + - + + @@ -84108,21 +85821,24 @@ - + - + - + - + - + - + + + + @@ -84255,6 +85971,14 @@ + + + + + + + + @@ -84412,18 +86136,18 @@ - + - + - + - + - + @@ -84542,6 +86266,7 @@ + @@ -84776,7 +86501,7 @@ - + @@ -84887,18 +86612,18 @@ - + - + - + - + - + @@ -84995,24 +86720,27 @@ - + - + - + - + - + - + - + + + + @@ -85044,105 +86772,108 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -85194,7 +86925,7 @@ - + @@ -85283,56 +87014,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - + @@ -85355,54 +87089,57 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -85514,7 +87251,7 @@ - + @@ -85690,18 +87427,18 @@ - + - + - + - + - + @@ -85714,6 +87451,11 @@ + + + + + @@ -85833,6 +87575,7 @@ + @@ -86089,7 +87832,7 @@ - + @@ -86195,21 +87938,21 @@ - + - + - + - + - + - + @@ -86217,12 +87960,12 @@ - + - + - + @@ -86444,7 +88187,7 @@ - + @@ -86458,7 +88201,7 @@ - + @@ -86663,59 +88406,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -86869,21 +88612,24 @@ - + - + - + - + - + - + + + + @@ -86914,21 +88660,21 @@ - + - + - + - + - + - + @@ -87287,30 +89033,30 @@ - + - + - + - + - + - + - + - + - + @@ -87398,128 +89144,128 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -87542,12 +89288,12 @@ - + - + - + @@ -87636,15 +89382,15 @@ - + - + - + - + @@ -88001,57 +89747,60 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -88065,7 +89814,33 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -88154,42 +89929,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -88349,27 +90127,27 @@ - + - + - + - + - + - + - + - + @@ -88408,54 +90186,60 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -88668,112 +90452,112 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88799,12 +90583,12 @@ - + - + - + @@ -88962,18 +90746,19 @@ - + - + - + + @@ -89018,24 +90803,24 @@ - + - + - + - + - + - + - + @@ -89084,15 +90869,15 @@ - + - + - + - + @@ -89128,7 +90913,8 @@ - + + @@ -89212,48 +90998,51 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -89305,84 +91094,87 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -89393,29 +91185,29 @@ - + - + - + - + - + - + - + - + @@ -89577,30 +91369,30 @@ - + - + - + - + - + - + - + - + - + @@ -89823,207 +91615,213 @@ - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + + + + + + + - + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + - + - + - - - - - + + - + - + + + + - + - - - - - - - - - - - - - - - - - - - - - - + - - - - + - + - - - - - + + - + - - - - + - + - + + + + + + + + + + + + + + + + + + + + + + - + + + + - - - - - - - - - - - - - + - + - + - + - - - - - - - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -90037,12 +91835,12 @@ - + - + - + @@ -90131,402 +91929,405 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -90535,15 +92336,15 @@ - + - + - + - + @@ -90564,120 +92365,123 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -90693,7 +92497,7 @@ - + @@ -90748,127 +92552,136 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + @@ -90921,15 +92734,15 @@ - + - + - + - + @@ -90940,59 +92753,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -91001,16 +92814,16 @@ - + - + - + - + @@ -91086,24 +92899,24 @@ - + - + - + - + - + - + - + @@ -91114,43 +92927,46 @@ + + + - + - + - + - + - + - + - + - + - + - + - + @@ -91418,126 +93234,129 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -91573,42 +93392,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -92152,6 +93977,12 @@ + + + + + + @@ -92352,36 +94183,56 @@ - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + @@ -92392,30 +94243,42 @@ - + - + - + - + - + - + - + - + - - + + + + + + + + + + + + + + @@ -92570,6 +94433,14 @@ + + + + + + + + @@ -92633,116 +94504,116 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92838,373 +94709,385 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -93212,39 +95095,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -93973,6 +95859,7 @@ + @@ -93992,48 +95879,51 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -94088,7 +95978,7 @@ - + @@ -94122,47 +96012,47 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -94214,48 +96104,51 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -94456,7 +96349,7 @@ - + @@ -94516,7 +96409,7 @@ - + @@ -94535,14 +96428,17 @@ - + - + - + + + + @@ -94556,48 +96452,60 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -94606,7 +96514,7 @@ - + @@ -94875,21 +96783,21 @@ - + - + - + - + - + - + @@ -95553,6 +97461,7 @@ + @@ -95575,15 +97484,15 @@ - + - + - + - + @@ -95607,20 +97516,20 @@ - + - + - + - + - + @@ -95722,33 +97631,39 @@ - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -95769,21 +97684,21 @@ - + - + - + - + - + - + @@ -95806,9 +97721,10 @@ + - + @@ -95979,7 +97895,7 @@ - + @@ -96020,30 +97936,30 @@ - + - + - + - + - + - + - + - + - + @@ -96163,7 +98079,7 @@ - + @@ -96196,7 +98112,7 @@ - + @@ -96331,7 +98247,18 @@ - + + + + + + + + + + + + @@ -96463,7 +98390,7 @@ - + @@ -96476,27 +98403,27 @@ - + - + - + - + - + - + - + @@ -96605,22 +98532,22 @@ - + - + - + - + - + - + @@ -96644,18 +98571,18 @@ - + - + - + - + - + @@ -96664,18 +98591,19 @@ - + + - + - + - + - + @@ -96790,12 +98718,12 @@ - + - + - + @@ -97006,6 +98934,7 @@ + @@ -97741,9 +99670,9 @@ - + - + @@ -97762,7 +99691,7 @@ - + @@ -97799,43 +99728,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -97896,7 +99825,7 @@ - + @@ -97953,7 +99882,7 @@ - + @@ -97969,7 +99898,7 @@ - + @@ -98000,31 +99929,31 @@ - + - + - + - + - + - + - + - + - + @@ -98045,9 +99974,9 @@ - + - + @@ -98060,11 +99989,11 @@ - + - + @@ -98193,27 +100122,27 @@ - + - + - + - + - + - + - + - + @@ -98320,83 +100249,83 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + @@ -98606,24 +100535,35 @@ + + + + + + + + + + + - + - + - + - + - + @@ -98675,75 +100615,87 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -98773,28 +100725,28 @@ - + - + - + - + - + - + - + - + @@ -98881,24 +100833,27 @@ - + - + - + - + - + - + - + + + + @@ -98995,33 +100950,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -99205,15 +101160,22 @@ - + + - + - + - + + + + + + + @@ -99249,39 +101211,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -99518,7 +101483,7 @@ - + @@ -99534,31 +101499,31 @@ - + - + - + - + - + - + - + - + - + @@ -99596,87 +101561,90 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -99685,83 +101653,83 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -99854,12 +101822,12 @@ - + - + - + @@ -100338,72 +102306,78 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + @@ -100433,169 +102407,175 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + + + + - + + + + - + - + - + - + - + + + + + + + + + + + + + - + @@ -100614,16 +102594,16 @@ - + - + - + - + @@ -100698,64 +102678,72 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + @@ -100825,84 +102813,84 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -101098,108 +103086,120 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + @@ -101226,7 +103226,7 @@ - + @@ -101343,12 +103343,15 @@ + + + - + @@ -101379,55 +103382,55 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -101472,15 +103475,15 @@ - + - + - + - + @@ -101610,6 +103613,7 @@ + @@ -101673,12 +103677,12 @@ - + - + - + @@ -102018,6 +104022,12 @@ + + + + + + @@ -102163,134 +104173,137 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - + + - - + + - - - - + - + - + - + - + - + - + - + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + @@ -102307,6 +104320,11 @@ + + + + + @@ -102318,7 +104336,7 @@ - + @@ -102328,119 +104346,122 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -102502,16 +104523,17 @@ - + - + - + + @@ -102628,44 +104650,44 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -102769,27 +104791,27 @@ - + - + - + - + - + - + - + - + @@ -102798,70 +104820,77 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + @@ -102876,30 +104905,33 @@ - + - + - + - + - + - + - + - + - + + + + @@ -102907,45 +104939,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -102992,6 +105027,17 @@ + + + + + + + + + + + @@ -103006,7 +105052,7 @@ - + @@ -103040,8 +105086,27 @@ + + + + + + + + + + + + + + + + + + + @@ -103315,81 +105380,84 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + @@ -103482,12 +105550,12 @@ - + - + - + @@ -103528,18 +105596,18 @@ - + - + - + - + - + @@ -103596,15 +105664,15 @@ - + - + - + - + @@ -103753,33 +105821,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -103842,41 +105910,47 @@ - + - + - + - + - + - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + @@ -103923,12 +105997,12 @@ - + - + - + @@ -103957,24 +106031,24 @@ - + - + - + - + - + - + - + @@ -103984,15 +106058,15 @@ - + - + - + - + @@ -104115,33 +106189,36 @@ - + - + - + - + - + - + - + - + - + - + + + + @@ -104180,39 +106257,39 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -104259,7 +106336,16 @@ - + + + + + + + + + + @@ -104356,6 +106442,7 @@ + @@ -104420,7 +106507,7 @@ - + @@ -104478,157 +106565,187 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + + + + + + + + + + - + - + - + - - - - + - - - - + - - - - + - + + + + - + + + + - + - - + + - - - - - - - - - - - - - - + + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -104813,93 +106930,105 @@ - + - + - + - + - - - - - - - - - - + - - - - - - - - - - - - - - - - + - + - + - + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -105105,15 +107234,15 @@ - + - + - + - + @@ -105146,24 +107275,27 @@ - + - + - + - + - + - + - + + + + @@ -105380,14 +107512,14 @@ - + - + @@ -105432,24 +107564,25 @@ - + + - + - + - + - + - + - + @@ -105570,9 +107703,9 @@ - - - + + + @@ -106291,8 +108424,8 @@ - - + + @@ -106412,8 +108545,8 @@ - - + + @@ -106513,8 +108646,8 @@ - - + + @@ -107733,6 +109866,19 @@ + + + + + + + + + + + + + @@ -108253,6 +110399,8 @@ + + @@ -108623,12 +110771,12 @@ - - - - - - + + + + + + @@ -108844,12 +110992,12 @@ - - + + - - + + @@ -109230,8 +111378,8 @@ - - + + @@ -109241,8 +111389,8 @@ - - + + @@ -109260,27 +111408,27 @@ - - + + - - + + - - + + - - - - + + + + - - - + + + @@ -109301,29 +111449,29 @@ - - - - + + + + - - - - - + + + + + - - + + - - + + - - + + @@ -109331,46 +111479,46 @@ - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - - - - + + + + + @@ -109383,27 +111531,27 @@ - - + + - - - + + + - - + + - - - + + + - - - + + + @@ -109414,22 +111562,22 @@ - - + + - - - + + + - - + + - - - + + + @@ -109442,21 +111590,21 @@ - - + + - - + + - - - + + + @@ -109465,18 +111613,18 @@ - - - - + + + + - - + + - - + + @@ -109504,19 +111652,19 @@ - - - + + + - - - + + + - - - + + + @@ -109632,28 +111780,28 @@ - - + + - - + + - - + + - - + + @@ -109661,10 +111809,10 @@ - - - - + + + + @@ -110439,19 +112587,19 @@ - - - + + + - - - - + + + + @@ -110988,6 +113136,12 @@ + + + + + + @@ -111164,6 +113318,12 @@ + + + + + + @@ -111290,9 +113450,9 @@ - - - + + + @@ -113106,16 +115266,16 @@ - - - - + + + + - - - - + + + + @@ -113181,26 +115341,26 @@ - - - + + + - - - + + + - - - - - + + + + + - - - + + + @@ -113210,16 +115370,16 @@ - - - + + + - - - - - + + + + + @@ -113235,51 +115395,51 @@ - - - + + + - - - + + + - - - - + + + + - - + + - - + + - - - - + + + + - - + + - - + + - - - + + + @@ -113603,6 +115763,11 @@ + + + + + @@ -113645,6 +115810,12 @@ + + + + + + @@ -114284,6 +116455,11 @@ + + + + + @@ -115154,6 +117330,12 @@ + + + + + + @@ -115889,10 +118071,10 @@ - - - - + + + + @@ -116146,7 +118328,7 @@ - + @@ -116271,10 +118453,10 @@ - - - - + + + + @@ -116766,12 +118948,12 @@ - - + + - - + + @@ -117499,6 +119681,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -119283,8 +121534,8 @@ - - + + @@ -119292,10 +121543,10 @@ - - - - + + + + @@ -119388,9 +121639,9 @@ - - - + + + @@ -119430,6 +121681,12 @@ + + + + + + @@ -120170,8 +122427,8 @@ - - + + @@ -120733,13 +122990,13 @@ - - + + - - - + + + @@ -121010,13 +123267,6 @@ - - - - - - - @@ -121070,6 +123320,10 @@ + + + + @@ -121373,9 +123627,9 @@ - - - + + + @@ -121441,15 +123695,15 @@ - + - - + + @@ -121546,16 +123800,16 @@ - - - - + + + + - - - - + + + + @@ -123009,13 +125263,13 @@ - - - + + + - - + + @@ -123925,21 +126179,21 @@ - - - + + + - - + + - - - - - - + + + + + + @@ -123952,10 +126206,10 @@ - - - - + + + + @@ -124357,8 +126611,8 @@ - - + + @@ -125429,6 +127683,7 @@ + @@ -125564,16 +127819,16 @@ - - - - - - + + + + + + - - + + @@ -125668,10 +127923,10 @@ - - - - + + + + @@ -125726,9 +127981,9 @@ - - - + + + @@ -125870,9 +128125,9 @@ - - - + + + @@ -126767,10 +129022,10 @@ - - - - + + + + @@ -126792,11 +129047,11 @@ - - - - - + + + + + @@ -126807,61 +129062,61 @@ - - + + - - - - - + + + + + - - - + + + - - - - - + + + + + - - - + + + - - - - + + + + - - - - - - + + + + + + - - - + + + - - - + + + - - - + + + @@ -126872,20 +129127,20 @@ - - - - - - + + + + + + - - + + @@ -127270,8 +129525,8 @@ - - + + @@ -128132,13 +130387,13 @@ - - - + + + - - + + @@ -128159,53 +130414,53 @@ - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -128221,16 +130476,16 @@ - - + + - - + + - - + + @@ -128447,12 +130702,6 @@ - - - - - - @@ -128585,11 +130834,11 @@ - - - - - + + + + + @@ -128755,13 +131004,13 @@ - - + + - - - + + + @@ -128829,12 +131078,12 @@ - - + + - - + + @@ -128908,12 +131157,12 @@ - - + + - - + + @@ -128923,8 +131172,8 @@ - - + + @@ -129042,22 +131291,22 @@ - - + + - - - + + + - - - + + + @@ -129072,8 +131321,8 @@ - - + + @@ -129105,7 +131354,7 @@ - + @@ -129215,10 +131464,10 @@ - - - - + + + + @@ -129263,10 +131512,10 @@ - - - - + + + + @@ -130538,22 +132787,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -130562,16 +132811,16 @@ - + - - - - - - - + + + + + + + @@ -130617,23 +132866,23 @@ - - - - + + + + - - - - + + + + - - - - - + + + + + diff --git a/android/abi_gki_aarch64_exynosauto b/android/abi_gki_aarch64_exynosauto index 681f0f27b372..d05a5baadab9 100644 --- a/android/abi_gki_aarch64_exynosauto +++ b/android/abi_gki_aarch64_exynosauto @@ -120,6 +120,7 @@ dma_buf_unmap_attachment dma_buf_vmap dma_buf_vunmap + dmaengine_unmap_put dma_fence_add_callback dma_fence_context_alloc dma_fence_default_wait @@ -338,6 +339,7 @@ of_property_read_string of_property_read_string_helper of_property_read_u32_index + of_property_read_variable_u16_array of_property_read_variable_u32_array of_property_read_variable_u8_array of_prop_next_string @@ -348,6 +350,7 @@ panic_notifier_list param_ops_bool param_ops_int + param_ops_string param_ops_uint PDE_DATA __per_cpu_offset @@ -358,7 +361,6 @@ pinctrl_lookup_state pinctrl_select_state platform_bus_type - platform_device_unregister __platform_driver_register platform_driver_unregister platform_get_irq @@ -563,6 +565,7 @@ wait_for_completion_timeout __wake_up wake_up_process + wakeup_source_register wakeup_source_unregister __warn_printk work_busy @@ -598,6 +601,20 @@ debugfs_create_x32 kernel_kobj +# required by dmatest.ko + dmaengine_get_unmap_data + __dma_request_channel + dma_sync_wait + kthread_should_stop + param_get_bool + param_get_string + param_set_bool + param_set_copystring + prandom_bytes + set_freezable + set_user_nice + strim + # required by dss.ko android_debug_symbol arch_timer_read_counter @@ -657,6 +674,7 @@ platform_device_alloc platform_device_del platform_device_put + platform_device_unregister platform_get_irq_byname_optional __pm_relax pm_runtime_allow @@ -723,16 +741,20 @@ __drm_atomic_helper_plane_destroy_state __drm_atomic_helper_plane_duplicate_state __drm_atomic_helper_plane_reset + __drm_atomic_helper_private_obj_duplicate_state drm_atomic_helper_set_config drm_atomic_helper_shutdown drm_atomic_helper_update_plane drm_atomic_helper_wait_for_vblanks drm_atomic_normalize_zpos + drm_atomic_private_obj_init drm_bridge_attach drm_compat_ioctl drm_connector_list_iter_begin drm_connector_list_iter_end drm_connector_list_iter_next + drm_connector_set_path_property + drm_connector_set_tile_property drm_connector_unregister drm_crtc_arm_vblank_event drm_crtc_cleanup @@ -758,7 +780,9 @@ drm_dp_aux_register drm_dp_aux_unregister drm_dp_bw_code_to_link_rate + drm_dp_calc_pbn_mode drm_dp_channel_eq_ok + drm_dp_check_act_status drm_dp_clock_recovery_ok drm_dp_downstream_debug drm_dp_dpcd_read @@ -766,11 +790,23 @@ drm_dp_dpcd_write drm_dp_dsc_sink_line_buf_depth drm_dp_dsc_sink_max_slice_count + drm_dp_find_vcpi_slots drm_dp_get_adjust_request_pre_emphasis drm_dp_get_adjust_request_voltage drm_dp_link_train_channel_eq_delay drm_dp_link_train_clock_recovery_delay + drm_dp_mst_allocate_vcpi + drm_dp_mst_deallocate_vcpi + drm_dp_mst_detect_port drm_dp_mst_dump_topology + drm_dp_mst_get_port_malloc + drm_dp_mst_hpd_irq + drm_dp_mst_put_port_malloc + drm_dp_mst_reset_vcpi_slots + drm_dp_mst_topology_mgr_init + drm_dp_mst_topology_mgr_set_mst + drm_dp_update_payload_part1 + drm_dp_update_payload_part2 drm_dsc_compute_rc_parameters drm_dsc_pps_payload_pack drm_encoder_cleanup @@ -798,6 +834,7 @@ drm_helper_connector_dpms drm_helper_hpd_irq_event drm_helper_mode_fill_fb_struct + drm_helper_probe_detect drm_ioctl drm_kms_helper_poll_fini drm_kms_helper_poll_init @@ -831,9 +868,11 @@ kstrtobool mipi_dsi_host_register mipi_dsi_host_unregister + mutex_is_locked of_drm_find_bridge of_drm_find_panel of_graph_get_endpoint_by_regs + of_graph_get_endpoint_count of_graph_get_next_endpoint of_graph_get_port_by_id of_graph_get_remote_port @@ -841,9 +880,7 @@ of_graph_parse_endpoint of_phandle_iterator_init of_phandle_iterator_next - platform_find_device_by_driver seq_hex_dump - strnstr # required by exynos_mfc.ko iommu_dma_reserve_iova @@ -857,6 +894,13 @@ _raw_write_lock_irqsave _raw_write_unlock_irqrestore +# required by exynos_thermal.ko + of_thermal_get_ntrips + of_thermal_is_trip_valid + thermal_zone_device_update + thermal_zone_of_sensor_register + thermal_zone_of_sensor_unregister + # required by exynos_tty.ko dma_get_slave_caps do_SAK @@ -901,10 +945,6 @@ # required by i2c-exynosauto.ko i2c_del_adapter -# required by lt8912.ko - drm_mode_duplicate - of_get_drm_display_mode - # required by mali_kbase.ko anon_inode_getfd __arch_clear_user @@ -1065,7 +1105,6 @@ dma_async_device_register dma_async_device_unregister dma_async_tx_descriptor_init - dmaengine_unmap_put dma_get_slave_channel dma_map_resource dma_unmap_resource @@ -1125,7 +1164,6 @@ kthread_flush_work kthread_flush_worker ktime_get_with_offset - param_ops_string phy_configure pm_relax pm_stay_awake @@ -1371,14 +1409,11 @@ # required by ufs-exynosauto-core.ko blk_ksm_init_passthrough - of_property_read_variable_u16_array __traceiter_android_vh_ufs_fill_prdt __tracepoint_android_vh_ufs_fill_prdt - ufshcd_alloc_host ufshcd_config_pwr_mode ufshcd_dme_get_attr ufshcd_dme_set_attr - ufshcd_init ufshcd_link_recovery ufshcd_pltfrm_init ufshcd_remove @@ -1386,9 +1421,6 @@ ufshcd_system_resume ufshcd_system_suspend -# required by vbpipe-module.ko - wakeup_source_register - # required by vbufq-be-module.ko drain_workqueue radix_tree_delete diff --git a/android/abi_gki_aarch64_fips140 b/android/abi_gki_aarch64_fips140 index 05df3848ff15..cf48b6d66aa9 100644 --- a/android/abi_gki_aarch64_fips140 +++ b/android/abi_gki_aarch64_fips140 @@ -1,12 +1,17 @@ [abi_symbol_list] +# commonly used symbols + module_layout + __put_task_struct # required by fips140.ko add_random_ready_callback aead_register_instance + arm64_const_caps_ready bcmp cancel_work_sync __cfi_slowpath cpu_have_feature + cpu_hwcap_keys crypto_aead_decrypt crypto_aead_encrypt crypto_aead_setauthsize @@ -16,11 +21,14 @@ crypto_alg_list crypto_alg_mod_lookup crypto_alg_sem + crypto_alloc_aead crypto_alloc_base crypto_alloc_rng crypto_alloc_shash + crypto_alloc_skcipher crypto_attr_alg_name crypto_check_attr_type + crypto_cipher_decrypt_one crypto_cipher_encrypt_one crypto_cipher_setkey crypto_destroy_tfm @@ -43,6 +51,7 @@ crypto_remove_final crypto_remove_spawns crypto_req_done + crypto_rng_reset crypto_shash_alg_has_setkey crypto_shash_digest crypto_shash_final @@ -77,6 +86,7 @@ kmalloc_caches kmalloc_order_trace kmem_cache_alloc_trace + kmemdup __list_add_valid __list_del_entry_valid memcpy @@ -89,6 +99,7 @@ preempt_schedule_notrace printk queue_work_on + refcount_warn_saturate scatterwalk_ffwd scatterwalk_map_and_copy sg_init_one diff --git a/android/abi_gki_aarch64_generic b/android/abi_gki_aarch64_generic index 20cbaa91b817..c13cbabe6d35 100644 --- a/android/abi_gki_aarch64_generic +++ b/android/abi_gki_aarch64_generic @@ -24,6 +24,8 @@ amba_driver_unregister android_rvh_probe_register anon_inode_getfd + arc4_crypt + arc4_setkey __arch_clear_user __arch_copy_from_user __arch_copy_in_user @@ -105,6 +107,7 @@ bus_unregister bus_unregister_notifier cache_line_size + call_rcu cancel_delayed_work cancel_delayed_work_sync cancel_work_sync @@ -225,6 +228,8 @@ cpus_read_lock cpus_read_unlock cpu_subsys + crc32_be + crc32_le crc8 crc8_populate_msb crypto_aead_decrypt @@ -234,19 +239,24 @@ crypto_alloc_aead crypto_alloc_base crypto_alloc_shash + crypto_alloc_skcipher crypto_alloc_sync_skcipher crypto_comp_compress crypto_comp_decompress crypto_destroy_tfm + __crypto_memneq crypto_register_alg crypto_register_scomp crypto_shash_digest crypto_shash_finup crypto_shash_setkey + crypto_shash_update + crypto_skcipher_decrypt crypto_skcipher_encrypt crypto_skcipher_setkey crypto_unregister_alg crypto_unregister_scomp + __crypto_xor csum_ipv6_magic csum_tcpudp_nofold _ctype @@ -264,6 +274,7 @@ debugfs_create_x32 debugfs_lookup debugfs_remove + debugfs_rename dec_zone_page_state default_llseek deferred_free @@ -273,11 +284,14 @@ del_timer_sync desc_to_gpio destroy_workqueue + dev_alloc_name + dev_change_net_namespace dev_close _dev_crit dev_driver_string _dev_emerg _dev_err + dev_fetch_sw_netstats devfreq_add_device devfreq_add_governor devfreq_monitor_resume @@ -290,12 +304,16 @@ devfreq_unregister_opp_notifier devfreq_update_interval dev_fwnode + __dev_get_by_index + dev_get_by_index dev_get_by_name + device_add device_add_disk device_add_groups device_create device_create_file device_create_with_groups + device_del device_destroy device_find_child device_for_each_child @@ -311,6 +329,7 @@ device_register device_remove_file device_remove_groups + device_rename device_set_wakeup_capable device_set_wakeup_enable device_unregister @@ -390,6 +409,8 @@ devm_thermal_zone_of_sensor_unregister devm_usb_get_phy_by_phandle _dev_notice + dev_pm_domain_attach_by_name + dev_pm_domain_detach dev_pm_opp_add dev_pm_opp_disable dev_pm_opp_find_freq_ceil @@ -406,6 +427,7 @@ dev_pm_opp_set_regulators dev_pm_qos_read_value dev_pm_qos_update_request + dev_printk dev_printk_emit dev_queue_xmit devres_add @@ -690,6 +712,9 @@ emergency_restart enable_irq enable_percpu_irq + ether_setup + eth_mac_addr + ethtool_op_get_link eth_type_trans eventfd_ctx_fdget eventfd_ctx_put @@ -719,6 +744,7 @@ find_vma finish_wait flush_dcache_page + flush_delayed_work flush_work flush_workqueue fput @@ -745,6 +771,7 @@ generic_file_llseek generic_handle_irq generic_iommu_put_resv_regions + genlmsg_multicast_allns genlmsg_put genl_register_family genl_unregister_family @@ -763,6 +790,8 @@ get_device __get_free_pages get_governor_parent_kobj + get_net_ns_by_fd + get_net_ns_by_pid get_pid_task get_random_bytes get_random_bytes_arch @@ -830,6 +859,9 @@ hrtimer_sleeper_start_expires hrtimer_start_range_ns hrtimer_try_to_cancel + __hw_addr_init + __hw_addr_sync + __hw_addr_unsync hwrng_register hwrng_unregister i2c_adapter_type @@ -858,6 +890,7 @@ idr_destroy idr_find idr_for_each + idr_get_next idr_preload idr_remove iio_device_unregister @@ -865,6 +898,7 @@ in6_pton in_aton inc_zone_page_state + inet_csk_get_port init_dummy_netdev init_net init_pseudo @@ -872,6 +906,7 @@ __init_swait_queue_head init_task init_timer_key + init_uts_ns init_wait_entry __init_waitqueue_head input_allocate_device @@ -960,9 +995,14 @@ kasprintf kernel_cpustat kernel_kobj + kernel_param_lock + kernel_param_unlock kernel_restart kern_mount kern_unmount + key_create_or_update + key_put + keyring_alloc __kfifo_alloc __kfifo_free __kfifo_in @@ -971,6 +1011,7 @@ kfree kfree_sensitive kfree_skb + kfree_skb_list kill_anon_super kill_fasync kimage_voffset @@ -1026,11 +1067,13 @@ kthread_stop kthread_worker_fn ktime_get + ktime_get_coarse_with_offset ktime_get_mono_fast_ns ktime_get_raw ktime_get_raw_ts64 ktime_get_real_seconds ktime_get_real_ts64 + ktime_get_seconds ktime_get_ts64 ktime_get_with_offset kvfree @@ -1049,6 +1092,8 @@ log_threaded_irq_wakeup_reason __log_write_mmio loops_per_jiffy + lru_cache_disable + lru_cache_enable lzo1x_1_compress lzo1x_decompress_safe lzorle1x_1_compress @@ -1116,19 +1161,31 @@ __napi_schedule napi_schedule_prep __netdev_alloc_skb + netdev_err + netdev_info + netdev_set_default_ethtool_ops netdev_state_change netdev_update_features + netif_carrier_off + netif_carrier_on netif_napi_add __netif_napi_del netif_receive_skb + netif_receive_skb_list netif_rx netif_rx_ni + netif_tx_stop_all_queues netif_tx_wake_queue + netlink_broadcast __netlink_kernel_create netlink_kernel_release + netlink_register_notifier netlink_unicast + netlink_unregister_notifier + net_ns_type_operations net_ratelimit nla_append + nla_find nla_memcpy __nla_parse nla_put @@ -1136,6 +1193,7 @@ nla_put_nohdr nla_reserve nla_strlcpy + __nla_validate __nlmsg_put no_llseek nonseekable_open @@ -1370,8 +1428,10 @@ proc_remove pskb_expand_head __pskb_pull_tail + ___pskb_trim put_device put_disk + __put_net __put_page put_sg_io_hdr __put_task_struct @@ -1409,6 +1469,7 @@ rb_next rb_prev rb_replace_node + rcu_barrier __rcu_read_lock __rcu_read_unlock rdev_get_drvdata @@ -1430,6 +1491,7 @@ register_netdev register_netdevice register_netdevice_notifier + register_pernet_device register_pernet_subsys register_pm_notifier register_reboot_notifier @@ -1489,11 +1551,24 @@ return_address revalidate_disk_size rfkill_alloc + rfkill_blocked rfkill_destroy rfkill_init_sw_state + rfkill_pause_polling rfkill_register + rfkill_resume_polling + rfkill_set_hw_state rfkill_unregister + rhashtable_free_and_destroy + rhashtable_insert_slow + rhltable_init + __rht_bucket_nested + rht_bucket_nested + rht_bucket_nested_insert root_task_group + round_jiffies + round_jiffies_relative + round_jiffies_up rps_needed rtc_class_close rtc_class_open @@ -1583,11 +1658,20 @@ single_open single_open_size single_release + skb_add_rx_frag skb_checksum + skb_checksum_help skb_clone + skb_clone_sk + skb_complete_wifi_ack skb_copy + skb_copy_bits + skb_copy_expand skb_dequeue skb_dequeue_tail + skb_ensure_writable + __skb_get_hash + __skb_gso_segment skb_pull skb_push skb_put @@ -1677,6 +1761,8 @@ snd_soc_unregister_component snprintf soc_device_register + __sock_create + sock_release sock_wfree softnet_data sort @@ -1735,8 +1821,10 @@ submit_bio submit_bio_wait subsys_system_register + __sw_hweight16 __sw_hweight32 __sw_hweight64 + __sw_hweight8 sync_file_create sync_file_get_fence synchronize_irq @@ -1761,6 +1849,7 @@ sysfs_streq sysfs_update_group sysrq_mask + system_freezable_wq system_freezing_cnt system_highpri_wq system_long_wq @@ -1821,6 +1910,7 @@ __traceiter_android_rvh_post_init_entity_util_avg __traceiter_android_rvh_preempt_disable __traceiter_android_rvh_preempt_enable + __traceiter_android_rvh_sched_fork __traceiter_android_rvh_select_task_rq_rt __traceiter_android_rvh_set_iowait __traceiter_android_rvh_typec_tcpci_chk_contaminant @@ -1841,6 +1931,8 @@ __traceiter_android_vh_pagecache_get_page __traceiter_android_vh_rmqueue __traceiter_android_vh_setscheduler_uclamp + __traceiter_android_vh_snd_compr_use_pause_in_drain + __traceiter_android_vh_sound_usb_support_cpu_suspend __traceiter_android_vh_thermal_pm_notify_suspend __traceiter_android_vh_timerfd_create __traceiter_android_vh_typec_store_partner_src_caps @@ -1891,6 +1983,7 @@ __tracepoint_android_rvh_post_init_entity_util_avg __tracepoint_android_rvh_preempt_disable __tracepoint_android_rvh_preempt_enable + __tracepoint_android_rvh_sched_fork __tracepoint_android_rvh_select_task_rq_rt __tracepoint_android_rvh_set_iowait __tracepoint_android_rvh_typec_tcpci_chk_contaminant @@ -1911,6 +2004,8 @@ __tracepoint_android_vh_pagecache_get_page __tracepoint_android_vh_rmqueue __tracepoint_android_vh_setscheduler_uclamp + __tracepoint_android_vh_snd_compr_use_pause_in_drain + __tracepoint_android_vh_sound_usb_support_cpu_suspend __tracepoint_android_vh_thermal_pm_notify_suspend __tracepoint_android_vh_timerfd_create __tracepoint_android_vh_typec_store_partner_src_caps @@ -2004,8 +2099,10 @@ unregister_inet6addr_notifier unregister_inetaddr_notifier unregister_netdev + unregister_netdevice_many unregister_netdevice_notifier unregister_netdevice_queue + unregister_pernet_device unregister_pernet_subsys unregister_pm_notifier unregister_reboot_notifier @@ -2113,6 +2210,7 @@ vb2_streamon vb2_vmalloc_memops vb2_wait_for_all_buffers + verify_pkcs7_signature vfree video_devdata video_device_alloc @@ -2164,6 +2262,7 @@ watchdog_register_device watchdog_set_restart_priority watchdog_unregister_device + wireless_nlevent_flush woken_wake_function work_busy __xfrm_state_destroy @@ -2178,6 +2277,7 @@ xhci_handle_event xhci_init_driver xhci_resume + xhci_ring_alloc xhci_ring_free xhci_run xhci_suspend diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index 972b6ed9acdd..d39aebce8bea 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -1052,6 +1052,20 @@ i2c_register_driver i2c_transfer i2c_transfer_buffer_flags + i3c_generic_ibi_alloc_pool + i3c_generic_ibi_free_pool + i3c_generic_ibi_get_free_slot + i3c_generic_ibi_recycle_slot + i3c_master_add_i3c_dev_locked + i3c_master_disec_locked + i3c_master_do_daa + i3c_master_enec_locked + i3c_master_entdaa_locked + i3c_master_get_free_addr + i3c_master_queue_ibi + i3c_master_register + i3c_master_set_info + i3c_master_unregister icc_disable icc_enable icc_get @@ -1069,6 +1083,7 @@ ida_free idr_alloc idr_alloc_cyclic + idr_alloc_u32 idr_destroy idr_find idr_for_each @@ -2923,6 +2938,7 @@ vmalloc_to_pfn vmap vmemdup_user + vm_event_states vmf_insert_mixed vmf_insert_pfn vm_get_page_prot diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 77216f65d4ee..9e75fba692ad 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -15,6 +15,7 @@ CONFIG_IKHEADERS=y CONFIG_UCLAMP_TASK=y CONFIG_UCLAMP_BUCKETS_COUNT=20 CONFIG_CGROUPS=y +CONFIG_MEMCG=y CONFIG_BLK_CGROUP=y CONFIG_CGROUP_SCHED=y CONFIG_UCLAMP_TASK_GROUP=y @@ -257,6 +258,7 @@ CONFIG_NET_ACT_GACT=y CONFIG_NET_ACT_MIRRED=y CONFIG_NET_ACT_SKBEDIT=y CONFIG_VSOCKETS=y +CONFIG_CGROUP_NET_PRIO=y CONFIG_BPF_JIT=y CONFIG_CAN=y CONFIG_BT=y diff --git a/arch/arm64/include/asm/asm_pointer_auth.h b/arch/arm64/include/asm/asm_pointer_auth.h index 52dead2a8640..8ca2dc0661ee 100644 --- a/arch/arm64/include/asm/asm_pointer_auth.h +++ b/arch/arm64/include/asm/asm_pointer_auth.h @@ -13,30 +13,12 @@ * so use the base value of ldp as thread.keys_user and offset as * thread.keys_user.ap*. */ - .macro ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3 + .macro __ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3 mov \tmp1, #THREAD_KEYS_USER add \tmp1, \tsk, \tmp1 -alternative_if_not ARM64_HAS_ADDRESS_AUTH - b .Laddr_auth_skip_\@ -alternative_else_nop_endif ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIA] msr_s SYS_APIAKEYLO_EL1, \tmp2 msr_s SYS_APIAKEYHI_EL1, \tmp3 - ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIB] - msr_s SYS_APIBKEYLO_EL1, \tmp2 - msr_s SYS_APIBKEYHI_EL1, \tmp3 - ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APDA] - msr_s SYS_APDAKEYLO_EL1, \tmp2 - msr_s SYS_APDAKEYHI_EL1, \tmp3 - ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APDB] - msr_s SYS_APDBKEYLO_EL1, \tmp2 - msr_s SYS_APDBKEYHI_EL1, \tmp3 -.Laddr_auth_skip_\@: -alternative_if ARM64_HAS_GENERIC_AUTH - ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APGA] - msr_s SYS_APGAKEYLO_EL1, \tmp2 - msr_s SYS_APGAKEYHI_EL1, \tmp3 -alternative_else_nop_endif .endm .macro __ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3 diff --git a/arch/arm64/include/asm/lse.h b/arch/arm64/include/asm/lse.h index 5d10051c3e62..29d0719bd2ed 100644 --- a/arch/arm64/include/asm/lse.h +++ b/arch/arm64/include/asm/lse.h @@ -4,7 +4,7 @@ #include -#ifdef CONFIG_ARM64_LSE_ATOMICS +#if defined(CONFIG_ARM64_LSE_ATOMICS) && !defined(BUILD_FIPS140_KO) #define __LSE_PREAMBLE ".arch_extension lse\n" diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 8bdf2cc006bc..67bf259ae768 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -40,7 +40,7 @@ void mte_free_tag_storage(char *storage); void mte_zero_clear_page_tags(void *addr); void mte_sync_tags(pte_t *ptep, pte_t pte); void mte_copy_page_tags(void *kto, const void *kfrom); -void flush_mte_state(void); +void mte_thread_init_user(void); void mte_thread_switch(struct task_struct *next); void mte_suspend_enter(void); void mte_suspend_exit(void); @@ -63,7 +63,7 @@ static inline void mte_sync_tags(pte_t *ptep, pte_t pte) static inline void mte_copy_page_tags(void *kto, const void *kfrom) { } -static inline void flush_mte_state(void) +static inline void mte_thread_init_user(void) { } static inline void mte_thread_switch(struct task_struct *next) diff --git a/arch/arm64/include/asm/pointer_auth.h b/arch/arm64/include/asm/pointer_auth.h index b112a11e9302..592968f0bc22 100644 --- a/arch/arm64/include/asm/pointer_auth.h +++ b/arch/arm64/include/asm/pointer_auth.h @@ -3,12 +3,16 @@ #define __ASM_POINTER_AUTH_H #include +#include #include #include #include #include +#define PR_PAC_ENABLED_KEYS_MASK \ + (PR_PAC_APIAKEY | PR_PAC_APIBKEY | PR_PAC_APDAKEY | PR_PAC_APDBKEY) + #ifdef CONFIG_ARM64_PTR_AUTH /* * Each key is a 128-bit quantity which is split across a pair of 64-bit @@ -34,6 +38,25 @@ struct ptrauth_keys_kernel { struct ptrauth_key apia; }; +#define __ptrauth_key_install_nosync(k, v) \ +do { \ + struct ptrauth_key __pki_v = (v); \ + write_sysreg_s(__pki_v.lo, SYS_ ## k ## KEYLO_EL1); \ + write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1); \ +} while (0) + +static inline void ptrauth_keys_install_user(struct ptrauth_keys_user *keys) +{ + if (system_supports_address_auth()) { + __ptrauth_key_install_nosync(APIB, keys->apib); + __ptrauth_key_install_nosync(APDA, keys->apda); + __ptrauth_key_install_nosync(APDB, keys->apdb); + } + + if (system_supports_generic_auth()) + __ptrauth_key_install_nosync(APGA, keys->apga); +} + static inline void ptrauth_keys_init_user(struct ptrauth_keys_user *keys) { if (system_supports_address_auth()) { @@ -45,14 +68,9 @@ static inline void ptrauth_keys_init_user(struct ptrauth_keys_user *keys) if (system_supports_generic_auth()) get_random_bytes(&keys->apga, sizeof(keys->apga)); -} -#define __ptrauth_key_install_nosync(k, v) \ -do { \ - struct ptrauth_key __pki_v = (v); \ - write_sysreg_s(__pki_v.lo, SYS_ ## k ## KEYLO_EL1); \ - write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1); \ -} while (0) + ptrauth_keys_install_user(keys); +} static __always_inline void ptrauth_keys_init_kernel(struct ptrauth_keys_kernel *keys) { @@ -71,6 +89,10 @@ static __always_inline void ptrauth_keys_switch_kernel(struct ptrauth_keys_kerne extern int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg); +extern int ptrauth_set_enabled_keys(struct task_struct *tsk, unsigned long keys, + unsigned long enabled); +extern int ptrauth_get_enabled_keys(struct task_struct *tsk); + static inline unsigned long ptrauth_strip_insn_pac(unsigned long ptr) { return ptrauth_clear_pac(ptr); @@ -85,8 +107,23 @@ static __always_inline void ptrauth_enable(void) isb(); } -#define ptrauth_thread_init_user(tsk) \ - ptrauth_keys_init_user(&(tsk)->thread.keys_user) +#define ptrauth_suspend_exit() \ + ptrauth_keys_install_user(¤t->thread.keys_user) + +#define ptrauth_thread_init_user() \ + do { \ + ptrauth_keys_init_user(¤t->thread.keys_user); \ + \ + /* enable all keys */ \ + if (system_supports_address_auth()) \ + ptrauth_set_enabled_keys(current, \ + PR_PAC_ENABLED_KEYS_MASK, \ + PR_PAC_ENABLED_KEYS_MASK); \ + } while (0) + +#define ptrauth_thread_switch_user(tsk) \ + ptrauth_keys_install_user(&(tsk)->thread.keys_user) + #define ptrauth_thread_init_kernel(tsk) \ ptrauth_keys_init_kernel(&(tsk)->thread.keys_kernel) #define ptrauth_thread_switch_kernel(tsk) \ @@ -95,9 +132,13 @@ static __always_inline void ptrauth_enable(void) #else /* CONFIG_ARM64_PTR_AUTH */ #define ptrauth_enable() #define ptrauth_prctl_reset_keys(tsk, arg) (-EINVAL) +#define ptrauth_set_enabled_keys(tsk, keys, enabled) (-EINVAL) +#define ptrauth_get_enabled_keys(tsk) (-EINVAL) #define ptrauth_strip_insn_pac(lr) (lr) -#define ptrauth_thread_init_user(tsk) +#define ptrauth_suspend_exit() +#define ptrauth_thread_init_user() #define ptrauth_thread_init_kernel(tsk) +#define ptrauth_thread_switch_user(tsk) #define ptrauth_thread_switch_kernel(tsk) #endif /* CONFIG_ARM64_PTR_AUTH */ diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index e1ad8bb0c79d..e85a712f1e3d 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -19,6 +19,12 @@ */ #define NET_IP_ALIGN 0 +#define MTE_CTRL_GCR_USER_EXCL_SHIFT 0 +#define MTE_CTRL_GCR_USER_EXCL_MASK 0xffff + +#define MTE_CTRL_TCF_SYNC (1UL << 16) +#define MTE_CTRL_TCF_ASYNC (1UL << 17) + #ifndef __ASSEMBLY__ #include @@ -156,12 +162,14 @@ struct thread_struct { struct ptrauth_keys_kernel keys_kernel; #endif #ifdef CONFIG_ARM64_MTE - u64 gcr_user_excl; + u64 mte_ctrl; #endif u64 sctlr_user; }; -#define SCTLR_USER_MASK SCTLR_EL1_TCF0_MASK +#define SCTLR_USER_MASK \ + (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | SCTLR_ELx_ENDA | SCTLR_ELx_ENDB | \ + SCTLR_EL1_TCF0_MASK) static inline void arch_thread_struct_whitelist(unsigned long *offset, unsigned long *size) @@ -254,7 +262,7 @@ extern void release_thread(struct task_struct *); unsigned long get_wchan(struct task_struct *p); -void set_task_sctlr_el1(u64 sctlr); +void update_sctlr_el1(u64 sctlr); /* Thread switching */ extern struct task_struct *cpu_switch_to(struct task_struct *prev, @@ -310,6 +318,11 @@ extern void __init minsigstksz_setup(void); /* PR_PAC_RESET_KEYS prctl */ #define PAC_RESET_KEYS(tsk, arg) ptrauth_prctl_reset_keys(tsk, arg) +/* PR_PAC_{SET,GET}_ENABLED_KEYS prctl */ +#define PAC_SET_ENABLED_KEYS(tsk, keys, enabled) \ + ptrauth_set_enabled_keys(tsk, keys, enabled) +#define PAC_GET_ENABLED_KEYS(tsk) ptrauth_get_enabled_keys(tsk) + #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI /* PR_{SET,GET}_TAGGED_ADDR_CTRL prctl */ long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg); diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 1093196db31d..6201de6fdaac 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -610,8 +610,10 @@ #define SCTLR_ELx_TCF_ASYNC (UL(0x2) << SCTLR_ELx_TCF_SHIFT) #define SCTLR_ELx_TCF_MASK (UL(0x3) << SCTLR_ELx_TCF_SHIFT) +#define SCTLR_ELx_ENIA_SHIFT 31 + #define SCTLR_ELx_ITFSB (BIT(37)) -#define SCTLR_ELx_ENIA (BIT(31)) +#define SCTLR_ELx_ENIA (BIT(SCTLR_ELx_ENIA_SHIFT)) #define SCTLR_ELx_ENIB (BIT(30)) #define SCTLR_ELx_ENDA (BIT(27)) #define SCTLR_ELx_EE (BIT(25)) diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index f396d951ee54..93da876a58e6 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -44,12 +44,13 @@ int main(void) #endif BLANK(); DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context)); + DEFINE(THREAD_SCTLR_USER, offsetof(struct task_struct, thread.sctlr_user)); #ifdef CONFIG_ARM64_PTR_AUTH DEFINE(THREAD_KEYS_USER, offsetof(struct task_struct, thread.keys_user)); DEFINE(THREAD_KEYS_KERNEL, offsetof(struct task_struct, thread.keys_kernel)); #endif #ifdef CONFIG_ARM64_MTE - DEFINE(THREAD_GCR_EL1_USER, offsetof(struct task_struct, thread.gcr_user_excl)); + DEFINE(THREAD_MTE_CTRL, offsetof(struct task_struct, thread.mte_ctrl)); #endif BLANK(); DEFINE(S_X0, offsetof(struct pt_regs, regs[0])); @@ -153,10 +154,6 @@ int main(void) #endif #ifdef CONFIG_ARM64_PTR_AUTH DEFINE(PTRAUTH_USER_KEY_APIA, offsetof(struct ptrauth_keys_user, apia)); - DEFINE(PTRAUTH_USER_KEY_APIB, offsetof(struct ptrauth_keys_user, apib)); - DEFINE(PTRAUTH_USER_KEY_APDA, offsetof(struct ptrauth_keys_user, apda)); - DEFINE(PTRAUTH_USER_KEY_APDB, offsetof(struct ptrauth_keys_user, apdb)); - DEFINE(PTRAUTH_USER_KEY_APGA, offsetof(struct ptrauth_keys_user, apga)); DEFINE(PTRAUTH_KERNEL_KEY_APIA, offsetof(struct ptrauth_keys_kernel, apia)); BLANK(); #endif diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 476caeb811ff..5369db4b53a4 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -148,42 +148,46 @@ alternative_cb_end .endm /* Check for MTE asynchronous tag check faults */ - .macro check_mte_async_tcf, tmp, ti_flags + .macro check_mte_async_tcf, tmp, ti_flags, thread_sctlr #ifdef CONFIG_ARM64_MTE .arch_extension lse alternative_if_not ARM64_MTE b 1f alternative_else_nop_endif + /* + * Asynchronous tag check faults are only possible in ASYNC (2) or + * ASYM (3) modes. In each of these modes bit 1 of SCTLR_EL1.TCF0 is + * set, so skip the check if it is unset. + */ + tbz \thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f mrs_s \tmp, SYS_TFSRE0_EL1 tbz \tmp, #SYS_TFSR_EL1_TF0_SHIFT, 1f /* Asynchronous TCF occurred for TTBR0 access, set the TI flag */ mov \tmp, #_TIF_MTE_ASYNC_FAULT add \ti_flags, tsk, #TSK_TI_FLAGS stset \tmp, [\ti_flags] - msr_s SYS_TFSRE0_EL1, xzr 1: #endif .endm /* Clear the MTE asynchronous tag check faults */ - .macro clear_mte_async_tcf + .macro clear_mte_async_tcf thread_sctlr #ifdef CONFIG_ARM64_MTE alternative_if ARM64_MTE + /* See comment in check_mte_async_tcf above. */ + tbz \thread_sctlr, #(SCTLR_EL1_TCF0_SHIFT + 1), 1f dsb ish msr_s SYS_TFSRE0_EL1, xzr +1: alternative_else_nop_endif #endif .endm - .macro mte_set_gcr, tmp, tmp2 + .macro mte_set_gcr, mte_ctrl, tmp #ifdef CONFIG_ARM64_MTE - /* - * Calculate and set the exclude mask preserving - * the RRND (bit[16]) setting. - */ - mrs_s \tmp2, SYS_GCR_EL1 - bfi \tmp2, \tmp, #0, #16 - msr_s SYS_GCR_EL1, \tmp2 + ubfx \tmp, \mte_ctrl, #MTE_CTRL_GCR_USER_EXCL_SHIFT, #16 + orr \tmp, \tmp, #SYS_GCR_EL1_RRND + msr_s SYS_GCR_EL1, \tmp #endif .endm @@ -195,7 +199,6 @@ alternative_else_nop_endif ldr_l \tmp, gcr_kernel_excl mte_set_gcr \tmp, \tmp2 - isb 1: #endif .endm @@ -205,7 +208,7 @@ alternative_else_nop_endif alternative_if_not ARM64_MTE b 1f alternative_else_nop_endif - ldr \tmp, [\tsk, #THREAD_GCR_EL1_USER] + ldr \tmp, [\tsk, #THREAD_MTE_CTRL] mte_set_gcr \tmp, \tmp2 1: @@ -246,13 +249,48 @@ alternative_else_nop_endif disable_step_tsk x19, x20 /* Check for asynchronous tag check faults in user space */ - check_mte_async_tcf x22, x23 + ldr x0, [tsk, THREAD_SCTLR_USER] + check_mte_async_tcf x22, x23, x0 + +#ifdef CONFIG_ARM64_PTR_AUTH +alternative_if ARM64_HAS_ADDRESS_AUTH + /* + * Enable IA for in-kernel PAC if the task had it disabled. Although + * this could be implemented with an unconditional MRS which would avoid + * a load, this was measured to be slower on Cortex-A75 and Cortex-A76. + * + * Install the kernel IA key only if IA was enabled in the task. If IA + * was disabled on kernel exit then we would have left the kernel IA + * installed so there is no need to install it again. + */ + tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f + __ptrauth_keys_install_kernel_nosync tsk, x20, x22, x23 + b 2f +1: + mrs x0, sctlr_el1 + orr x0, x0, SCTLR_ELx_ENIA + msr sctlr_el1, x0 +2: +alternative_else_nop_endif +#endif + apply_ssbd 1, x22, x23 - ptrauth_keys_install_kernel tsk, x20, x22, x23 - mte_set_kernel_gcr x22, x23 + /* + * Any non-self-synchronizing system register updates required for + * kernel entry should be placed before this point. + */ +alternative_if ARM64_MTE + isb + b 1f +alternative_else_nop_endif +alternative_if ARM64_HAS_ADDRESS_AUTH + isb +alternative_else_nop_endif +1: + scs_load tsk, x20 .else add x21, sp, #S_FRAME_SIZE @@ -367,8 +405,29 @@ alternative_else_nop_endif 3: scs_save tsk, x0 - /* No kernel C function calls after this as user keys are set. */ - ptrauth_keys_install_user tsk, x0, x1, x2 + /* Ignore asynchronous tag check faults in the uaccess routines */ + ldr x0, [tsk, THREAD_SCTLR_USER] + clear_mte_async_tcf x0 + +#ifdef CONFIG_ARM64_PTR_AUTH +alternative_if ARM64_HAS_ADDRESS_AUTH + /* + * IA was enabled for in-kernel PAC. Disable it now if needed, or + * alternatively install the user's IA. All other per-task keys and + * SCTLR bits were updated on task switch. + * + * No kernel C function calls after this. + */ + tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f + __ptrauth_keys_install_user tsk, x0, x1, x2 + b 2f +1: + mrs x0, sctlr_el1 + bic x0, x0, SCTLR_ELx_ENIA + msr sctlr_el1, x0 +2: +alternative_else_nop_endif +#endif mte_set_user_gcr tsk, x0, x1 @@ -784,8 +843,6 @@ SYM_CODE_START_LOCAL(ret_to_user) cbnz x2, work_pending finish_ret_to_user: user_enter_irqoff - /* Ignore asynchronous tag check faults in the uaccess routines */ - clear_mte_async_tcf enable_step_tsk x19, x2 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK bl stackleak_erase diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 8da597e41795..50ae1a7435ae 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -4,6 +4,7 @@ */ #include +#include #include #include #include @@ -26,6 +27,8 @@ u64 gcr_kernel_excl __ro_after_init; static bool report_fault_once = true; +static DEFINE_PER_CPU_READ_MOSTLY(u64, mte_tcf_preferred); + #ifdef CONFIG_KASAN_HW_TAGS /* Whether the MTE asynchronous mode is enabled. */ DEFINE_STATIC_KEY_FALSE(mte_async_mode); @@ -197,17 +200,29 @@ static void update_gcr_el1_excl(u64 excl) sysreg_clear_set_s(SYS_GCR_EL1, SYS_GCR_EL1_EXCL_MASK, excl); } -static void set_gcr_el1_excl(u64 excl) +static void mte_update_sctlr_user(struct task_struct *task) { - current->thread.gcr_user_excl = excl; - /* - * SYS_GCR_EL1 will be set to current->thread.gcr_user_excl value - * by mte_set_user_gcr() in kernel_exit, + * This must be called with preemption disabled and can only be called + * on the current or next task since the CPU must match where the thread + * is going to run. The caller is responsible for calling + * update_sctlr_el1() later in the same preemption disabled block. */ + unsigned long sctlr = task->thread.sctlr_user; + unsigned long mte_ctrl = task->thread.mte_ctrl; + unsigned long pref, resolved_mte_tcf; + + pref = __this_cpu_read(mte_tcf_preferred); + resolved_mte_tcf = (mte_ctrl & pref) ? pref : mte_ctrl; + sctlr &= ~SCTLR_EL1_TCF0_MASK; + if (resolved_mte_tcf & MTE_CTRL_TCF_ASYNC) + sctlr |= SCTLR_EL1_TCF0_ASYNC; + else if (resolved_mte_tcf & MTE_CTRL_TCF_SYNC) + sctlr |= SCTLR_EL1_TCF0_SYNC; + task->thread.sctlr_user = sctlr; } -void flush_mte_state(void) +void mte_thread_init_user(void) { if (!system_supports_mte()) return; @@ -216,15 +231,14 @@ void flush_mte_state(void) dsb(ish); write_sysreg_s(0, SYS_TFSRE0_EL1); clear_thread_flag(TIF_MTE_ASYNC_FAULT); - /* disable tag checking */ - set_task_sctlr_el1((current->thread.sctlr_user & ~SCTLR_EL1_TCF0_MASK) | - SCTLR_EL1_TCF0_NONE); - /* reset tag generation mask */ - set_gcr_el1_excl(SYS_GCR_EL1_EXCL_MASK); + /* disable tag checking and reset tag generation mask */ + set_mte_ctrl(current, 0); } void mte_thread_switch(struct task_struct *next) { + mte_update_sctlr_user(next); + /* * Check if an async tag exception occurred at EL1. * @@ -262,33 +276,23 @@ void mte_suspend_exit(void) long set_mte_ctrl(struct task_struct *task, unsigned long arg) { - u64 sctlr = task->thread.sctlr_user & ~SCTLR_EL1_TCF0_MASK; - u64 gcr_excl = ~((arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT) & - SYS_GCR_EL1_EXCL_MASK; + u64 mte_ctrl = (~((arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT) & + SYS_GCR_EL1_EXCL_MASK) << MTE_CTRL_GCR_USER_EXCL_SHIFT; if (!system_supports_mte()) return 0; - switch (arg & PR_MTE_TCF_MASK) { - case PR_MTE_TCF_NONE: - sctlr |= SCTLR_EL1_TCF0_NONE; - break; - case PR_MTE_TCF_SYNC: - sctlr |= SCTLR_EL1_TCF0_SYNC; - break; - case PR_MTE_TCF_ASYNC: - sctlr |= SCTLR_EL1_TCF0_ASYNC; - break; - default: - return -EINVAL; - } + if (arg & PR_MTE_TCF_ASYNC) + mte_ctrl |= MTE_CTRL_TCF_ASYNC; + if (arg & PR_MTE_TCF_SYNC) + mte_ctrl |= MTE_CTRL_TCF_SYNC; - if (task != current) { - task->thread.sctlr_user = sctlr; - task->thread.gcr_user_excl = gcr_excl; - } else { - set_task_sctlr_el1(sctlr); - set_gcr_el1_excl(gcr_excl); + task->thread.mte_ctrl = mte_ctrl; + if (task == current) { + preempt_disable(); + mte_update_sctlr_user(task); + update_sctlr_el1(task->thread.sctlr_user); + preempt_enable(); } return 0; @@ -297,24 +301,18 @@ long set_mte_ctrl(struct task_struct *task, unsigned long arg) long get_mte_ctrl(struct task_struct *task) { unsigned long ret; - u64 incl = ~task->thread.gcr_user_excl & SYS_GCR_EL1_EXCL_MASK; + u64 mte_ctrl = task->thread.mte_ctrl; + u64 incl = (~mte_ctrl >> MTE_CTRL_GCR_USER_EXCL_SHIFT) & + SYS_GCR_EL1_EXCL_MASK; if (!system_supports_mte()) return 0; ret = incl << PR_MTE_TAG_SHIFT; - - switch (task->thread.sctlr_user & SCTLR_EL1_TCF0_MASK) { - case SCTLR_EL1_TCF0_NONE: - ret |= PR_MTE_TCF_NONE; - break; - case SCTLR_EL1_TCF0_SYNC: - ret |= PR_MTE_TCF_SYNC; - break; - case SCTLR_EL1_TCF0_ASYNC: + if (mte_ctrl & MTE_CTRL_TCF_ASYNC) ret |= PR_MTE_TCF_ASYNC; - break; - } + if (mte_ctrl & MTE_CTRL_TCF_SYNC) + ret |= PR_MTE_TCF_SYNC; return ret; } @@ -453,3 +451,54 @@ int mte_ptrace_copy_tags(struct task_struct *child, long request, return ret; } + +static ssize_t mte_tcf_preferred_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + switch (per_cpu(mte_tcf_preferred, dev->id)) { + case MTE_CTRL_TCF_ASYNC: + return sysfs_emit(buf, "async\n"); + case MTE_CTRL_TCF_SYNC: + return sysfs_emit(buf, "sync\n"); + default: + return sysfs_emit(buf, "???\n"); + } +} + +static ssize_t mte_tcf_preferred_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + u64 tcf; + + if (sysfs_streq(buf, "async")) + tcf = MTE_CTRL_TCF_ASYNC; + else if (sysfs_streq(buf, "sync")) + tcf = MTE_CTRL_TCF_SYNC; + else + return -EINVAL; + + device_lock(dev); + per_cpu(mte_tcf_preferred, dev->id) = tcf; + device_unlock(dev); + + return count; +} +static DEVICE_ATTR_RW(mte_tcf_preferred); + +static int register_mte_tcf_preferred_sysctl(void) +{ + unsigned int cpu; + + if (!system_supports_mte()) + return 0; + + for_each_possible_cpu(cpu) { + per_cpu(mte_tcf_preferred, cpu) = MTE_CTRL_TCF_ASYNC; + device_create_file(get_cpu_device(cpu), + &dev_attr_mte_tcf_preferred); + } + + return 0; +} +subsys_initcall(register_mte_tcf_preferred_sysctl); diff --git a/arch/arm64/kernel/pointer_auth.c b/arch/arm64/kernel/pointer_auth.c index adb955fd9bdd..2708b620b4ae 100644 --- a/arch/arm64/kernel/pointer_auth.c +++ b/arch/arm64/kernel/pointer_auth.c @@ -43,6 +43,71 @@ int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg) get_random_bytes(&keys->apdb, sizeof(keys->apdb)); if (arg & PR_PAC_APGAKEY) get_random_bytes(&keys->apga, sizeof(keys->apga)); + ptrauth_keys_install_user(keys); return 0; } + +static u64 arg_to_enxx_mask(unsigned long arg) +{ + u64 sctlr_enxx_mask = 0; + + WARN_ON(arg & ~PR_PAC_ENABLED_KEYS_MASK); + if (arg & PR_PAC_APIAKEY) + sctlr_enxx_mask |= SCTLR_ELx_ENIA; + if (arg & PR_PAC_APIBKEY) + sctlr_enxx_mask |= SCTLR_ELx_ENIB; + if (arg & PR_PAC_APDAKEY) + sctlr_enxx_mask |= SCTLR_ELx_ENDA; + if (arg & PR_PAC_APDBKEY) + sctlr_enxx_mask |= SCTLR_ELx_ENDB; + return sctlr_enxx_mask; +} + +int ptrauth_set_enabled_keys(struct task_struct *tsk, unsigned long keys, + unsigned long enabled) +{ + u64 sctlr; + + if (!system_supports_address_auth()) + return -EINVAL; + + if (is_compat_thread(task_thread_info(tsk))) + return -EINVAL; + + if ((keys & ~PR_PAC_ENABLED_KEYS_MASK) || (enabled & ~keys)) + return -EINVAL; + + preempt_disable(); + sctlr = tsk->thread.sctlr_user; + sctlr &= ~arg_to_enxx_mask(keys); + sctlr |= arg_to_enxx_mask(enabled); + tsk->thread.sctlr_user = sctlr; + if (tsk == current) + update_sctlr_el1(sctlr); + preempt_enable(); + + return 0; +} + +int ptrauth_get_enabled_keys(struct task_struct *tsk) +{ + int retval = 0; + + if (!system_supports_address_auth()) + return -EINVAL; + + if (is_compat_thread(task_thread_info(tsk))) + return -EINVAL; + + if (tsk->thread.sctlr_user & SCTLR_ELx_ENIA) + retval |= PR_PAC_APIAKEY; + if (tsk->thread.sctlr_user & SCTLR_ELx_ENIB) + retval |= PR_PAC_APIBKEY; + if (tsk->thread.sctlr_user & SCTLR_ELx_ENDA) + retval |= PR_PAC_APDAKEY; + if (tsk->thread.sctlr_user & SCTLR_ELx_ENDB) + retval |= PR_PAC_APDBKEY; + + return retval; +} diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 9f5bfd1afb25..132261037b42 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -337,7 +337,6 @@ void flush_thread(void) tls_thread_flush(); flush_ptrace_hw_breakpoint(current); flush_tagged_addr_state(); - flush_mte_state(); } void release_thread(struct task_struct *dead_task) @@ -538,27 +537,24 @@ static void erratum_1418040_thread_switch(struct task_struct *prev, write_sysreg(val, cntkctl_el1); } -static void update_sctlr_el1(u64 sctlr) +/* + * __switch_to() checks current->thread.sctlr_user as an optimisation. Therefore + * this function must be called with preemption disabled and the update to + * sctlr_user must be made in the same preemption disabled block so that + * __switch_to() does not see the variable update before the SCTLR_EL1 one. + */ +void update_sctlr_el1(u64 sctlr) { - sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK, sctlr); + /* + * EnIA must not be cleared while in the kernel as this is necessary for + * in-kernel PAC. It will be cleared on kernel exit if needed. + */ + sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr); /* ISB required for the kernel uaccess routines when setting TCF0. */ isb(); } -void set_task_sctlr_el1(u64 sctlr) -{ - /* - * __switch_to() checks current->thread.sctlr as an - * optimisation. Disable preemption so that it does not see - * the variable update before the SCTLR_EL1 one. - */ - preempt_disable(); - current->thread.sctlr_user = sctlr; - update_sctlr_el1(sctlr); - preempt_enable(); -} - /* * Thread switching. */ @@ -575,6 +571,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev, uao_thread_switch(next); ssbs_thread_switch(next); erratum_1418040_thread_switch(prev, next); + ptrauth_thread_switch_user(next); /* * vendor hook is needed before the dsb(), * because MPAM is related to cache maintenance. @@ -666,7 +663,8 @@ void arch_setup_new_exec(void) } current->mm->context.flags = mmflags; - ptrauth_thread_init_user(current); + ptrauth_thread_init_user(); + mte_thread_init_user(); if (task_spec_ssb_noexec(current)) { arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS, diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 1d75471979cb..7d84854bc0e8 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -908,6 +908,38 @@ static int pac_mask_get(struct task_struct *target, return membuf_write(&to, &uregs, sizeof(uregs)); } +static int pac_enabled_keys_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + long enabled_keys = ptrauth_get_enabled_keys(target); + + if (IS_ERR_VALUE(enabled_keys)) + return enabled_keys; + + return membuf_write(&to, &enabled_keys, sizeof(enabled_keys)); +} + +static int pac_enabled_keys_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int ret; + long enabled_keys = ptrauth_get_enabled_keys(target); + + if (IS_ERR_VALUE(enabled_keys)) + return enabled_keys; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &enabled_keys, 0, + sizeof(long)); + if (ret) + return ret; + + return ptrauth_set_enabled_keys(target, PR_PAC_ENABLED_KEYS_MASK, + enabled_keys); +} + #ifdef CONFIG_CHECKPOINT_RESTORE static __uint128_t pac_key_to_user(const struct ptrauth_key *key) { @@ -1073,6 +1105,7 @@ enum aarch64_regset { #endif #ifdef CONFIG_ARM64_PTR_AUTH REGSET_PAC_MASK, + REGSET_PAC_ENABLED_KEYS, #ifdef CONFIG_CHECKPOINT_RESTORE REGSET_PACA_KEYS, REGSET_PACG_KEYS, @@ -1159,6 +1192,14 @@ static const struct user_regset aarch64_regsets[] = { .regset_get = pac_mask_get, /* this cannot be set dynamically */ }, + [REGSET_PAC_ENABLED_KEYS] = { + .core_note_type = NT_ARM_PAC_ENABLED_KEYS, + .n = 1, + .size = sizeof(long), + .align = sizeof(long), + .regset_get = pac_enabled_keys_get, + .set = pac_enabled_keys_set, + }, #ifdef CONFIG_CHECKPOINT_RESTORE [REGSET_PACA_KEYS] = { .core_note_type = NT_ARM_PACA_KEYS, diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index adf86a26e252..1e8dc6f7d178 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -75,8 +75,9 @@ void notrace __cpu_suspend_exit(void) */ spectre_v4_enable_mitigation(NULL); - /* Restore additional MTE-specific configuration */ + /* Restore additional feature-specific configuration */ mte_suspend_exit(); + ptrauth_suspend_exit(); } /* diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index 18c2c6f34b45..c4f89cd04566 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -17,6 +17,7 @@ CONFIG_IKHEADERS=y CONFIG_UCLAMP_TASK=y CONFIG_UCLAMP_BUCKETS_COUNT=20 CONFIG_CGROUPS=y +CONFIG_MEMCG=y CONFIG_BLK_CGROUP=y CONFIG_CGROUP_SCHED=y CONFIG_UCLAMP_TASK_GROUP=y @@ -233,6 +234,7 @@ CONFIG_NET_ACT_GACT=y CONFIG_NET_ACT_MIRRED=y CONFIG_NET_ACT_SKBEDIT=y CONFIG_VSOCKETS=y +CONFIG_CGROUP_NET_PRIO=y CONFIG_BPF_JIT=y CONFIG_CAN=y CONFIG_BT=y diff --git a/block/blk-core.c b/block/blk-core.c index 2d53e2ff48ff..a00bce9f46d8 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -441,8 +440,7 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) * responsible for ensuring that that counter is * globally visible before the queue is unfrozen. */ - if ((pm && queue_rpm_status(q) != RPM_SUSPENDED) || - !blk_queue_pm_only(q)) { + if (pm || !blk_queue_pm_only(q)) { success = true; } else { percpu_ref_put(&q->q_usage_counter); @@ -467,7 +465,8 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) wait_event(q->mq_freeze_wq, (!q->mq_freeze_depth && - blk_pm_resume_queue(pm, q)) || + (pm || (blk_pm_request_resume(q), + !blk_queue_pm_only(q)))) || blk_queue_dying(q)); if (blk_queue_dying(q)) return -ENODEV; diff --git a/block/blk-mq.c b/block/blk-mq.c index 4590fe39b69b..a4343848b835 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -41,6 +41,8 @@ #include "blk-mq-sched.h" #include "blk-rq-qos.h" +#include + static DEFINE_PER_CPU(struct list_head, blk_cpu_done); static void blk_mq_poll_stats_start(struct request_queue *q); @@ -341,6 +343,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, } data->hctx->queued++; + trace_android_vh_blk_rq_ctx_init(rq, tags, data, alloc_time_ns); return rq; } @@ -2480,6 +2483,7 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, */ rq_size = round_up(sizeof(struct request) + set->cmd_size, cache_line_size()); + trace_android_vh_blk_alloc_rqs(&rq_size, set, tags); left = rq_size * depth; for (i = 0; i < depth; ) { diff --git a/block/blk-pm.h b/block/blk-pm.h index a2283cc9f716..ea5507d23e75 100644 --- a/block/blk-pm.h +++ b/block/blk-pm.h @@ -6,14 +6,11 @@ #include #ifdef CONFIG_PM -static inline int blk_pm_resume_queue(const bool pm, struct request_queue *q) +static inline void blk_pm_request_resume(struct request_queue *q) { - if (!q->dev || !blk_queue_pm_only(q)) - return 1; /* Nothing to do */ - if (pm && q->rpm_status != RPM_SUSPENDED) - return 1; /* Request allowed */ - pm_request_resume(q->dev); - return 0; + if (q->dev && (q->rpm_status == RPM_SUSPENDED || + q->rpm_status == RPM_SUSPENDING)) + pm_request_resume(q->dev); } static inline void blk_pm_mark_last_busy(struct request *rq) @@ -47,9 +44,8 @@ static inline void blk_pm_put_request(struct request *rq) --rq->q->nr_pending; } #else -static inline int blk_pm_resume_queue(const bool pm, struct request_queue *q) +static inline void blk_pm_request_resume(struct request_queue *q) { - return 1; } static inline void blk_pm_mark_last_busy(struct request *rq) diff --git a/build.config.common b/build.config.common index 778c4a7af699..b321ae20ecea 100644 --- a/build.config.common +++ b/build.config.common @@ -1,5 +1,5 @@ BRANCH=android12-5.10 -KMI_GENERATION=8 +KMI_GENERATION=9 LLVM=1 DEPMOD=depmod diff --git a/crypto/Kconfig b/crypto/Kconfig index ca514e82431a..d726b1c3a7f8 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -32,13 +32,35 @@ config CRYPTO_FIPS certification. You should say no unless you know what this is. +# CRYPTO_FIPS140 just enables the support in the kernel for loading fips140.ko. +# The module still needs to be built and loaded if you need FIPS 140 compliance. config CRYPTO_FIPS140 def_bool y depends on MODULES && ARM64 && ARM64_MODULE_PLTS config CRYPTO_FIPS140_MOD - bool "Enable FIPS140 integrity self-checked loadable module" + bool "Enable FIPS 140 cryptographic module" depends on LTO_CLANG && CRYPTO_FIPS140 + help + This option enables building a loadable module fips140.ko, which + contains various crypto algorithms that are also built into vmlinux. + At load time, this module overrides the built-in implementations of + these algorithms with its implementations. It also runs self-tests on + these algorithms and verifies the integrity of its code and data. If + either of these steps fails, the kernel will panic. + + This module is intended to be loaded at early boot time in order to + meet FIPS 140 and NIAP FPT_TST_EXT.1 requirements. It shouldn't be + used if you don't need to meet these requirements. + +config CRYPTO_FIPS140_MOD_ERROR_INJECTION + bool "Support injecting failures into the FIPS 140 self-tests" + depends on CRYPTO_FIPS140_MOD + help + This option adds a module parameter "broken_alg" to the fips140 module + which can be used to fail the self-tests for a particular algorithm, + causing a kernel panic. This option is for FIPS lab testing only, and + it shouldn't be enabled on production systems. config CRYPTO_ALGAPI tristate diff --git a/crypto/Makefile b/crypto/Makefile index 63874cb2d4d4..aff52de1f84b 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -228,7 +228,7 @@ $(obj)/lib-crypto-%-fips.o: $(srctree)/lib/crypto/%.c FORCE $(obj)/crypto-fips.a: $(addprefix $(obj)/,$(crypto-fips-objs)) FORCE $(call if_changed,ar_and_symver) -fips140-objs := fips140-module.o crypto-fips.a +fips140-objs := fips140-module.o fips140-selftests.o crypto-fips.a obj-m += fips140.o CFLAGS_fips140-module.o += $(FIPS140_CFLAGS) diff --git a/crypto/fips140-generated-testvecs.h b/crypto/fips140-generated-testvecs.h new file mode 100644 index 000000000000..9328e9bacca7 --- /dev/null +++ b/crypto/fips140-generated-testvecs.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2021 Google LLC */ + +/* + * This header was automatically generated by gen_fips140_testvecs.py. + * Don't edit it directly. + */ + +static const u8 fips_message[32] __initconst = + "This is a 32-byte test message."; + +static const u8 fips_aes_key[16] __initconst = "128-bit AES key"; + +static const u8 fips_aes_iv[16] __initconst = "ABCDEFGHIJKL"; + +static const u8 fips_aes_cbc_ciphertext[32] __initconst = + "\xc4\x6d\xad\xa4\x04\x52\x11\x5a\x7a\xb3\x7c\x68\x85\x8d\x90\xf0" + "\x55\xc3\xd3\x35\xc1\x75\x31\x90\xdf\x90\x4b\x5a\x56\xfd\xa7\x89"; + +static const u8 fips_aes_ecb_ciphertext[32] __initconst = + "\xc1\x9d\xe6\xb8\xb2\x90\xff\xfe\xf2\x77\x18\xb0\x55\xd3\xee\xa9" + "\xe2\x6f\x4a\x32\x67\xfd\xb7\xa5\x2f\x4b\x6e\x1a\x86\x2b\x6e\x3a"; + +static const u8 fips_aes_ctr_ciphertext[32] __initconst = + "\x92\xbe\x23\xa1\x80\x88\x5d\x31\x27\xb3\x9c\x40\x58\x57\x1d\xde" + "\xc1\x8d\x5b\xe7\x42\x93\x09\xf8\xd4\xf7\x49\x42\xcf\x40\x62\x7e"; + +static const u8 fips_aes_gcm_assoc[22] __initconst = "associated data string"; + +static const u8 fips_aes_gcm_ciphertext[48] __initconst = + "\x37\x88\x3e\x1d\x58\x50\xda\x10\x07\xeb\x52\xdf\xea\x0a\x54\xd4" + "\x44\xbf\x88\x2a\xf3\x03\x03\x84\xaf\x8b\x96\xbd\xea\x65\x60\x6f" + "\x82\xfa\x51\xf4\x28\xad\x0c\xf1\xce\x0f\x91\xdd\x1a\x4c\x77\x5f"; + +static const u8 fips_aes_xts_key[32] __initconst = + "This is an AES-128-XTS key."; + +static const u8 fips_aes_xts_ciphertext[32] __initconst = + "\x5e\xb9\x98\xd6\x26\xb3\x55\xbf\x44\xab\x3e\xae\x73\xc0\x81\xc9" + "\xf4\x29\x0e\x17\x1e\xc5\xc8\x90\x79\x99\xf1\x43\x3a\x23\x08\x5a"; + +static const u8 fips_hmac_key[16] __initconst = "128-bit HMAC key"; + +static const u8 fips_sha1_digest[20] __initconst = + "\x1b\x78\xc7\x4b\xd5\xd4\x83\xb1\x58\xc5\x96\x83\x4f\x16\x8d\x15" + "\xb4\xaa\x22\x8c"; + +static const u8 fips_sha256_digest[32] __initconst = + "\x4e\x11\x83\x0c\x53\x80\x1e\x5f\x9b\x38\x33\x38\xe8\x74\x43\xb0" + "\xc1\x3a\xbe\xbf\x75\xf0\x12\x0f\x21\x33\xf5\x16\x33\xf1\xb0\x81"; + +static const u8 fips_hmac_sha256_digest[32] __initconst = + "\x63\x0e\xb5\x73\x79\xfc\xaf\x5f\x86\xe3\xaf\xf0\xc8\x36\xef\xd5" + "\x35\x8d\x40\x25\x38\xb3\x65\x72\x98\xf3\x59\xd8\x1e\x54\x4c\xa1"; + +static const u8 fips_sha512_digest[64] __initconst = + "\x32\xe0\x44\x23\xbd\xe3\xec\x28\xbf\xf1\x34\x11\xd5\xae\xbf\xd5" + "\xc0\x8e\xb5\xa1\x04\xef\x2f\x07\x84\xf1\xd9\x83\x0f\x6c\x31\xab" + "\xf7\xe7\x57\xfa\xf7\xae\xf0\x6f\xb2\x16\x08\x32\xcf\xc7\xef\x35" + "\xb3\x3b\x51\xb9\xfd\xe7\xff\x5e\xb2\x8b\xc6\x79\xe6\x14\x04\xb4"; + +/* + * This header was automatically generated by gen_fips140_testvecs.py. + * Don't edit it directly. + */ diff --git a/crypto/fips140-module.c b/crypto/fips140-module.c index 225c6bb020aa..a91735c37c67 100644 --- a/crypto/fips140-module.c +++ b/crypto/fips140-module.c @@ -3,16 +3,17 @@ * Copyright 2021 Google LLC * Author: Ard Biesheuvel * - * This file is the core of the fips140.ko, which carries a number of crypto - * algorithms and chaining mode templates that are also built into vmlinux. - * This modules performs a load time integrity check, as mandated by FIPS 140, - * and replaces registered crypto algorithms that appear on the FIPS 140 list - * with ones provided by this module. This meets the FIPS 140 requirements for - * a cryptographic software module. + * This file is the core of fips140.ko, which contains various crypto algorithms + * that are also built into vmlinux. At load time, this module overrides the + * built-in implementations of these algorithms with its implementations. It + * also runs self-tests on these algorithms and verifies the integrity of its + * code and data. If either of these steps fails, the kernel will panic. + * + * This module is intended to be loaded at early boot time in order to meet + * FIPS 140 and NIAP FPT_TST_EXT.1 requirements. It shouldn't be used if you + * don't need to meet these requirements. */ -#define pr_fmt(fmt) "fips140: " fmt - #include #include #include @@ -23,8 +24,18 @@ #include #include +#include "fips140-module.h" #include "internal.h" +/* + * This option allows deliberately failing the self-tests for a particular + * algorithm. This is for FIPS lab testing only. + */ +#ifdef CONFIG_CRYPTO_FIPS140_MOD_ERROR_INJECTION +char *fips140_broken_alg; +module_param_named(broken_alg, fips140_broken_alg, charp, 0); +#endif + /* * FIPS 140-2 prefers the use of HMAC with a public key over a plain hash. */ @@ -52,6 +63,12 @@ const u32 *__initcall_start = &__initcall_start_marker; const u8 *__text_start = &__fips140_text_start; const u8 *__rodata_start = &__fips140_rodata_start; +/* + * The list of the crypto API algorithms (by cra_name) that will be unregistered + * by this module, in preparation for the module registering its own + * implementation(s) of them. When adding a new algorithm here, make sure to + * consider whether it needs a self-test added to fips140_selftests[] as well. + */ static const char * const fips140_algorithms[] __initconst = { "aes", @@ -566,13 +583,16 @@ fips140_init(void) */ synchronize_rcu_tasks(); - /* insert self tests here */ + if (!fips140_run_selftests()) + goto panic; /* * It may seem backward to perform the integrity check last, but this * is intentional: the check itself uses hmac(sha256) which is one of * the algorithms that are replaced with versions from this module, and - * the integrity check must use the replacement version. + * the integrity check must use the replacement version. Also, to be + * ready for FIPS 140-3, the integrity check algorithm must have already + * been self-tested. */ if (!check_fips140_module_hmac()) { diff --git a/crypto/fips140-module.h b/crypto/fips140-module.h new file mode 100644 index 000000000000..b30a2e2928a6 --- /dev/null +++ b/crypto/fips140-module.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2021 Google LLC + */ + +#ifndef _CRYPTO_FIPS140_MODULE_H +#define _CRYPTO_FIPS140_MODULE_H + +#include + +#undef pr_fmt +#define pr_fmt(fmt) "fips140: " fmt + +#ifdef CONFIG_CRYPTO_FIPS140_MOD_ERROR_INJECTION +extern char *fips140_broken_alg; +#endif + +bool __init __must_check fips140_run_selftests(void); + +#endif /* _CRYPTO_FIPS140_MODULE_H */ diff --git a/crypto/fips140-selftests.c b/crypto/fips140-selftests.c new file mode 100644 index 000000000000..3ebf5a914cf0 --- /dev/null +++ b/crypto/fips140-selftests.c @@ -0,0 +1,867 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2021 Google LLC + * + * Authors: Elena Petrova , + * Eric Biggers + * + * Self-tests of fips140.ko cryptographic functionality. These are run at + * module load time to fulfill FIPS 140 and NIAP FPT_TST_EXT.1 requirements. + * + * The actual requirements for these self-tests are somewhat vague, but + * section 9 ("Self-Tests") of the FIPS 140-2 Implementation Guidance document + * (https://csrc.nist.gov/csrc/media/projects/cryptographic-module-validation-program/documents/fips140-2/fips1402ig.pdf) + * is somewhat helpful. Basically, all implementations of all FIPS approved + * algorithms (including modes of operation) must be tested. However: + * + * - If an implementation won't be used, it doesn't have to be tested. So + * when multiple implementations of the same algorithm are registered with + * the crypto API, we only have to test the default (highest-priority) one. + * + * - There are provisions for skipping tests that are already sufficiently + * covered by other tests. E.g., HMAC-SHA256 may cover SHA-256. + * + * - Only one test vector is required per algorithm, and it can be generated + * by any known-good implementation or taken from any official document. + * + * - For ciphers, both encryption and decryption must be tested. + * + * - Only one key size per algorithm needs to be tested. + * + * See fips140_selftests[] for the list of tests we've selected. Currently, all + * our test vectors except the DRBG ones were generated by the script + * tools/crypto/gen_fips140_testvecs.py, using the known-good implementations in + * the Python packages hashlib, pycryptodome, and cryptography. The DRBG test + * vectors were manually extracted from + * https://csrc.nist.gov/CSRC/media/Projects/Cryptographic-Algorithm-Validation-Program/documents/drbg/drbgtestvectors.zip. + * + * Note that we don't reuse the upstream crypto API's self-tests + * (crypto/testmgr.{c,h}), for several reasons: + * + * - To meet FIPS requirements, the self-tests must be located within the FIPS + * module boundary (fips140.ko). But testmgr is integrated into the crypto + * API framework and can't be extracted into the module. + * + * - testmgr is much more heavyweight than required for FIPS and NIAP; it + * tests more algorithms and does more tests per algorithm, as it's meant to + * do proper testing and not just meet certification requirements. We need + * tests that can run with minimal overhead on every boot-up. + * + * - Despite being more heavyweight in general, testmgr doesn't test the + * SHA-256 and AES library APIs, despite that being needed here. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fips140-module.h" + +/* Test vector for a block cipher algorithm */ +struct blockcipher_testvec { + const u8 *key; + size_t key_size; + const u8 *plaintext; + const u8 *ciphertext; + size_t block_size; +}; + +/* Test vector for an AEAD algorithm */ +struct aead_testvec { + const u8 *key; + size_t key_size; + const u8 *iv; + size_t iv_size; + const u8 *assoc; + size_t assoc_size; + const u8 *plaintext; + size_t plaintext_size; + const u8 *ciphertext; + size_t ciphertext_size; +}; + +/* Test vector for a length-preserving encryption algorithm */ +struct skcipher_testvec { + const u8 *key; + size_t key_size; + const u8 *iv; + size_t iv_size; + const u8 *plaintext; + const u8 *ciphertext; + size_t message_size; +}; + +/* Test vector for a hash algorithm */ +struct hash_testvec { + const u8 *key; + size_t key_size; + const u8 *message; + size_t message_size; + const u8 *digest; + size_t digest_size; +}; + +/* Test vector for a DRBG algorithm */ +struct drbg_testvec { + const u8 *entropy; + size_t entropy_size; + const u8 *pers; + size_t pers_size; + const u8 *entpr_a; + const u8 *entpr_b; + size_t entpr_size; + const u8 *add_a; + const u8 *add_b; + size_t add_size; + const u8 *output; + size_t out_size; +}; + +/* + * A struct which specifies an algorithm name (using crypto API syntax), a test + * function for that algorithm, and a test vector used by that test function. + */ +struct fips_test { + const char *alg; + int __must_check (*func)(const struct fips_test *test); + union { + struct blockcipher_testvec blockcipher; + struct aead_testvec aead; + struct skcipher_testvec skcipher; + struct hash_testvec hash; + struct drbg_testvec drbg; + }; +}; + +/* Maximum IV size (in bytes) among any algorithm tested here */ +#define MAX_IV_SIZE 16 + +static int __init __must_check +fips_check_result(const struct fips_test *test, u8 *result, + const u8 *expected_result, size_t result_size, + const char *operation) +{ +#ifdef CONFIG_CRYPTO_FIPS140_MOD_ERROR_INJECTION + /* Inject a failure (via corrupting the result) if requested. */ + if (fips140_broken_alg && strcmp(test->alg, fips140_broken_alg) == 0) + result[0] ^= 0xff; +#endif + if (memcmp(result, expected_result, result_size) != 0) { + pr_err("wrong result from %s %s\n", test->alg, operation); + return -EBADMSG; + } + return 0; +} + +/* + * None of the algorithms should be ASYNC, as the FIPS module doesn't register + * any ASYNC algorithms. (The ASYNC flag is only declared by hardware + * algorithms, which would need their own FIPS certification.) + * + * Ideally we would verify alg->cra_module == THIS_MODULE here as well, but that + * doesn't work because the files are compiled as built-in code. + */ +static int __init __must_check +fips_validate_alg(const struct crypto_alg *alg) +{ + if (alg->cra_flags & CRYPTO_ALG_ASYNC) { + pr_err("unexpectedly got async implementation of %s (%s)\n", + alg->cra_name, alg->cra_driver_name); + return -EINVAL; + } + return 0; +} + +/* Test a block cipher using the crypto_cipher API. */ +static int __init __must_check +fips_test_blockcipher(const struct fips_test *test) +{ + const struct blockcipher_testvec *vec = &test->blockcipher; + struct crypto_cipher *tfm; + u8 block[MAX_CIPHER_BLOCKSIZE]; + int err; + + if (WARN_ON(vec->block_size > MAX_CIPHER_BLOCKSIZE)) + return -EINVAL; + + tfm = crypto_alloc_cipher(test->alg, 0, 0); + if (IS_ERR(tfm)) { + err = PTR_ERR(tfm); + pr_err("failed to allocate %s tfm: %d\n", test->alg, err); + return err; + } + err = fips_validate_alg(tfm->base.__crt_alg); + if (err) + goto out; + if (crypto_cipher_blocksize(tfm) != vec->block_size) { + pr_err("%s has wrong block size\n", test->alg); + err = -EINVAL; + goto out; + } + + err = crypto_cipher_setkey(tfm, vec->key, vec->key_size); + if (err) { + pr_err("failed to set %s key: %d\n", test->alg, err); + goto out; + } + + /* Encrypt the plaintext, then verify the resulting ciphertext. */ + memcpy(block, vec->plaintext, vec->block_size); + crypto_cipher_encrypt_one(tfm, block, block); + err = fips_check_result(test, block, vec->ciphertext, vec->block_size, + "encryption"); + if (err) + goto out; + + /* Decrypt the ciphertext, then verify the resulting plaintext. */ + crypto_cipher_decrypt_one(tfm, block, block); + err = fips_check_result(test, block, vec->plaintext, vec->block_size, + "decryption"); +out: + crypto_free_cipher(tfm); + return err; +} + +/* + * Test for plain AES (no mode of operation). We test this separately from the + * AES modes because the implementation of AES which is used by the "aes" + * crypto_cipher isn't necessarily the same as that used by the AES modes such + * as "ecb(aes)". Similarly, the aes_{encrypt,decrypt}() library functions may + * use a different implementation as well, so we test them separately too. + */ +static int __init __must_check +fips_test_aes(const struct fips_test *test) +{ + const struct blockcipher_testvec *vec = &test->blockcipher; + struct crypto_aes_ctx ctx; + u8 block[AES_BLOCK_SIZE]; + int err; + + if (WARN_ON(vec->block_size != AES_BLOCK_SIZE)) + return -EINVAL; + + err = fips_test_blockcipher(test); + if (err) + return err; + + err = aes_expandkey(&ctx, vec->key, vec->key_size); + if (err) { + pr_err("aes_expandkey() failed: %d\n", err); + return err; + } + aes_encrypt(&ctx, block, vec->plaintext); + err = fips_check_result(test, block, vec->ciphertext, AES_BLOCK_SIZE, + "encryption (library API)"); + if (err) + return err; + aes_decrypt(&ctx, block, block); + return fips_check_result(test, block, vec->plaintext, AES_BLOCK_SIZE, + "decryption (library API)"); +} + +/* Test a length-preserving symmetric cipher using the crypto_skcipher API. */ +static int __init __must_check +fips_test_skcipher(const struct fips_test *test) +{ + const struct skcipher_testvec *vec = &test->skcipher; + struct crypto_skcipher *tfm; + struct skcipher_request *req = NULL; + u8 *message = NULL; + struct scatterlist sg; + u8 iv[MAX_IV_SIZE]; + int err; + + if (WARN_ON(vec->iv_size > MAX_IV_SIZE)) + return -EINVAL; + + tfm = crypto_alloc_skcipher(test->alg, 0, 0); + if (IS_ERR(tfm)) { + err = PTR_ERR(tfm); + pr_err("failed to allocate %s tfm: %d\n", test->alg, err); + return err; + } + err = fips_validate_alg(&crypto_skcipher_alg(tfm)->base); + if (err) + goto out; + if (crypto_skcipher_ivsize(tfm) != vec->iv_size) { + pr_err("%s has wrong IV size\n", test->alg); + err = -EINVAL; + goto out; + } + + req = skcipher_request_alloc(tfm, GFP_KERNEL); + message = kmemdup(vec->plaintext, vec->message_size, GFP_KERNEL); + if (!req || !message) { + err = -ENOMEM; + goto out; + } + sg_init_one(&sg, message, vec->message_size); + + skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); + skcipher_request_set_crypt(req, &sg, &sg, vec->message_size, iv); + + err = crypto_skcipher_setkey(tfm, vec->key, vec->key_size); + if (err) { + pr_err("failed to set %s key: %d\n", test->alg, err); + goto out; + } + + /* Encrypt the plaintext, then verify the resulting ciphertext. */ + memcpy(iv, vec->iv, vec->iv_size); + err = crypto_skcipher_encrypt(req); + if (err) { + pr_err("%s encryption failed: %d\n", test->alg, err); + goto out; + } + err = fips_check_result(test, message, vec->ciphertext, + vec->message_size, "encryption"); + if (err) + goto out; + + /* Decrypt the ciphertext, then verify the resulting plaintext. */ + memcpy(iv, vec->iv, vec->iv_size); + err = crypto_skcipher_decrypt(req); + if (err) { + pr_err("%s decryption failed: %d\n", test->alg, err); + goto out; + } + err = fips_check_result(test, message, vec->plaintext, + vec->message_size, "decryption"); +out: + kfree(message); + skcipher_request_free(req); + crypto_free_skcipher(tfm); + return err; +} + +/* Test an AEAD using the crypto_aead API. */ +static int __init __must_check +fips_test_aead(const struct fips_test *test) +{ + const struct aead_testvec *vec = &test->aead; + const int tag_size = vec->ciphertext_size - vec->plaintext_size; + struct crypto_aead *tfm; + struct aead_request *req = NULL; + u8 *assoc = NULL; + u8 *message = NULL; + struct scatterlist sg[2]; + int sg_idx = 0; + u8 iv[MAX_IV_SIZE]; + int err; + + if (WARN_ON(vec->iv_size > MAX_IV_SIZE)) + return -EINVAL; + if (WARN_ON(vec->ciphertext_size <= vec->plaintext_size)) + return -EINVAL; + + tfm = crypto_alloc_aead(test->alg, 0, 0); + if (IS_ERR(tfm)) { + err = PTR_ERR(tfm); + pr_err("failed to allocate %s tfm: %d\n", test->alg, err); + return err; + } + err = fips_validate_alg(&crypto_aead_alg(tfm)->base); + if (err) + goto out; + if (crypto_aead_ivsize(tfm) != vec->iv_size) { + pr_err("%s has wrong IV size\n", test->alg); + err = -EINVAL; + goto out; + } + + req = aead_request_alloc(tfm, GFP_KERNEL); + assoc = kmemdup(vec->assoc, vec->assoc_size, GFP_KERNEL); + message = kzalloc(vec->ciphertext_size, GFP_KERNEL); + if (!req || !assoc || !message) { + err = -ENOMEM; + goto out; + } + memcpy(message, vec->plaintext, vec->plaintext_size); + + sg_init_table(sg, ARRAY_SIZE(sg)); + if (vec->assoc_size) + sg_set_buf(&sg[sg_idx++], assoc, vec->assoc_size); + sg_set_buf(&sg[sg_idx++], message, vec->ciphertext_size); + + aead_request_set_ad(req, vec->assoc_size); + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + + err = crypto_aead_setkey(tfm, vec->key, vec->key_size); + if (err) { + pr_err("failed to set %s key: %d\n", test->alg, err); + goto out; + } + + err = crypto_aead_setauthsize(tfm, tag_size); + if (err) { + pr_err("failed to set %s authentication tag size: %d\n", + test->alg, err); + goto out; + } + + /* + * Encrypt the plaintext, then verify the resulting ciphertext (which + * includes the authentication tag). + */ + memcpy(iv, vec->iv, vec->iv_size); + aead_request_set_crypt(req, sg, sg, vec->plaintext_size, iv); + err = crypto_aead_encrypt(req); + if (err) { + pr_err("%s encryption failed: %d\n", test->alg, err); + goto out; + } + err = fips_check_result(test, message, vec->ciphertext, + vec->ciphertext_size, "encryption"); + if (err) + goto out; + + /* + * Decrypt the ciphertext (which includes the authentication tag), then + * verify the resulting plaintext. + */ + memcpy(iv, vec->iv, vec->iv_size); + aead_request_set_crypt(req, sg, sg, vec->ciphertext_size, iv); + err = crypto_aead_decrypt(req); + if (err) { + pr_err("%s decryption failed: %d\n", test->alg, err); + goto out; + } + err = fips_check_result(test, message, vec->plaintext, + vec->plaintext_size, "decryption"); +out: + kfree(message); + kfree(assoc); + aead_request_free(req); + crypto_free_aead(tfm); + return err; +} + +/* + * Test a hash algorithm using the crypto_shash API. + * + * Note that we don't need to test the crypto_ahash API too, since none of the + * hash algorithms in the FIPS module have the ASYNC flag, and thus there will + * be no hash algorithms that can be accessed only through crypto_ahash. + */ +static int __init __must_check +fips_test_hash(const struct fips_test *test) +{ + const struct hash_testvec *vec = &test->hash; + struct crypto_shash *tfm; + u8 digest[HASH_MAX_DIGESTSIZE]; + int err; + + if (WARN_ON(vec->digest_size > HASH_MAX_DIGESTSIZE)) + return -EINVAL; + + tfm = crypto_alloc_shash(test->alg, 0, 0); + if (IS_ERR(tfm)) { + err = PTR_ERR(tfm); + pr_err("failed to allocate %s tfm: %d\n", test->alg, err); + return err; + } + err = fips_validate_alg(&crypto_shash_alg(tfm)->base); + if (err) + goto out; + if (crypto_shash_digestsize(tfm) != vec->digest_size) { + pr_err("%s has wrong digest size\n", test->alg); + err = -EINVAL; + goto out; + } + + if (vec->key) { + err = crypto_shash_setkey(tfm, vec->key, vec->key_size); + if (err) { + pr_err("failed to set %s key: %d\n", test->alg, err); + goto out; + } + } + + err = crypto_shash_tfm_digest(tfm, vec->message, vec->message_size, + digest); + if (err) { + pr_err("%s digest computation failed: %d\n", test->alg, err); + goto out; + } + err = fips_check_result(test, digest, vec->digest, vec->digest_size, + "digest"); +out: + crypto_free_shash(tfm); + return err; +} + +/* + * Test the sha256() library function, as it may not be covered by the "sha256" + * crypto_shash, and thus may not be covered by the "hmac(sha256)" test we do. + */ +static int __init __must_check +fips_test_sha256_library(const struct fips_test *test) +{ + const struct hash_testvec *vec = &test->hash; + u8 digest[SHA256_DIGEST_SIZE]; + + if (WARN_ON(vec->digest_size != SHA256_DIGEST_SIZE)) + return -EINVAL; + + sha256(vec->message, vec->message_size, digest); + return fips_check_result(test, digest, vec->digest, vec->digest_size, + "digest (library API)"); +} + +/* Test a DRBG using the crypto_rng API. */ +static int __init __must_check +fips_test_drbg(const struct fips_test *test) +{ + const struct drbg_testvec *vec = &test->drbg; + struct crypto_rng *rng; + u8 *output = NULL; + struct drbg_test_data test_data; + struct drbg_string addtl, pers, testentropy; + int err; + + rng = crypto_alloc_rng(test->alg, 0, 0); + if (IS_ERR(rng)) { + err = PTR_ERR(rng); + pr_err("failed to allocate %s tfm: %d\n", test->alg, err); + return PTR_ERR(rng); + } + err = fips_validate_alg(&crypto_rng_alg(rng)->base); + if (err) + goto out; + + output = kzalloc(vec->out_size, GFP_KERNEL); + if (!output) { + err = -ENOMEM; + goto out; + } + + /* + * Initialize the DRBG with the entropy and personalization string given + * in the test vector. + */ + test_data.testentropy = &testentropy; + drbg_string_fill(&testentropy, vec->entropy, vec->entropy_size); + drbg_string_fill(&pers, vec->pers, vec->pers_size); + err = crypto_drbg_reset_test(rng, &pers, &test_data); + if (err) { + pr_err("failed to reset %s\n", test->alg); + goto out; + } + + /* + * Generate some random bytes using the additional data string provided + * in the test vector. Also use the additional entropy if provided + * (relevant for the prediction-resistant DRBG variants only). + */ + drbg_string_fill(&addtl, vec->add_a, vec->add_size); + if (vec->entpr_size) { + drbg_string_fill(&testentropy, vec->entpr_a, vec->entpr_size); + err = crypto_drbg_get_bytes_addtl_test(rng, output, + vec->out_size, &addtl, + &test_data); + } else { + err = crypto_drbg_get_bytes_addtl(rng, output, vec->out_size, + &addtl); + } + if (err) { + pr_err("failed to get bytes from %s (try 1): %d\n", + test->alg, err); + goto out; + } + + /* + * Do the same again, using a second additional data string, and (when + * applicable) a second additional entropy string. + */ + drbg_string_fill(&addtl, vec->add_b, vec->add_size); + if (test->drbg.entpr_size) { + drbg_string_fill(&testentropy, vec->entpr_b, vec->entpr_size); + err = crypto_drbg_get_bytes_addtl_test(rng, output, + vec->out_size, &addtl, + &test_data); + } else { + err = crypto_drbg_get_bytes_addtl(rng, output, vec->out_size, + &addtl); + } + if (err) { + pr_err("failed to get bytes from %s (try 2): %d\n", + test->alg, err); + goto out; + } + + /* Check that the DRBG generated the expected output. */ + err = fips_check_result(test, output, vec->output, vec->out_size, + "get_bytes"); +out: + kfree(output); + crypto_free_rng(rng); + return err; +} + +/* Include the test vectors generated by the Python script. */ +#include "fips140-generated-testvecs.h" + +/* List of all self-tests. Keep this in sync with fips140_algorithms[]. */ +static const struct fips_test fips140_selftests[] __initconst = { + /* + * Tests for AES and AES modes. + * + * The full list of AES algorithms we potentially need to test are AES + * by itself, AES-CBC, AES-CTR, AES-ECB, AES-GCM, and AES-XTS. We can + * follow the FIPS 140-2 Implementation Guidance (IG) document to try to + * reduce this list, but we run into the issue that the architecture- + * specific implementations of these algorithms in Linux often don't + * share the "same" underlying AES implementation. E.g., the ARMv8 CE + * optimized implementations issue ARMv8 CE instructions directly rather + * than going through a separate AES implementation. In this case, + * separate tests are needed according to section 9.2 of the IG. + */ + { + .alg = "aes", + .func = fips_test_aes, + .blockcipher = { + .key = fips_aes_key, + .key_size = sizeof(fips_aes_key), + .plaintext = fips_message, + .ciphertext = fips_aes_ecb_ciphertext, + .block_size = 16, + } + }, { + .alg = "cbc(aes)", + .func = fips_test_skcipher, + .skcipher = { + .key = fips_aes_key, + .key_size = sizeof(fips_aes_key), + .iv = fips_aes_iv, + .iv_size = sizeof(fips_aes_iv), + .plaintext = fips_message, + .ciphertext = fips_aes_cbc_ciphertext, + .message_size = sizeof(fips_message), + } + }, { + .alg = "ctr(aes)", + .func = fips_test_skcipher, + .skcipher = { + .key = fips_aes_key, + .key_size = sizeof(fips_aes_key), + .iv = fips_aes_iv, + .iv_size = sizeof(fips_aes_iv), + .plaintext = fips_message, + .ciphertext = fips_aes_ctr_ciphertext, + .message_size = sizeof(fips_message), + } + }, { + .alg = "ecb(aes)", + .func = fips_test_skcipher, + .skcipher = { + .key = fips_aes_key, + .key_size = sizeof(fips_aes_key), + .plaintext = fips_message, + .ciphertext = fips_aes_ecb_ciphertext, + .message_size = sizeof(fips_message) + } + }, { + .alg = "gcm(aes)", + .func = fips_test_aead, + .aead = { + .key = fips_aes_key, + .key_size = sizeof(fips_aes_key), + .iv = fips_aes_iv, + /* The GCM implementation assumes an IV size of 12. */ + .iv_size = 12, + .assoc = fips_aes_gcm_assoc, + .assoc_size = sizeof(fips_aes_gcm_assoc), + .plaintext = fips_message, + .plaintext_size = sizeof(fips_message), + .ciphertext = fips_aes_gcm_ciphertext, + .ciphertext_size = sizeof(fips_aes_gcm_ciphertext), + } + }, { + .alg = "xts(aes)", + .func = fips_test_skcipher, + .skcipher = { + .key = fips_aes_xts_key, + .key_size = sizeof(fips_aes_xts_key), + .iv = fips_aes_iv, + .iv_size = sizeof(fips_aes_iv), + .plaintext = fips_message, + .ciphertext = fips_aes_xts_ciphertext, + .message_size = sizeof(fips_message), + } + /* + * Tests for SHA-1, SHA-256, HMAC-SHA256, and SHA-512. + * + * The selection of these specific tests follows the guidance from + * section 9 of the FIPS 140-2 Implementation Guidance (IG) document to + * achieve a minimal list of tests, rather than testing all of + * SHA-{1,224,256,384,512} and HMAC-SHA{1,224,256,384,512}. As per the + * IG, testing SHA-224 is only required if SHA-256 isn't implemented, + * and testing SHA-384 is only required if SHA-512 isn't implemented. + * Also, HMAC only has to be tested with one underlying SHA, and the + * HMAC test also fulfills the test for its underlying SHA. That would + * result in a test list of e.g. SHA-1, HMAC-SHA256, and SHA-512. + * + * However we also need to take into account cases where implementations + * aren't shared in the "natural" way assumed by the IG. Currently the + * only known exception w.r.t. SHA-* and HMAC-* is the sha256() library + * function which may not be covered by the test of the "hmac(sha256)" + * crypto_shash. So, we test sha256() separately. + */ + }, { + .alg = "sha1", + .func = fips_test_hash, + .hash = { + .message = fips_message, + .message_size = sizeof(fips_message), + .digest = fips_sha1_digest, + .digest_size = sizeof(fips_sha1_digest) + } + }, { + .alg = "sha256", + .func = fips_test_sha256_library, + .hash = { + .message = fips_message, + .message_size = sizeof(fips_message), + .digest = fips_sha256_digest, + .digest_size = sizeof(fips_sha256_digest) + } + }, { + .alg = "hmac(sha256)", + .func = fips_test_hash, + .hash = { + .key = fips_hmac_key, + .key_size = sizeof(fips_hmac_key), + .message = fips_message, + .message_size = sizeof(fips_message), + .digest = fips_hmac_sha256_digest, + .digest_size = sizeof(fips_hmac_sha256_digest) + } + }, { + .alg = "sha512", + .func = fips_test_hash, + .hash = { + .message = fips_message, + .message_size = sizeof(fips_message), + .digest = fips_sha512_digest, + .digest_size = sizeof(fips_sha512_digest) + } + /* + * Tests for DRBG algorithms. + * + * Only the default variant (the one that users get when they request + * "stdrng") is required to be tested, as we don't consider the other + * variants to be used / usable in the FIPS security policy. This is + * similar to how e.g. we don't test both "xts(aes-generic)" and + * "xts-aes-ce" but rather just "xts(aes)". + * + * Currently the default one is "drbg_nopr_hmac_sha256"; however, just + * in case we also test the prediction-resistant enabled variant too. + */ + }, { + .alg = "drbg_nopr_hmac_sha256", + .func = fips_test_drbg, + .drbg = { + .entropy = + "\xf9\x7a\x3c\xfd\x91\xfa\xa0\x46\xb9\xe6\x1b\x94" + "\x93\xd4\x36\xc4\x93\x1f\x60\x4b\x22\xf1\x08\x15" + "\x21\xb3\x41\x91\x51\xe8\xff\x06\x11\xf3\xa7\xd4" + "\x35\x95\x35\x7d\x58\x12\x0b\xd1\xe2\xdd\x8a\xed", + .entropy_size = 48, + .output = + "\xc6\x87\x1c\xff\x08\x24\xfe\x55\xea\x76\x89\xa5" + "\x22\x29\x88\x67\x30\x45\x0e\x5d\x36\x2d\xa5\xbf" + "\x59\x0d\xcf\x9a\xcd\x67\xfe\xd4\xcb\x32\x10\x7d" + "\xf5\xd0\x39\x69\xa6\x6b\x1f\x64\x94\xfd\xf5\xd6" + "\x3d\x5b\x4d\x0d\x34\xea\x73\x99\xa0\x7d\x01\x16" + "\x12\x6d\x0d\x51\x8c\x7c\x55\xba\x46\xe1\x2f\x62" + "\xef\xc8\xfe\x28\xa5\x1c\x9d\x42\x8e\x6d\x37\x1d" + "\x73\x97\xab\x31\x9f\xc7\x3d\xed\x47\x22\xe5\xb4" + "\xf3\x00\x04\x03\x2a\x61\x28\xdf\x5e\x74\x97\xec" + "\xf8\x2c\xa7\xb0\xa5\x0e\x86\x7e\xf6\x72\x8a\x4f" + "\x50\x9a\x8c\x85\x90\x87\x03\x9c", + .out_size = 128, + .add_a = + "\x51\x72\x89\xaf\xe4\x44\xa0\xfe\x5e\xd1\xa4\x1d" + "\xbb\xb5\xeb\x17\x15\x00\x79\xbd\xd3\x1e\x29\xcf" + "\x2f\xf3\x00\x34\xd8\x26\x8e\x3b", + .add_b = + "\x88\x02\x8d\x29\xef\x80\xb4\xe6\xf0\xfe\x12\xf9" + "\x1d\x74\x49\xfe\x75\x06\x26\x82\xe8\x9c\x57\x14" + "\x40\xc0\xc9\xb5\x2c\x42\xa6\xe0", + .add_size = 32, + } + }, { + .alg = "drbg_pr_hmac_sha256", + .func = fips_test_drbg, + .drbg = { + .entropy = + "\xc7\xcc\xbc\x67\x7e\x21\x66\x1e\x27\x2b\x63\xdd" + "\x3a\x78\xdc\xdf\x66\x6d\x3f\x24\xae\xcf\x37\x01" + "\xa9\x0d\x89\x8a\xa7\xdc\x81\x58\xae\xb2\x10\x15" + "\x7e\x18\x44\x6d\x13\xea\xdf\x37\x85\xfe\x81\xfb", + .entropy_size = 48, + .entpr_a = + "\x7b\xa1\x91\x5b\x3c\x04\xc4\x1b\x1d\x19\x2f\x1a" + "\x18\x81\x60\x3c\x6c\x62\x91\xb7\xe9\xf5\xcb\x96" + "\xbb\x81\x6a\xcc\xb5\xae\x55\xb6", + .entpr_b = + "\x99\x2c\xc7\x78\x7e\x3b\x88\x12\xef\xbe\xd3\xd2" + "\x7d\x2a\xa5\x86\xda\x8d\x58\x73\x4a\x0a\xb2\x2e" + "\xbb\x4c\x7e\xe3\x9a\xb6\x81\xc1", + .entpr_size = 32, + .output = + "\x95\x6f\x95\xfc\x3b\xb7\xfe\x3e\xd0\x4e\x1a\x14" + "\x6c\x34\x7f\x7b\x1d\x0d\x63\x5e\x48\x9c\x69\xe6" + "\x46\x07\xd2\x87\xf3\x86\x52\x3d\x98\x27\x5e\xd7" + "\x54\xe7\x75\x50\x4f\xfb\x4d\xfd\xac\x2f\x4b\x77" + "\xcf\x9e\x8e\xcc\x16\xa2\x24\xcd\x53\xde\x3e\xc5" + "\x55\x5d\xd5\x26\x3f\x89\xdf\xca\x8b\x4e\x1e\xb6" + "\x88\x78\x63\x5c\xa2\x63\x98\x4e\x6f\x25\x59\xb1" + "\x5f\x2b\x23\xb0\x4b\xa5\x18\x5d\xc2\x15\x74\x40" + "\x59\x4c\xb4\x1e\xcf\x9a\x36\xfd\x43\xe2\x03\xb8" + "\x59\x91\x30\x89\x2a\xc8\x5a\x43\x23\x7c\x73\x72" + "\xda\x3f\xad\x2b\xba\x00\x6b\xd1", + .out_size = 128, + .add_a = + "\x18\xe8\x17\xff\xef\x39\xc7\x41\x5c\x73\x03\x03" + "\xf6\x3d\xe8\x5f\xc8\xab\xe4\xab\x0f\xad\xe8\xd6" + "\x86\x88\x55\x28\xc1\x69\xdd\x76", + .add_b = + "\xac\x07\xfc\xbe\x87\x0e\xd3\xea\x1f\x7e\xb8\xe7" + "\x9d\xec\xe8\xe7\xbc\xf3\x18\x25\x77\x35\x4a\xaa" + "\x00\x99\x2a\xdd\x0a\x00\x50\x82", + .add_size = 32, + .pers = + "\xbc\x55\xab\x3c\xf6\x52\xb0\x11\x3d\x7b\x90\xb8" + "\x24\xc9\x26\x4e\x5a\x1e\x77\x0d\x3d\x58\x4a\xda" + "\xd1\x81\xe9\xf8\xeb\x30\x8f\x6f", + .pers_size = 32, + } + } +}; + +bool __init fips140_run_selftests(void) +{ + int i; + + pr_info("running self-tests\n"); + for (i = 0; i < ARRAY_SIZE(fips140_selftests); i++) { + const struct fips_test *test = &fips140_selftests[i]; + int err; + + err = test->func(test); + if (err) { + pr_emerg("self-tests failed for algorithm %s: %d\n", + test->alg, err); + /* The caller is responsible for calling panic(). */ + return false; + } + } + pr_info("all self-tests passed\n"); + return true; +} diff --git a/drivers/android/debug_symbols.c b/drivers/android/debug_symbols.c index 61e55575ed5f..6a4a6f8992f2 100644 --- a/drivers/android/debug_symbols.c +++ b/drivers/android/debug_symbols.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include struct ads_entry { char *name; @@ -59,6 +61,13 @@ static const struct ads_entry ads_entries[ADS_END] = { #ifdef CONFIG_SWAP ADS_ENTRY(ADS_NR_SWAP_PAGES, &nr_swap_pages), #endif +#ifdef CONFIG_MMU + ADS_ENTRY(ADS_MMAP_MIN_ADDR, &mmap_min_addr), +#endif + ADS_ENTRY(ADS_STACK_GUARD_GAP, &stack_guard_gap), +#ifdef CONFIG_SYSCTL + ADS_ENTRY(ADS_SYSCTL_LEGACY_VA_LAYOUT, &sysctl_legacy_va_layout), +#endif }; /* diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 07eb8efd7c40..6ebc82a0ff43 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -68,6 +69,9 @@ #include #include #include +#include +#include +#include /* * Export tracepoints that act as a bare tracehook (ie: have no trace event @@ -87,6 +91,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_prepare_prio_fork); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_finish_prio_fork); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_user_nice); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_setscheduler); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sk_alloc); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sk_free); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_nf_conn_alloc); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_nf_conn_free); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_arch_set_freq_scale); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_is_fpsimd_save); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_transaction_init); @@ -237,6 +245,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_insert); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_node_delete); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_node_replace); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_lookup); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_alloc_rqs); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_rq_ctx_init); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_commit_creds); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_exit_creds); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_override_creds); @@ -353,3 +363,6 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_file_open); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_check_bpf_syscall); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_usb_dev_suspend); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_usb_dev_resume); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ipv6_gen_linklocal_addr); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sound_usb_support_cpu_suspend); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_snd_compr_use_pause_in_drain); diff --git a/drivers/dma-buf/dma-buf-sysfs-stats.c b/drivers/dma-buf/dma-buf-sysfs-stats.c index e5f9e33044ed..2389a363bd3a 100644 --- a/drivers/dma-buf/dma-buf-sysfs-stats.c +++ b/drivers/dma-buf/dma-buf-sysfs-stats.c @@ -84,120 +84,6 @@ static struct kobj_type dma_buf_ktype = { .default_groups = dma_buf_stats_default_groups, }; -#define to_dma_buf_attach_entry_from_kobj(x) container_of(x, struct dma_buf_attach_sysfs_entry, kobj) - -struct dma_buf_attach_stats_attribute { - struct attribute attr; - ssize_t (*show)(struct dma_buf_attach_sysfs_entry *sysfs_entry, - struct dma_buf_attach_stats_attribute *attr, char *buf); -}; -#define to_dma_buf_attach_stats_attr(x) container_of(x, struct dma_buf_attach_stats_attribute, attr) - -static ssize_t dma_buf_attach_stats_attribute_show(struct kobject *kobj, - struct attribute *attr, - char *buf) -{ - struct dma_buf_attach_stats_attribute *attribute; - struct dma_buf_attach_sysfs_entry *sysfs_entry; - - attribute = to_dma_buf_attach_stats_attr(attr); - sysfs_entry = to_dma_buf_attach_entry_from_kobj(kobj); - - if (!attribute->show) - return -EIO; - - return attribute->show(sysfs_entry, attribute, buf); -} - -static const struct sysfs_ops dma_buf_attach_stats_sysfs_ops = { - .show = dma_buf_attach_stats_attribute_show, -}; - -static ssize_t map_counter_show(struct dma_buf_attach_sysfs_entry *sysfs_entry, - struct dma_buf_attach_stats_attribute *attr, - char *buf) -{ - return sysfs_emit(buf, "%u\n", sysfs_entry->map_counter); -} - -static struct dma_buf_attach_stats_attribute map_counter_attribute = - __ATTR_RO(map_counter); - -static struct attribute *dma_buf_attach_stats_default_attrs[] = { - &map_counter_attribute.attr, - NULL, -}; -ATTRIBUTE_GROUPS(dma_buf_attach_stats_default); - -static void dma_buf_attach_sysfs_release(struct kobject *kobj) -{ - struct dma_buf_attach_sysfs_entry *sysfs_entry; - - sysfs_entry = to_dma_buf_attach_entry_from_kobj(kobj); - kfree(sysfs_entry); -} - -static struct kobj_type dma_buf_attach_ktype = { - .sysfs_ops = &dma_buf_attach_stats_sysfs_ops, - .release = dma_buf_attach_sysfs_release, - .default_groups = dma_buf_attach_stats_default_groups, -}; - -void dma_buf_attach_stats_teardown(struct dma_buf_attachment *attach) -{ - struct dma_buf_attach_sysfs_entry *sysfs_entry; - - sysfs_entry = attach->sysfs_entry; - if (!sysfs_entry) - return; - - sysfs_delete_link(&sysfs_entry->kobj, &attach->dev->kobj, "device"); - - kobject_del(&sysfs_entry->kobj); - kobject_put(&sysfs_entry->kobj); -} - -int dma_buf_attach_stats_setup(struct dma_buf_attachment *attach, - unsigned int uid) -{ - struct dma_buf_attach_sysfs_entry *sysfs_entry; - int ret; - struct dma_buf *dmabuf; - - if (!attach) - return -EINVAL; - - dmabuf = attach->dmabuf; - - sysfs_entry = kzalloc(sizeof(struct dma_buf_attach_sysfs_entry), - GFP_KERNEL); - if (!sysfs_entry) - return -ENOMEM; - - sysfs_entry->kobj.kset = dmabuf->sysfs_entry->attach_stats_kset; - - attach->sysfs_entry = sysfs_entry; - - ret = kobject_init_and_add(&sysfs_entry->kobj, &dma_buf_attach_ktype, - NULL, "%u", uid); - if (ret) - goto kobj_err; - - ret = sysfs_create_link(&sysfs_entry->kobj, &attach->dev->kobj, - "device"); - if (ret) - goto link_err; - - return 0; - -link_err: - kobject_del(&sysfs_entry->kobj); -kobj_err: - kobject_put(&sysfs_entry->kobj); - attach->sysfs_entry = NULL; - - return ret; -} void dma_buf_stats_teardown(struct dma_buf *dmabuf) { struct dma_buf_sysfs_entry *sysfs_entry; @@ -206,7 +92,6 @@ void dma_buf_stats_teardown(struct dma_buf *dmabuf) if (!sysfs_entry) return; - kset_unregister(sysfs_entry->attach_stats_kset); kobject_del(&sysfs_entry->kobj); kobject_put(&sysfs_entry->kobj); } @@ -254,7 +139,6 @@ int dma_buf_stats_setup(struct dma_buf *dmabuf) { struct dma_buf_sysfs_entry *sysfs_entry; int ret; - struct kset *attach_stats_kset; if (!dmabuf || !dmabuf->file) return -EINVAL; @@ -279,21 +163,8 @@ int dma_buf_stats_setup(struct dma_buf *dmabuf) if (ret) goto err_sysfs_dmabuf; - /* create the directory for attachment stats */ - attach_stats_kset = kset_create_and_add("attachments", - &dmabuf_sysfs_no_uevent_ops, - &sysfs_entry->kobj); - if (!attach_stats_kset) { - ret = -ENOMEM; - goto err_sysfs_attach; - } - - sysfs_entry->attach_stats_kset = attach_stats_kset; - return 0; -err_sysfs_attach: - kobject_del(&sysfs_entry->kobj); err_sysfs_dmabuf: kobject_put(&sysfs_entry->kobj); dmabuf->sysfs_entry = NULL; diff --git a/drivers/dma-buf/dma-buf-sysfs-stats.h b/drivers/dma-buf/dma-buf-sysfs-stats.h index 5f4703249117..a49c6e2650cc 100644 --- a/drivers/dma-buf/dma-buf-sysfs-stats.h +++ b/drivers/dma-buf/dma-buf-sysfs-stats.h @@ -14,23 +14,8 @@ int dma_buf_init_sysfs_statistics(void); void dma_buf_uninit_sysfs_statistics(void); int dma_buf_stats_setup(struct dma_buf *dmabuf); -int dma_buf_attach_stats_setup(struct dma_buf_attachment *attach, - unsigned int uid); -static inline void dma_buf_update_attachment_map_count(struct dma_buf_attachment *attach, - int delta) -{ - struct dma_buf_attach_sysfs_entry *entry = attach->sysfs_entry; - entry->map_counter += delta; -} void dma_buf_stats_teardown(struct dma_buf *dmabuf); -void dma_buf_attach_stats_teardown(struct dma_buf_attachment *attach); -static inline unsigned int dma_buf_update_attach_uid(struct dma_buf *dmabuf) -{ - struct dma_buf_sysfs_entry *entry = dmabuf->sysfs_entry; - - return entry->attachment_uid++; -} #else static inline int dma_buf_init_sysfs_statistics(void) @@ -44,19 +29,7 @@ static inline int dma_buf_stats_setup(struct dma_buf *dmabuf) { return 0; } -static inline int dma_buf_attach_stats_setup(struct dma_buf_attachment *attach, - unsigned int uid) -{ - return 0; -} static inline void dma_buf_stats_teardown(struct dma_buf *dmabuf) {} -static inline void dma_buf_attach_stats_teardown(struct dma_buf_attachment *attach) {} -static inline void dma_buf_update_attachment_map_count(struct dma_buf_attachment *attach, - int delta) {} -static inline unsigned int dma_buf_update_attach_uid(struct dma_buf *dmabuf) -{ - return 0; -} #endif #endif // _DMA_BUF_SYSFS_STATS_H diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index eb8b66b569c8..47eeeb02513b 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -730,7 +730,6 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev, { struct dma_buf_attachment *attach; int ret; - unsigned int attach_uid; if (WARN_ON(!dmabuf || !dev)) return ERR_PTR(-EINVAL); @@ -756,13 +755,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev, } dma_resv_lock(dmabuf->resv, NULL); list_add(&attach->node, &dmabuf->attachments); - attach_uid = dma_buf_update_attach_uid(dmabuf); dma_resv_unlock(dmabuf->resv); - ret = dma_buf_attach_stats_setup(attach, attach_uid); - if (ret) - goto err_sysfs; - /* When either the importer or the exporter can't handle dynamic * mappings we cache the mapping here to avoid issues with the * reservation object lock. @@ -789,7 +783,6 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev, dma_resv_unlock(attach->dmabuf->resv); attach->sgt = sgt; attach->dir = DMA_BIDIRECTIONAL; - dma_buf_update_attachment_map_count(attach, 1 /* delta */); } return attach; @@ -806,7 +799,6 @@ err_unlock: if (dma_buf_is_dynamic(attach->dmabuf)) dma_resv_unlock(attach->dmabuf->resv); -err_sysfs: dma_buf_detach(dmabuf, attach); return ERR_PTR(ret); } @@ -845,7 +837,6 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) dma_resv_lock(attach->dmabuf->resv, NULL); dmabuf->ops->unmap_dma_buf(attach, attach->sgt, attach->dir); - dma_buf_update_attachment_map_count(attach, -1 /* delta */); if (dma_buf_is_dynamic(attach->dmabuf)) { dma_buf_unpin(attach); @@ -859,7 +850,6 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) if (dmabuf->ops->detach) dmabuf->ops->detach(dmabuf, attach); - dma_buf_attach_stats_teardown(attach); kfree(attach); } EXPORT_SYMBOL_GPL(dma_buf_detach); @@ -965,9 +955,6 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach, attach->dir = direction; } - if (!IS_ERR(sg_table)) - dma_buf_update_attachment_map_count(attach, 1 /* delta */); - return sg_table; } EXPORT_SYMBOL_GPL(dma_buf_map_attachment); @@ -1005,8 +992,6 @@ void dma_buf_unmap_attachment(struct dma_buf_attachment *attach, if (dma_buf_is_dynamic(attach->dmabuf) && !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) dma_buf_unpin(attach); - - dma_buf_update_attachment_map_count(attach, -1 /* delta */); } EXPORT_SYMBOL_GPL(dma_buf_unmap_attachment); diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h index 0770c036e2ff..63f51491fffe 100644 --- a/drivers/mmc/host/sdhci.h +++ b/drivers/mmc/host/sdhci.h @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -605,6 +606,8 @@ struct sdhci_host { u64 data_timeout; + ANDROID_KABI_RESERVE(1); + unsigned long private[] ____cacheline_aligned; }; @@ -652,6 +655,8 @@ struct sdhci_ops { void (*request_done)(struct sdhci_host *host, struct mmc_request *mrq); void (*dump_vendor_regs)(struct sdhci_host *host); + + ANDROID_KABI_RESERVE(1); }; #ifdef CONFIG_MMC_SDHCI_IO_ACCESSORS diff --git a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h index d69032d65463..1ee80cda3d01 100644 --- a/drivers/scsi/ufs/ufs.h +++ b/drivers/scsi/ufs/ufs.h @@ -13,6 +13,7 @@ #include #include +#include #include #define GENERAL_UPIU_REQUEST_SIZE (sizeof(struct utp_upiu_req)) @@ -594,6 +595,7 @@ struct ufs_dev_info { u8 b_presrv_uspc_en; /* UFS HPB related flag */ bool hpb_enabled; + ANDROID_KABI_RESERVE(1); }; /** diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index d044ecbb06ad..f4d0fb6fdd97 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -43,6 +43,7 @@ #include #include #include +#include #include "ufs.h" #include "ufs_quirks.h" @@ -217,6 +218,8 @@ struct ufshcd_lrb { #endif bool req_abort_skip; + + ANDROID_KABI_RESERVE(1); }; /** @@ -353,6 +356,11 @@ struct ufs_hba_variant_ops { const union ufs_crypto_cfg_entry *cfg, int slot); void (*event_notify)(struct ufs_hba *hba, enum ufs_event_type evt, void *data); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /* clock gating state */ @@ -392,6 +400,8 @@ struct ufs_clk_gating { bool is_initialized; int active_reqs; struct workqueue_struct *clk_gating_workq; + + ANDROID_KABI_RESERVE(1); }; struct ufs_saved_pwr_info { @@ -438,6 +448,8 @@ struct ufs_clk_scaling { bool is_initialized; bool is_busy_started; bool is_suspended; + + ANDROID_KABI_RESERVE(1); }; #define UFS_EVENT_HIST_LENGTH 8 @@ -915,6 +927,11 @@ struct ufs_hba { #ifdef CONFIG_DEBUG_FS struct dentry *debugfs_root; #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /* Returns true if clocks can be gated. Otherwise false */ diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c index 900cad5d79da..05e47ce9657d 100644 --- a/drivers/staging/android/ashmem.c +++ b/drivers/staging/android/ashmem.c @@ -915,6 +915,15 @@ static const struct file_operations ashmem_fops = { #endif }; +/* + * is_ashmem_file - Check if struct file* is associated with ashmem + */ +int is_ashmem_file(struct file *file) +{ + return file->f_op == &ashmem_fops; +} +EXPORT_SYMBOL_GPL(is_ashmem_file); + static struct miscdevice ashmem_misc = { .minor = MISC_DYNAMIC_MINOR, .name = "ashmem", diff --git a/drivers/staging/android/ashmem.h b/drivers/staging/android/ashmem.h index 1a478173cd21..9fa72ed7b7ed 100644 --- a/drivers/staging/android/ashmem.h +++ b/drivers/staging/android/ashmem.h @@ -21,4 +21,6 @@ #define COMPAT_ASHMEM_SET_PROT_MASK _IOW(__ASHMEMIOC, 5, unsigned int) #endif +int is_ashmem_file(struct file *file); + #endif /* _LINUX_ASHMEM_H */ diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 8ec816e4c796..3121ff82cb81 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -1267,6 +1267,7 @@ static void dwc3_get_properties(struct dwc3 *dwc) u8 rx_max_burst_prd; u8 tx_thr_num_pkt_prd; u8 tx_max_burst_prd; + u8 tx_fifo_resize_max_num; const char *usb_psy_name; int ret; @@ -1282,6 +1283,13 @@ static void dwc3_get_properties(struct dwc3 *dwc) */ hird_threshold = 12; + /* + * default to a TXFIFO size large enough to fit 6 max packets. This + * allows for systems with larger bus latencies to have some headroom + * for endpoints that have a large bMaxBurst value. + */ + tx_fifo_resize_max_num = 6; + dwc->maximum_speed = usb_get_maximum_speed(dev); dwc->max_ssp_rate = usb_get_maximum_ssp_rate(dev); dwc->dr_mode = usb_get_dr_mode(dev); @@ -1325,6 +1333,11 @@ static void dwc3_get_properties(struct dwc3 *dwc) &tx_thr_num_pkt_prd); device_property_read_u8(dev, "snps,tx-max-burst-prd", &tx_max_burst_prd); + dwc->do_fifo_resize = device_property_read_bool(dev, + "tx-fifo-resize"); + if (dwc->do_fifo_resize) + device_property_read_u8(dev, "tx-fifo-max-num", + &tx_fifo_resize_max_num); dwc->disable_scramble_quirk = device_property_read_bool(dev, "snps,disable_scramble_quirk"); @@ -1390,6 +1403,8 @@ static void dwc3_get_properties(struct dwc3 *dwc) dwc->tx_max_burst_prd = tx_max_burst_prd; dwc->imod_interval = 0; + + dwc->tx_fifo_resize_max_num = tx_fifo_resize_max_num; } /* check whether the core supports IMOD */ diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 05dfd01183f7..07d70418b7bd 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1038,6 +1038,7 @@ struct dwc3_scratchpad_array { * @rx_max_burst_prd: max periodic ESS receive burst size * @tx_thr_num_pkt_prd: periodic ESS transmit packet count * @tx_max_burst_prd: max periodic ESS transmit burst size + * @tx_fifo_resize_max_num: max number of fifos allocated during txfifo resize * @hsphy_interface: "utmi" or "ulpi" * @connected: true when we're connected to a host, false otherwise * @delayed_status: true when gadget driver asks for delayed status @@ -1052,6 +1053,7 @@ struct dwc3_scratchpad_array { * 1 - utmi_l1_suspend_n * @is_fpga: true when we are using the FPGA board * @pending_events: true when we have pending IRQs to be handled + * @do_fifo_resize: true when txfifo resizing is enabled for dwc3 endpoints * @pullups_connected: true when Run/Stop bit is set * @setup_packet_pending: true when there's a Setup Packet in FIFO. Workaround * @three_stage_setup: set if we perform a three phase setup @@ -1094,6 +1096,11 @@ struct dwc3_scratchpad_array { * @dis_split_quirk: set to disable split boundary. * @imod_interval: set the interrupt moderation interval in 250ns * increments or 0 to disable. + * @max_cfg_eps: current max number of IN eps used across all USB configs. + * @last_fifo_depth: last fifo depth used to determine next fifo ram start + * address. + * @num_ep_resized: carries the current number endpoints which have had its tx + * fifo resized. */ struct dwc3 { struct work_struct drd_work; @@ -1249,6 +1256,7 @@ struct dwc3 { u8 rx_max_burst_prd; u8 tx_thr_num_pkt_prd; u8 tx_max_burst_prd; + u8 tx_fifo_resize_max_num; const char *hsphy_interface; @@ -1262,6 +1270,7 @@ struct dwc3 { unsigned is_utmi_l1_suspend:1; unsigned is_fpga:1; unsigned pending_events:1; + unsigned do_fifo_resize:1; unsigned pullups_connected:1; unsigned setup_packet_pending:1; unsigned three_stage_setup:1; @@ -1295,9 +1304,14 @@ struct dwc3 { unsigned dis_metastability_quirk:1; unsigned dis_split_quirk:1; + unsigned async_callbacks:1; u16 imod_interval; + int max_cfg_eps; + int last_fifo_depth; + int num_ep_resized; + ANDROID_KABI_RESERVE(1); ANDROID_KABI_RESERVE(2); ANDROID_KABI_RESERVE(3); @@ -1534,6 +1548,7 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd, u32 param); void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt); +void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc); #else static inline int dwc3_gadget_init(struct dwc3 *dwc) { return 0; } @@ -1556,6 +1571,8 @@ static inline int dwc3_send_gadget_generic_command(struct dwc3 *dwc, static inline void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) { } +static inline void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc) +{ } #endif #if IS_ENABLED(CONFIG_USB_DWC3_DUAL_ROLE) diff --git a/drivers/usb/dwc3/dwc3-qcom.c b/drivers/usb/dwc3/dwc3-qcom.c index 8bd077fb1190..a47af6082cc4 100644 --- a/drivers/usb/dwc3/dwc3-qcom.c +++ b/drivers/usb/dwc3/dwc3-qcom.c @@ -640,6 +640,7 @@ static int dwc3_qcom_of_register_core(struct platform_device *pdev) struct dwc3_qcom *qcom = platform_get_drvdata(pdev); struct device_node *np = pdev->dev.of_node, *dwc3_np; struct device *dev = &pdev->dev; + struct property *prop; int ret; dwc3_np = of_get_child_by_name(np, "dwc3"); @@ -648,6 +649,20 @@ static int dwc3_qcom_of_register_core(struct platform_device *pdev) return -ENODEV; } + prop = devm_kzalloc(dev, sizeof(*prop), GFP_KERNEL); + if (!prop) { + ret = -ENOMEM; + dev_err(dev, "unable to allocate memory for property\n"); + goto node_put; + } + + prop->name = "tx-fifo-resize"; + ret = of_add_property(dwc3_np, prop); + if (ret) { + dev_err(dev, "unable to add property\n"); + goto node_put; + } + ret = of_platform_populate(np, NULL, NULL, dev); if (ret) { dev_err(dev, "failed to register dwc3 core - %d\n", ret); diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index 3cd294264372..658739410992 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -597,11 +597,13 @@ static int dwc3_ep0_set_address(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl) static int dwc3_ep0_delegate_req(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl) { - int ret; + int ret = -EINVAL; - spin_unlock(&dwc->lock); - ret = dwc->gadget_driver->setup(dwc->gadget, ctrl); - spin_lock(&dwc->lock); + if (dwc->async_callbacks) { + spin_unlock(&dwc->lock); + ret = dwc->gadget_driver->setup(dwc->gadget, ctrl); + spin_lock(&dwc->lock); + } return ret; } @@ -619,6 +621,8 @@ static int dwc3_ep0_set_config(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl) return -EINVAL; case USB_STATE_ADDRESS: + dwc3_gadget_clear_tx_fifos(dwc); + ret = dwc3_ep0_delegate_req(dwc, ctrl); /* if the cfg matches and the cfg is non zero */ if (cfg && (!ret || (ret == USB_GADGET_DELAYED_STATUS))) { diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 99e9d7c5584c..74ad24fe098a 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -629,6 +629,187 @@ static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action) return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms); } +/** + * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value + * @dwc: pointer to the DWC3 context + * @nfifos: number of fifos to calculate for + * + * Calculates the size value based on the equation below: + * + * DWC3 revision 280A and prior: + * fifo_size = mult * (max_packet / mdwidth) + 1; + * + * DWC3 revision 290A and onwards: + * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 + * + * The max packet size is set to 1024, as the txfifo requirements mainly apply + * to super speed USB use cases. However, it is safe to overestimate the fifo + * allocations for other scenarios, i.e. high speed USB. + */ +static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult) +{ + int max_packet = 1024; + int fifo_size; + int mdwidth; + + mdwidth = dwc3_mdwidth(dwc); + + /* MDWIDTH is represented in bits, we need it in bytes */ + mdwidth >>= 3; + + if (DWC3_VER_IS_PRIOR(DWC3, 290A)) + fifo_size = mult * (max_packet / mdwidth) + 1; + else + fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1; + return fifo_size; +} + +/** + * dwc3_gadget_clear_tx_fifo_size - Clears txfifo allocation + * @dwc: pointer to the DWC3 context + * + * Iterates through all the endpoint registers and clears the previous txfifo + * allocations. + */ +void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc) +{ + struct dwc3_ep *dep; + int fifo_depth; + int size; + int num; + + if (!dwc->do_fifo_resize) + return; + + /* Read ep0IN related TXFIFO size */ + dep = dwc->eps[1]; + size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); + if (DWC3_IP_IS(DWC3)) + fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size); + else + fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size); + + dwc->last_fifo_depth = fifo_depth; + /* Clear existing TXFIFO for all IN eps except ep0 */ + for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM); + num += 2) { + dep = dwc->eps[num]; + /* Don't change TXFRAMNUM on usb31 version */ + size = DWC3_IP_IS(DWC3) ? 0 : + dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) & + DWC31_GTXFIFOSIZ_TXFRAMNUM; + + dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size); + } + dwc->num_ep_resized = 0; +} + +/* + * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case + * @dwc: pointer to our context structure + * + * This function will a best effort FIFO allocation in order + * to improve FIFO usage and throughput, while still allowing + * us to enable as many endpoints as possible. + * + * Keep in mind that this operation will be highly dependent + * on the configured size for RAM1 - which contains TxFifo -, + * the amount of endpoints enabled on coreConsultant tool, and + * the width of the Master Bus. + * + * In general, FIFO depths are represented with the following equation: + * + * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 + * + * In conjunction with dwc3_gadget_check_config(), this resizing logic will + * ensure that all endpoints will have enough internal memory for one max + * packet per endpoint. + */ +static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep) +{ + struct dwc3 *dwc = dep->dwc; + int fifo_0_start; + int ram1_depth; + int fifo_size; + int min_depth; + int num_in_ep; + int remaining; + int num_fifos = 1; + int fifo; + int tmp; + + if (!dwc->do_fifo_resize) + return 0; + + /* resize IN endpoints except ep0 */ + if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1) + return 0; + + ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); + + if ((dep->endpoint.maxburst > 1 && + usb_endpoint_xfer_bulk(dep->endpoint.desc)) || + usb_endpoint_xfer_isoc(dep->endpoint.desc)) + num_fifos = 3; + + if (dep->endpoint.maxburst > 6 && + usb_endpoint_xfer_bulk(dep->endpoint.desc) && DWC3_IP_IS(DWC31)) + num_fifos = dwc->tx_fifo_resize_max_num; + + /* FIFO size for a single buffer */ + fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1); + + /* Calculate the number of remaining EPs w/o any FIFO */ + num_in_ep = dwc->max_cfg_eps; + num_in_ep -= dwc->num_ep_resized; + + /* Reserve at least one FIFO for the number of IN EPs */ + min_depth = num_in_ep * (fifo + 1); + remaining = ram1_depth - min_depth - dwc->last_fifo_depth; + remaining = max_t(int, 0, remaining); + /* + * We've already reserved 1 FIFO per EP, so check what we can fit in + * addition to it. If there is not enough remaining space, allocate + * all the remaining space to the EP. + */ + fifo_size = (num_fifos - 1) * fifo; + if (remaining < fifo_size) + fifo_size = remaining; + + fifo_size += fifo; + /* Last increment according to the TX FIFO size equation */ + fifo_size++; + + /* Check if TXFIFOs start at non-zero addr */ + tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); + fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp); + + fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16)); + if (DWC3_IP_IS(DWC3)) + dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); + else + dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); + + /* Check fifo size allocation doesn't exceed available RAM size. */ + if (dwc->last_fifo_depth >= ram1_depth) { + dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n", + dwc->last_fifo_depth, ram1_depth, + dep->endpoint.name, fifo_size); + if (DWC3_IP_IS(DWC3)) + fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); + else + fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); + + dwc->last_fifo_depth -= fifo_size; + return -ENOMEM; + } + + dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size); + dwc->num_ep_resized++; + + return 0; +} + /** * __dwc3_gadget_ep_enable - initializes a hw endpoint * @dep: endpoint to be initialized @@ -646,6 +827,10 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) int ret; if (!(dep->flags & DWC3_EP_ENABLED)) { + ret = dwc3_gadget_resize_tx_fifos(dep); + if (ret) + return ret; + ret = dwc3_gadget_start_config(dep); if (ret) return ret; @@ -2510,6 +2695,7 @@ static int dwc3_gadget_stop(struct usb_gadget *g) spin_lock_irqsave(&dwc->lock, flags); dwc->gadget_driver = NULL; + dwc->max_cfg_eps = 0; spin_unlock_irqrestore(&dwc->lock, flags); free_irq(dwc->irq_gadget, dwc->ev_buf); @@ -2597,6 +2783,61 @@ static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA) return ret; } +/** + * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration + * @g: pointer to the USB gadget + * + * Used to record the maximum number of endpoints being used in a USB composite + * device. (across all configurations) This is to be used in the calculation + * of the TXFIFO sizes when resizing internal memory for individual endpoints. + * It will help ensured that the resizing logic reserves enough space for at + * least one max packet. + */ +static int dwc3_gadget_check_config(struct usb_gadget *g) +{ + struct dwc3 *dwc = gadget_to_dwc(g); + struct usb_ep *ep; + int fifo_size = 0; + int ram1_depth; + int ep_num = 0; + + if (!dwc->do_fifo_resize) + return 0; + + list_for_each_entry(ep, &g->ep_list, ep_list) { + /* Only interested in the IN endpoints */ + if (ep->claimed && (ep->address & USB_DIR_IN)) + ep_num++; + } + + if (ep_num <= dwc->max_cfg_eps) + return 0; + + /* Update the max number of eps in the composition */ + dwc->max_cfg_eps = ep_num; + + fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps); + /* Based on the equation, increment by one for every ep */ + fifo_size += dwc->max_cfg_eps; + + /* Check if we can fit a single fifo per endpoint */ + ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); + if (fifo_size > ram1_depth) + return -ENOMEM; + + return 0; +} + +static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable) +{ + struct dwc3 *dwc = gadget_to_dwc(g); + unsigned long flags; + + spin_lock_irqsave(&dwc->lock, flags); + dwc->async_callbacks = enable; + spin_unlock_irqrestore(&dwc->lock, flags); +} + static const struct usb_gadget_ops dwc3_gadget_ops = { .get_frame = dwc3_gadget_get_frame, .wakeup = dwc3_gadget_wakeup, @@ -2608,6 +2849,8 @@ static const struct usb_gadget_ops dwc3_gadget_ops = { .udc_set_ssp_rate = dwc3_gadget_set_ssp_rate, .get_config_params = dwc3_gadget_config_params, .vbus_draw = dwc3_gadget_vbus_draw, + .check_config = dwc3_gadget_check_config, + .udc_async_callbacks = dwc3_gadget_async_callbacks, }; /* -------------------------------------------------------------------------- */ @@ -3241,7 +3484,7 @@ static void dwc3_endpoint_interrupt(struct dwc3 *dwc, static void dwc3_disconnect_gadget(struct dwc3 *dwc) { - if (dwc->gadget_driver && dwc->gadget_driver->disconnect) { + if (dwc->async_callbacks && dwc->gadget_driver->disconnect) { spin_unlock(&dwc->lock); dwc->gadget_driver->disconnect(dwc->gadget); spin_lock(&dwc->lock); @@ -3250,7 +3493,7 @@ static void dwc3_disconnect_gadget(struct dwc3 *dwc) static void dwc3_suspend_gadget(struct dwc3 *dwc) { - if (dwc->gadget_driver && dwc->gadget_driver->suspend) { + if (dwc->async_callbacks && dwc->gadget_driver->suspend) { spin_unlock(&dwc->lock); dwc->gadget_driver->suspend(dwc->gadget); spin_lock(&dwc->lock); @@ -3259,7 +3502,7 @@ static void dwc3_suspend_gadget(struct dwc3 *dwc) static void dwc3_resume_gadget(struct dwc3 *dwc) { - if (dwc->gadget_driver && dwc->gadget_driver->resume) { + if (dwc->async_callbacks && dwc->gadget_driver->resume) { spin_unlock(&dwc->lock); dwc->gadget_driver->resume(dwc->gadget); spin_lock(&dwc->lock); @@ -3271,7 +3514,7 @@ static void dwc3_reset_gadget(struct dwc3 *dwc) if (!dwc->gadget_driver) return; - if (dwc->gadget->speed != USB_SPEED_UNKNOWN) { + if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) { spin_unlock(&dwc->lock); usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver); spin_lock(&dwc->lock); @@ -3601,7 +3844,7 @@ static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc) * implemented. */ - if (dwc->gadget_driver && dwc->gadget_driver->resume) { + if (dwc->async_callbacks && dwc->gadget_driver->resume) { spin_unlock(&dwc->lock); dwc->gadget_driver->resume(dwc->gadget); spin_lock(&dwc->lock); diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index 9fa88eb25af2..2d7b7cd6d3a6 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -1436,6 +1436,10 @@ static int configfs_composite_bind(struct usb_gadget *gadget, goto err_purge_funcs; } } + ret = usb_gadget_check_config(cdev->gadget); + if (ret) + goto err_purge_funcs; + usb_ep_autoconfig_reset(cdev->gadget); } if (cdev->use_os_string) { diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index 1ecb94f2737f..edb83c8825de 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -1006,6 +1006,25 @@ int usb_gadget_ep_match_desc(struct usb_gadget *gadget, } EXPORT_SYMBOL_GPL(usb_gadget_ep_match_desc); +/** + * usb_gadget_check_config - checks if the UDC can support the binded + * configuration + * @gadget: controller to check the USB configuration + * + * Ensure that a UDC is able to support the requested resources by a + * configuration, and that there are no resource limitations, such as + * internal memory allocated to all requested endpoints. + * + * Returns zero on success, else a negative errno. + */ +int usb_gadget_check_config(struct usb_gadget *gadget) +{ + if (gadget->ops->check_config) + return gadget->ops->check_config(gadget); + return 0; +} +EXPORT_SYMBOL_GPL(usb_gadget_check_config); + /* ------------------------------------------------------------------------- */ static void usb_gadget_state_work(struct work_struct *work) @@ -1150,6 +1169,53 @@ static inline void usb_gadget_udc_set_speed(struct usb_udc *udc, gadget->ops->udc_set_speed(gadget, s); } +/** + * usb_gadget_enable_async_callbacks - tell usb device controller to enable asynchronous callbacks + * @udc: The UDC which should enable async callbacks + * + * This routine is used when binding gadget drivers. It undoes the effect + * of usb_gadget_disable_async_callbacks(); the UDC driver should enable IRQs + * (if necessary) and resume issuing callbacks. + * + * This routine will always be called in process context. + */ +static inline void usb_gadget_enable_async_callbacks(struct usb_udc *udc) +{ + struct usb_gadget *gadget = udc->gadget; + + if (gadget->ops->udc_async_callbacks) + gadget->ops->udc_async_callbacks(gadget, true); +} + +/** + * usb_gadget_disable_async_callbacks - tell usb device controller to disable asynchronous callbacks + * @udc: The UDC which should disable async callbacks + * + * This routine is used when unbinding gadget drivers. It prevents a race: + * The UDC driver doesn't know when the gadget driver's ->unbind callback + * runs, so unless it is told to disable asynchronous callbacks, it might + * issue a callback (such as ->disconnect) after the unbind has completed. + * + * After this function runs, the UDC driver must suppress all ->suspend, + * ->resume, ->disconnect, ->reset, and ->setup callbacks to the gadget driver + * until async callbacks are again enabled. A simple-minded but effective + * way to accomplish this is to tell the UDC hardware not to generate any + * more IRQs. + * + * Request completion callbacks must still be issued. However, it's okay + * to defer them until the request is cancelled, since the pull-up will be + * turned off during the time period when async callbacks are disabled. + * + * This routine will always be called in process context. + */ +static inline void usb_gadget_disable_async_callbacks(struct usb_udc *udc) +{ + struct usb_gadget *gadget = udc->gadget; + + if (gadget->ops->udc_async_callbacks) + gadget->ops->udc_async_callbacks(gadget, false); +} + /** * usb_udc_release - release the usb_udc struct * @dev: the dev member within usb_udc @@ -1364,6 +1430,7 @@ static void usb_gadget_remove_driver(struct usb_udc *udc) kobject_uevent(&udc->dev.kobj, KOBJ_CHANGE); usb_gadget_disconnect(udc->gadget); + usb_gadget_disable_async_callbacks(udc); if (udc->gadget->irq) synchronize_irq(udc->gadget->irq); udc->driver->unbind(udc->gadget); @@ -1445,6 +1512,7 @@ static int udc_bind_to_driver(struct usb_udc *udc, struct usb_gadget_driver *dri driver->unbind(udc->gadget); goto err1; } + usb_gadget_enable_async_callbacks(udc); usb_udc_connect_control(udc); kobject_uevent(&udc->dev.kobj, KOBJ_CHANGE); diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index bca461ea6456..8277c2907b8e 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "bus.h" @@ -20,6 +21,7 @@ struct typec_plug { enum typec_plug_index index; struct ida mode_ids; int num_altmodes; + ANDROID_KABI_RESERVE(1); }; struct typec_cable { @@ -28,6 +30,7 @@ struct typec_cable { struct usb_pd_identity *identity; unsigned int active:1; u16 pd_revision; /* 0300H = "3.0" */ + ANDROID_KABI_RESERVE(1); }; struct typec_partner { @@ -39,6 +42,7 @@ struct typec_partner { int num_altmodes; u16 pd_revision; /* 0300H = "3.0" */ enum usb_pd_svdm_ver svdm_version; + ANDROID_KABI_RESERVE(1); }; struct typec_port { @@ -60,6 +64,7 @@ struct typec_port { const struct typec_capability *cap; const struct typec_operations *ops; + ANDROID_KABI_RESERVE(1); }; #define to_typec_port(_dev_) container_of(_dev_, struct typec_port, dev) diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c index 181e2f18beae..41bd84335b23 100644 --- a/drivers/virtio/virtio_mem.c +++ b/drivers/virtio/virtio_mem.c @@ -1307,11 +1307,12 @@ static int virtio_mem_mb_unplug_sb_online(struct virtio_mem *vm, const unsigned long nr_pages = PFN_DOWN(vm->subblock_size) * count; unsigned long start_pfn; int rc; + struct acr_info dummy; start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) + sb_id * vm->subblock_size); rc = alloc_contig_range(start_pfn, start_pfn + nr_pages, - MIGRATE_MOVABLE, GFP_KERNEL); + MIGRATE_MOVABLE, GFP_KERNEL, &dummy); if (rc == -ENOMEM) /* whoops, out of memory */ return rc; diff --git a/include/linux/amba/bus.h b/include/linux/amba/bus.h index 6cc93ab5b809..55e2f02bd45a 100644 --- a/include/linux/amba/bus.h +++ b/include/linux/amba/bus.h @@ -17,6 +17,7 @@ #include #include #include +#include #define AMBA_NR_IRQS 9 #define AMBA_CID 0xb105f00d @@ -71,6 +72,8 @@ struct amba_device { struct amba_cs_uci_id uci; unsigned int irq[AMBA_NR_IRQS]; char *driver_override; + + ANDROID_KABI_RESERVE(1); }; struct amba_driver { @@ -79,6 +82,8 @@ struct amba_driver { void (*remove)(struct amba_device *); void (*shutdown)(struct amba_device *); const struct amba_id *id_table; + + ANDROID_KABI_RESERVE(1); }; /* diff --git a/include/linux/android_debug_symbols.h b/include/linux/android_debug_symbols.h index 3fc44fb36fb7..c80f063ffef4 100644 --- a/include/linux/android_debug_symbols.h +++ b/include/linux/android_debug_symbols.h @@ -29,6 +29,13 @@ enum android_debug_symbol { #endif #ifdef CONFIG_SWAP ADS_NR_SWAP_PAGES, +#endif +#ifdef CONFIG_MMU + ADS_MMAP_MIN_ADDR, +#endif + ADS_STACK_GUARD_GAP, +#ifdef CONFIG_SYSCTL + ADS_SYSCTL_LEGACY_VA_LAYOUT, #endif ADS_END }; diff --git a/include/linux/backing-dev-defs.h b/include/linux/backing-dev-defs.h index fff9367a6348..f3314adb6a8d 100644 --- a/include/linux/backing-dev-defs.h +++ b/include/linux/backing-dev-defs.h @@ -13,6 +13,7 @@ #include #include #include +#include struct page; struct device; @@ -160,6 +161,9 @@ struct bdi_writeback { struct rcu_head rcu; }; #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct backing_dev_info { @@ -198,6 +202,9 @@ struct backing_dev_info { #ifdef CONFIG_DEBUG_FS struct dentry *debug_dir; #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; enum { diff --git a/include/linux/bio.h b/include/linux/bio.h index 23b7a73cd757..0d4d61743b1a 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -10,6 +10,7 @@ #include /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */ #include +#include #define BIO_DEBUG @@ -321,6 +322,10 @@ struct bio_integrity_payload { struct work_struct bip_work; /* I/O completion */ struct bio_vec *bip_vec; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + struct bio_vec bip_inline_vecs[];/* embedded bvec array */ }; @@ -694,6 +699,11 @@ struct bio_set { struct bio_list rescue_list; struct work_struct rescue_work; struct workqueue_struct *rescue_workqueue; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; struct biovec_slab { diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index a42b751da268..3fb97c2af669 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -48,6 +48,11 @@ struct block_device { /* Mutex for freeze */ struct mutex bd_fsfreeze_mutex; struct super_block *bd_fsfreeze_sb; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); } __randomize_layout; /* diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index dcb7b342f0d3..78eeb317027e 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -26,6 +26,8 @@ #include #include #include +#include +#include struct module; struct scsi_ioctl_command; @@ -242,6 +244,8 @@ struct request { */ rq_end_io_fn *end_io; void *end_io_data; + + ANDROID_KABI_RESERVE(1); }; static inline bool blk_op_is_scsi(unsigned int op) @@ -346,6 +350,8 @@ struct queue_limits { unsigned char discard_misaligned; unsigned char raid_partial_stripes_expensive; enum blk_zoned_model zoned; + + ANDROID_KABI_RESERVE(1); }; typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx, @@ -589,6 +595,10 @@ struct request_queue { #define BLK_MAX_WRITE_HINTS 5 u64 write_hints[BLK_MAX_WRITE_HINTS]; + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); ANDROID_OEM_DATA(1); }; @@ -694,18 +704,6 @@ static inline bool queue_is_mq(struct request_queue *q) return q->mq_ops; } -#ifdef CONFIG_PM -static inline enum rpm_status queue_rpm_status(struct request_queue *q) -{ - return q->rpm_status; -} -#else -static inline enum rpm_status queue_rpm_status(struct request_queue *q) -{ - return RPM_ACTIVE; -} -#endif - static inline enum blk_zoned_model blk_queue_zoned_model(struct request_queue *q) { @@ -1870,6 +1868,10 @@ struct block_device_operations { char *(*devnode)(struct gendisk *disk, umode_t *mode); struct module *owner; const struct pr_ops *pr_ops; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_OEM_DATA(1); }; #ifdef CONFIG_COMPAT diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 6460b57b4b79..62ff6b56a929 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -21,6 +21,7 @@ #include #include #include +#include struct bpf_verifier_env; struct bpf_verifier_log; @@ -132,6 +133,9 @@ struct bpf_map_ops { /* bpf_iter info used to open a seq_file */ const struct bpf_iter_seq_info *iter_seq_info; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct bpf_map_memory { @@ -218,6 +222,8 @@ struct bpf_map_dev_ops { int (*map_update_elem)(struct bpf_offloaded_map *map, void *key, void *value, u64 flags); int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); + + ANDROID_KABI_RESERVE(1); }; struct bpf_offloaded_map { @@ -459,6 +465,7 @@ struct bpf_verifier_ops { const struct btf_type *t, int off, int size, enum bpf_access_type atype, u32 *next_btf_id); + ANDROID_KABI_RESERVE(1); }; struct bpf_prog_offload_ops { @@ -474,6 +481,7 @@ struct bpf_prog_offload_ops { int (*prepare)(struct bpf_prog *prog); int (*translate)(struct bpf_prog *prog); void (*destroy)(struct bpf_prog *prog); + ANDROID_KABI_RESERVE(1); }; struct bpf_prog_offload { @@ -851,6 +859,7 @@ struct bpf_prog_aux { struct work_struct work; struct rcu_head rcu; }; + ANDROID_KABI_RESERVE(1); }; struct bpf_array_aux { diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 2739a6431b9e..b83f1dd5c719 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -7,6 +7,7 @@ #include /* for enum bpf_reg_type */ #include /* for MAX_BPF_STACK */ #include +#include /* Maximum variable offset umax_value permitted when resolving memory accesses. * In practice this is far bigger than any realistic pointer offset; this limit @@ -370,6 +371,8 @@ struct bpf_subprog_info { bool has_tail_call; bool tail_call_reachable; bool has_ld_abs; + + ANDROID_KABI_RESERVE(1); }; /* single container for all structs @@ -425,6 +428,9 @@ struct bpf_verifier_env { u32 peak_states; /* longest register parentage chain walked for liveness marking */ u32 longest_mark_read_walk; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 14b808b02872..67b2190b1dae 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -24,6 +24,7 @@ #include #include #include +#include #include @@ -66,6 +67,8 @@ struct css_task_iter { struct css_set *cur_dcset; struct task_struct *cur_task; struct list_head iters_node; /* css_set->task_iters */ + + ANDROID_KABI_RESERVE(1); }; extern struct cgroup_root cgrp_dfl_root; diff --git a/include/linux/cleancache.h b/include/linux/cleancache.h index 5f5730c1d324..b6c42ce7f65b 100644 --- a/include/linux/cleancache.h +++ b/include/linux/cleancache.h @@ -5,6 +5,7 @@ #include #include #include +#include #define CLEANCACHE_NO_POOL -1 #define CLEANCACHE_NO_BACKEND -2 @@ -36,6 +37,7 @@ struct cleancache_ops { void (*invalidate_page)(int, struct cleancache_filekey, pgoff_t); void (*invalidate_inode)(int, struct cleancache_filekey); void (*invalidate_fs)(int); + ANDROID_OEM_DATA(1); }; extern int cleancache_register_ops(const struct cleancache_ops *ops); diff --git a/include/linux/cma.h b/include/linux/cma.h index d6c02d08ddbc..4f29ac17067d 100644 --- a/include/linux/cma.h +++ b/include/linux/cma.h @@ -22,6 +22,15 @@ struct cma; +struct cma_alloc_info { + unsigned long nr_migrated; + unsigned long nr_reclaimed; + unsigned long nr_mapped; + unsigned int nr_isolate_fail; + unsigned int nr_migrate_fail; + unsigned int nr_test_fail; +}; + extern unsigned long totalcma_pages; extern phys_addr_t cma_get_base(const struct cma *cma); extern unsigned long cma_get_size(const struct cma *cma); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 1366cfbfdc78..843bb056b8c6 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -195,6 +195,10 @@ enum cpuhp_state { CPUHP_AP_X86_KVM_CLK_ONLINE, CPUHP_AP_DTPM_CPU_ONLINE, CPUHP_AP_ACTIVE, + CPUHP_ANDROID_RESERVED_1, + CPUHP_ANDROID_RESERVED_2, + CPUHP_ANDROID_RESERVED_3, + CPUHP_ANDROID_RESERVED_4, CPUHP_ONLINE, }; diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h index bd605b5585cf..4b51019dd2b8 100644 --- a/include/linux/cpuidle.h +++ b/include/linux/cpuidle.h @@ -14,6 +14,7 @@ #include #include #include +#include #define CPUIDLE_STATE_MAX 10 #define CPUIDLE_NAME_LEN 16 @@ -110,6 +111,8 @@ struct cpuidle_device { cpumask_t coupled_cpus; struct cpuidle_coupled *coupled; #endif + + ANDROID_KABI_RESERVE(1); }; DECLARE_PER_CPU(struct cpuidle_device *, cpuidle_devices); @@ -135,6 +138,8 @@ struct cpuidle_driver { /* preferred governor to switch at register time */ const char *governor; + + ANDROID_KABI_RESERVE(1); }; #ifdef CONFIG_CPU_IDLE diff --git a/include/linux/dcache.h b/include/linux/dcache.h index 49d0b6f732f2..8836cf7c4874 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -13,6 +13,7 @@ #include #include #include +#include struct path; struct vfsmount; @@ -118,6 +119,9 @@ struct dentry { struct hlist_bl_node d_in_lookup_hash; /* only for in-lookup ones */ struct rcu_head d_rcu; } d_u; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); } __randomize_layout; /* @@ -148,6 +152,10 @@ struct dentry_operations { int (*d_manage)(const struct path *, bool); struct dentry *(*d_real)(struct dentry *, const struct inode *); void (*d_canonical_path)(const struct path *, struct path *); + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); } ____cacheline_aligned; /* diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 8858ffd6fb10..ccb909edca38 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -13,6 +13,7 @@ #include #include #include +#include struct dm_dev; struct dm_target; @@ -198,6 +199,9 @@ struct target_type { dm_dax_copy_iter_fn dax_copy_to_iter; dm_dax_zero_page_range_fn dax_zero_page_range; + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + /* For internal device-mapper use. */ struct list_head list; }; @@ -349,6 +353,9 @@ struct dm_target { * Set if we need to limit the number of in-flight bios when swapping. */ bool limit_swap_bios:1; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; void *dm_per_bio_data(struct bio *bio, size_t data_size); diff --git a/include/linux/device.h b/include/linux/device.h index 4e1e42798879..540a14f1e052 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -30,6 +30,7 @@ #include #include #include +#include #include struct device; @@ -553,6 +554,14 @@ struct device { #ifdef CONFIG_DMA_OPS_BYPASS bool dma_ops_bypass : 1; #endif + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); + ANDROID_KABI_RESERVE(5); + ANDROID_KABI_RESERVE(6); + ANDROID_KABI_RESERVE(7); + ANDROID_KABI_RESERVE(8); }; /** diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index 1ea5e1d1545b..93ed259748f8 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -112,6 +112,11 @@ struct bus_type { struct lock_class_key lock_key; bool need_parent_lock; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; extern int __must_check bus_register(struct bus_type *bus); diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 04e983415c3d..9923e4835526 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -76,7 +76,11 @@ struct class { const struct dev_pm_ops *pm; struct subsys_private *p; + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; struct class_dev_iter { diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index ee7ba5b5417e..d5ad474e4fab 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -118,6 +118,11 @@ struct device_driver { void (*coredump) (struct device *dev); struct driver_private *p; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index 303f559d7b30..63ffe57a61d5 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -21,6 +21,7 @@ #include #include #include +#include struct device; struct dma_buf; @@ -353,6 +354,9 @@ struct dma_buf_ops { * will be populated with the buffer's flags. */ int (*get_flags)(struct dma_buf *dmabuf, unsigned long *flags); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /** @@ -379,8 +383,6 @@ struct dma_buf_ops { * @cb_excl: for userspace poll support * @cb_shared: for userspace poll support * @sysfs_entry: for exposing information about this buffer in sysfs. - * The attachment_uid member of @sysfs_entry is protected by dma_resv lock - * and is incremented on each attach. * * This represents a shared buffer, created by calling dma_buf_export(). The * userspace representation is a normal file descriptor, which can be created by @@ -421,10 +423,11 @@ struct dma_buf { struct dma_buf_sysfs_entry { struct kobject kobj; struct dma_buf *dmabuf; - unsigned int attachment_uid; - struct kset *attach_stats_kset; } *sysfs_entry; #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /** @@ -476,7 +479,6 @@ struct dma_buf_attach_ops { * @importer_priv: importer specific attachment data. * @dma_map_attrs: DMA attributes to be used when the exporter maps the buffer * through dma_buf_map_attachment. - * @sysfs_entry: For exposing information about this attachment in sysfs. * * This structure holds the attachment information between the dma_buf buffer * and its user device(s). The list contains one attachment struct per device @@ -498,13 +500,9 @@ struct dma_buf_attachment { void *importer_priv; void *priv; unsigned long dma_map_attrs; -#ifdef CONFIG_DMABUF_SYSFS_STATS - /* for sysfs stats */ - struct dma_buf_attach_sysfs_entry { - struct kobject kobj; - unsigned int map_counter; - } *sysfs_entry; -#endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /** @@ -528,6 +526,9 @@ struct dma_buf_export_info { int flags; struct dma_resv *resv; void *priv; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /** diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 493a047ed0a2..3f8dcaa83a49 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -12,6 +12,7 @@ #include #include #include +#include #include /** @@ -941,6 +942,11 @@ struct dma_device { void (*dbg_summary_show)(struct seq_file *s, struct dma_device *dev); struct dentry *dbg_dev_root; #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; static inline int dmaengine_slave_config(struct dma_chan *chan, diff --git a/include/linux/elevator.h b/include/linux/elevator.h index dcb2f9022c1d..0c75ed0604aa 100644 --- a/include/linux/elevator.h +++ b/include/linux/elevator.h @@ -4,6 +4,7 @@ #include #include +#include #ifdef CONFIG_BLOCK @@ -50,6 +51,11 @@ struct elevator_mq_ops { struct request *(*next_request)(struct request_queue *, struct request *); void (*init_icq)(struct io_cq *); void (*exit_icq)(struct io_cq *); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #define ELV_NAME_MAX (16) @@ -86,6 +92,9 @@ struct elevator_type /* managed by elevator core */ char icq_cache_name[ELV_NAME_MAX + 6]; /* elvname + "_io_cq" */ struct list_head list; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; #define ELV_HASH_BITS 6 diff --git a/include/linux/fs.h b/include/linux/fs.h index 342a93553eaf..090cb2938fdc 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -39,9 +39,11 @@ #include #include #include +#include #include #include +#include struct backing_dev_info; struct bdi_writeback; @@ -413,6 +415,11 @@ struct address_space_operations { int (*swap_activate)(struct swap_info_struct *sis, struct file *file, sector_t *span); void (*swap_deactivate)(struct file *file); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; extern const struct address_space_operations empty_aops; @@ -468,6 +475,11 @@ struct address_space { spinlock_t private_lock; struct list_head private_list; void *private_data; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); } __attribute__((aligned(sizeof(long)))) __randomize_layout; /* * On most architectures that alignment is already the case; but @@ -718,6 +730,9 @@ struct inode { #endif void *i_private; /* fs or device private pointer */ + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); } __randomize_layout; struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode); @@ -952,6 +967,10 @@ struct file { struct address_space *f_mapping; errseq_t f_wb_err; errseq_t f_sb_err; /* for syncfs */ + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_OEM_DATA(1); } __randomize_layout __attribute__((aligned(4))); /* lest something weird decides that 2 is OK */ @@ -1011,6 +1030,9 @@ struct file_lock; struct file_lock_operations { void (*fl_copy_lock)(struct file_lock *, struct file_lock *); void (*fl_release_private)(struct file_lock *); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct lock_manager_operations { @@ -1022,6 +1044,9 @@ struct lock_manager_operations { int (*lm_change)(struct file_lock *, int, struct list_head *); void (*lm_setup)(struct file_lock *, void **); bool (*lm_breaker_owns_lease)(struct file_lock *); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct lock_manager { @@ -1095,6 +1120,10 @@ struct file_lock { unsigned int debug_id; } afs; } fl_u; + + struct list_head android_reserved1; /* not a macro as we might just need it as-is */ + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); } __randomize_layout; struct file_lock_context { @@ -1547,6 +1576,11 @@ struct super_block { spinlock_t s_inode_wblist_lock; struct list_head s_inodes_wb; /* writeback inodes */ + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); } __randomize_layout; /* Helper functions so that in most cases filesystems will @@ -1859,6 +1893,11 @@ struct file_operations { struct file *file_out, loff_t pos_out, loff_t len, unsigned int remap_flags); int (*fadvise)(struct file *, loff_t, loff_t, int); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); } __randomize_layout; struct inode_operations { @@ -1889,6 +1928,11 @@ struct inode_operations { umode_t create_mode); int (*tmpfile) (struct inode *, struct dentry *, umode_t); int (*set_acl)(struct inode *, struct posix_acl *, int); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); } ____cacheline_aligned; static inline ssize_t call_read_iter(struct file *file, struct kiocb *kio, @@ -1965,6 +2009,11 @@ struct super_operations { struct shrink_control *); long (*free_cached_objects)(struct super_block *, struct shrink_control *); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /* @@ -2252,6 +2301,11 @@ struct file_system_type { struct lock_class_key i_lock_key; struct lock_class_key i_mutex_key; struct lock_class_key i_mutex_dir_key; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #define MODULE_ALIAS_FS(NAME) MODULE_ALIAS("fs-" NAME) diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h index 554b219c62cd..3eb7cf0cb34d 100644 --- a/include/linux/fscrypt.h +++ b/include/linux/fscrypt.h @@ -70,6 +70,13 @@ struct fscrypt_operations { int (*get_num_devices)(struct super_block *sb); void (*get_devices)(struct super_block *sb, struct request_queue **devs); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); + + ANDROID_OEM_DATA_ARRAY(1, 4); }; static inline struct fscrypt_info *fscrypt_get_info(const struct inode *inode) diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 44f613870fa3..fbed5dd273db 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -642,9 +642,24 @@ static inline bool pm_suspended_storage(void) #endif /* CONFIG_PM_SLEEP */ #ifdef CONFIG_CONTIG_ALLOC +extern unsigned long pfn_max_align_up(unsigned long pfn); + +#define ACR_ERR_ISOLATE (1 << 0) +#define ACR_ERR_MIGRATE (1 << 1) +#define ACR_ERR_TEST (1 << 2) + +struct acr_info { + unsigned long nr_mapped; + unsigned long nr_migrated; + unsigned long nr_reclaimed; + unsigned int err; + unsigned long failed_pfn; +}; + /* The below functions must be run on a range from a single zone. */ extern int alloc_contig_range(unsigned long start, unsigned long end, - unsigned migratetype, gfp_t gfp_mask); + unsigned migratetype, gfp_t gfp_mask, + struct acr_info *info); extern struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask, int nid, nodemask_t *nodemask); #endif diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 8e144306e262..ff3e6e3eb9c1 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -10,6 +10,7 @@ #include #include #include +#include struct gpio_desc; struct of_phandle_args; @@ -266,6 +267,9 @@ struct gpio_irq_chip { * Store old irq_chip irq_mask callback */ void (*irq_mask)(struct irq_data *data); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /** @@ -469,6 +473,9 @@ struct gpio_chip { int (*of_xlate)(struct gpio_chip *gc, const struct of_phandle_args *gpiospec, u32 *flags); #endif /* CONFIG_OF_GPIO */ + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; extern const char *gpiochip_is_requested(struct gpio_chip *gc, diff --git a/include/linux/input.h b/include/linux/input.h index 56f2fd32e609..4d752cedf581 100644 --- a/include/linux/input.h +++ b/include/linux/input.h @@ -7,6 +7,7 @@ #include #include +#include #include /* Implementation details, userspace should not care about these */ #define ABS_MT_FIRST ABS_MT_TOUCH_MAJOR @@ -201,6 +202,11 @@ struct input_dev { bool devres_managed; ktime_t timestamp[INPUT_CLK_MAX]; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #define to_input_dev(d) container_of(d, struct input_dev, dev) @@ -320,6 +326,8 @@ struct input_handler { struct list_head h_list; struct list_head node; + + ANDROID_KABI_RESERVE(1); }; /** @@ -346,6 +354,8 @@ struct input_handle { struct list_head d_node; struct list_head h_node; + + ANDROID_KABI_RESERVE(1); }; struct input_dev __must_check *input_allocate_device(void); @@ -550,6 +560,9 @@ struct ff_device { int max_effects; struct ff_effect *effects; + + ANDROID_KABI_RESERVE(1); + struct file *effect_owners[]; }; diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 8a6add3abb3f..257ffd4d921f 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -89,6 +89,8 @@ struct iomap { void *inline_data; void *private; /* filesystem private */ const struct iomap_page_ops *page_ops; + + ANDROID_KABI_RESERVE(1); }; static inline sector_t diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 32809624d422..6175b3e0d220 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -113,7 +113,7 @@ struct static_key { #endif /* CONFIG_JUMP_LABEL */ #endif /* __ASSEMBLY__ */ -#ifdef CONFIG_JUMP_LABEL +#if defined(CONFIG_JUMP_LABEL) && !defined(BUILD_FIPS140_KO) #include #ifndef __ASSEMBLY__ @@ -188,7 +188,28 @@ enum jump_label_type { struct module; -#ifdef CONFIG_JUMP_LABEL +#ifdef BUILD_FIPS140_KO + +static inline int static_key_count(struct static_key *key) +{ + return atomic_read(&key->enabled); +} + +static __always_inline bool static_key_false(struct static_key *key) +{ + if (unlikely(static_key_count(key) > 0)) + return true; + return false; +} + +static __always_inline bool static_key_true(struct static_key *key) +{ + if (likely(static_key_count(key) > 0)) + return true; + return false; +} + +#elif defined(CONFIG_JUMP_LABEL) #define JUMP_TYPE_FALSE 0UL #define JUMP_TYPE_TRUE 1UL @@ -393,7 +414,7 @@ extern bool ____wrong_branch_error(void); static_key_count((struct static_key *)x) > 0; \ }) -#ifdef CONFIG_JUMP_LABEL +#if defined(CONFIG_JUMP_LABEL) && !defined(BUILD_FIPS140_KO) /* * Combine the right initial value (type) with the right branch order diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index c8529e9fa8de..44349791737a 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -157,6 +157,8 @@ struct kernfs_node { unsigned short flags; umode_t mode; struct kernfs_iattrs *iattr; + + ANDROID_KABI_RESERVE(1); }; /* @@ -198,6 +200,8 @@ struct kernfs_root { struct list_head supers; wait_queue_head_t deactivate_waitq; + + ANDROID_KABI_RESERVE(1); }; struct kernfs_open_file { @@ -218,6 +222,8 @@ struct kernfs_open_file { bool mmapped:1; bool released:1; const struct vm_operations_struct *vm_ops; + + ANDROID_KABI_RESERVE(1); }; struct kernfs_ops { diff --git a/include/linux/key-type.h b/include/linux/key-type.h index 2ab2d6d6aeab..5e85fe05c7b1 100644 --- a/include/linux/key-type.h +++ b/include/linux/key-type.h @@ -10,6 +10,7 @@ #include #include +#include #ifdef CONFIG_KEYS @@ -155,6 +156,9 @@ struct key_type { int (*asym_verify_signature)(struct kernel_pkey_params *params, const void *in, const void *in2); + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + /* internal fields */ struct list_head link; /* link in types list */ struct lock_class_key lock_class; /* key->sem lock class */ diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index f99a651c9d21..394aff2668b9 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -331,6 +331,7 @@ struct mem_cgroup { struct deferred_split deferred_split_queue; #endif + ANDROID_OEM_DATA(1); struct mem_cgroup_per_node *nodeinfo[0]; /* WARNING: nodeinfo must be the last member here */ }; diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 4398d84d2f8b..c853f612a815 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -598,6 +598,8 @@ struct mm_struct { #ifdef CONFIG_IOMMU_SUPPORT u32 pasid; #endif + + ANDROID_KABI_RESERVE(1); } __randomize_layout; /* diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h index 6f13f15e95f0..bb62eae4d88a 100644 --- a/include/linux/mmc/card.h +++ b/include/linux/mmc/card.h @@ -9,6 +9,7 @@ #include #include +#include struct mmc_cid { unsigned int manfid; @@ -236,6 +237,8 @@ struct mmc_part { #define MMC_BLK_DATA_AREA_BOOT (1<<1) #define MMC_BLK_DATA_AREA_GP (1<<2) #define MMC_BLK_DATA_AREA_RPMB (1<<3) + + ANDROID_KABI_RESERVE(1); }; /* @@ -314,6 +317,8 @@ struct mmc_card { unsigned int bouncesz; /* Bounce buffer size */ struct workqueue_struct *complete_wq; /* Private workqueue */ + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); ANDROID_VENDOR_DATA(1); }; diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index 7891e314599d..6fa9e476e778 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -176,6 +177,9 @@ struct mmc_host_ops { */ int (*multi_io_quirk)(struct mmc_card *card, unsigned int direction, int blk_size); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct mmc_cqe_ops { @@ -220,6 +224,9 @@ struct mmc_cqe_ops { * will have zero data bytes transferred. */ void (*cqe_recovery_finish)(struct mmc_host *host); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct mmc_async_req { @@ -485,6 +492,8 @@ struct mmc_host { /* Host Software Queue support */ bool hsq_enabled; + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); ANDROID_VENDOR_DATA(1); ANDROID_OEM_DATA(1); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 6aaebd16ba6d..08ccefa35267 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -778,6 +778,7 @@ typedef struct pglist_data { int kswapd_failures; /* Number of 'reclaimed == 0' runs */ + ANDROID_OEM_DATA(1); #ifdef CONFIG_COMPACTION int kcompactd_max_order; enum zone_type kcompactd_highest_zoneidx; diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h index 0101747de549..a12c285222a8 100644 --- a/include/linux/netfilter.h +++ b/include/linux/netfilter.h @@ -14,6 +14,7 @@ #include #include #include +#include #include static inline int NF_DROP_GETERR(int verdict) @@ -171,6 +172,8 @@ struct nf_sockopt_ops { int (*get)(struct sock *sk, int optval, void __user *user, int *len); /* Use the module struct to lock set/get code in place */ struct module *owner; + + ANDROID_KABI_RESERVE(1); }; /* Function to register/unregister hook points. */ @@ -373,6 +376,8 @@ struct nf_nat_hook { unsigned int (*manip_pkt)(struct sk_buff *skb, struct nf_conn *ct, enum nf_nat_manip_type mtype, enum ip_conntrack_dir dir); + + ANDROID_KABI_RESERVE(1); }; extern struct nf_nat_hook __rcu *nf_nat_hook; @@ -457,6 +462,8 @@ struct nf_ct_hook { void (*destroy)(struct nf_conntrack *); bool (*get_tuple_skb)(struct nf_conntrack_tuple *, const struct sk_buff *); + + ANDROID_KABI_RESERVE(1); }; extern struct nf_ct_hook __rcu *nf_ct_hook; @@ -474,6 +481,8 @@ struct nfnl_ct_hook { u32 portid, u32 report); void (*seq_adjust)(struct sk_buff *skb, struct nf_conn *ct, enum ip_conntrack_info ctinfo, s32 off); + + ANDROID_KABI_RESERVE(1); }; extern struct nfnl_ct_hook __rcu *nfnl_ct_hook; diff --git a/include/linux/netfilter/ipset/ip_set.h b/include/linux/netfilter/ipset/ip_set.h index ab192720e2d6..5206d0991022 100644 --- a/include/linux/netfilter/ipset/ip_set.h +++ b/include/linux/netfilter/ipset/ip_set.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -190,6 +191,8 @@ struct ip_set_type_variant { bool (*same_set)(const struct ip_set *a, const struct ip_set *b); /* Region-locking is used */ bool region_lock; + + ANDROID_KABI_RESERVE(1); }; struct ip_set_region { @@ -228,6 +231,8 @@ struct ip_set_type { /* Set this to THIS_MODULE if you are a module, otherwise NULL */ struct module *me; + + ANDROID_KABI_RESERVE(1); }; /* register and unregister set type */ @@ -270,6 +275,8 @@ struct ip_set { size_t offset[IPSET_EXT_ID_MAX]; /* The type specific data */ void *data; + + ANDROID_KABI_RESERVE(1); }; static inline void diff --git a/include/linux/netfilter/nfnetlink.h b/include/linux/netfilter/nfnetlink.h index f6267e2883f2..15775bc5b174 100644 --- a/include/linux/netfilter/nfnetlink.h +++ b/include/linux/netfilter/nfnetlink.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -22,6 +23,8 @@ struct nfnl_callback { struct netlink_ext_ack *extack); const struct nla_policy *policy; /* netlink attribute policy */ const u_int16_t attr_count; /* number of nlattr's */ + + ANDROID_KABI_RESERVE(1); }; enum nfnl_abort_action { @@ -41,6 +44,8 @@ struct nfnetlink_subsystem { enum nfnl_abort_action action); void (*cleanup)(struct net *net); bool (*valid_genid)(struct net *net, u32 genid); + + ANDROID_KABI_RESERVE(1); }; int nfnetlink_subsys_register(const struct nfnetlink_subsystem *n); diff --git a/include/linux/netfilter_ipv6.h b/include/linux/netfilter_ipv6.h index 48314ade1506..c3d17fe68426 100644 --- a/include/linux/netfilter_ipv6.h +++ b/include/linux/netfilter_ipv6.h @@ -7,6 +7,7 @@ #ifndef __LINUX_IP6_NETFILTER_H #define __LINUX_IP6_NETFILTER_H +#include #include #include @@ -65,6 +66,8 @@ struct nf_ipv6_ops { const struct nf_bridge_frag_data *data, struct sk_buff *)); #endif + + ANDROID_KABI_RESERVE(1); }; #ifdef CONFIG_NETFILTER diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h index 572458016331..ccd3ed46434f 100644 --- a/include/linux/page-isolation.h +++ b/include/linux/page-isolation.h @@ -44,7 +44,8 @@ int move_freepages_block(struct zone *zone, struct page *page, */ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, - unsigned migratetype, int flags); + unsigned migratetype, int flags, + unsigned long *failed_pfn); /* * Changes MIGRATE_ISOLATE to MIGRATE_MOVABLE. @@ -58,7 +59,7 @@ undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, * Test all pages in [start_pfn, end_pfn) are isolated or not. */ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn, - int isol_flags); + int isol_flags, unsigned long *failed_pfn); struct page *alloc_migrate_target(struct page *page, unsigned long private); diff --git a/include/linux/pci.h b/include/linux/pci.h index cc772efa9713..76ea634f24a7 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -558,6 +558,10 @@ struct pci_host_bridge { resource_size_t start, resource_size_t size, resource_size_t align); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + unsigned long private[] ____cacheline_aligned; }; @@ -745,6 +749,8 @@ struct pci_ops { void __iomem *(*map_bus)(struct pci_bus *bus, unsigned int devfn, int where); int (*read)(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val); int (*write)(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val); + + ANDROID_KABI_RESERVE(1); }; /* @@ -820,6 +826,8 @@ struct pci_error_handlers { /* Device driver may resume normal operations */ void (*resume)(struct pci_dev *dev); + + ANDROID_KABI_RESERVE(1); }; diff --git a/include/linux/phy.h b/include/linux/phy.h index 08725a262f32..afeda1c01e3a 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -645,6 +646,11 @@ struct phy_device { /* MACsec management functions */ const struct macsec_ops *macsec_ops; #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #define to_phy_device(d) container_of(to_mdio_device(d), \ struct phy_device, mdio) @@ -875,6 +881,9 @@ struct phy_driver { int (*get_sqi)(struct phy_device *dev); /** @get_sqi_max: Get the maximum signal quality indication */ int (*get_sqi_max)(struct phy_device *dev); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; #define to_phy_driver(d) container_of(to_mdio_common_driver(d), \ struct phy_driver, mdiodrv) diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h index 17f9cd5626c8..72b8a954e6bb 100644 --- a/include/linux/platform_device.h +++ b/include/linux/platform_device.h @@ -11,6 +11,7 @@ #define _PLATFORM_DEVICE_H_ #include +#include #define PLATFORM_DEVID_NONE (-1) #define PLATFORM_DEVID_AUTO (-2) @@ -37,6 +38,9 @@ struct platform_device { /* arch specific additions */ struct pdev_archdata archdata; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; #define platform_get_device_id(pdev) ((pdev)->id_entry) @@ -94,6 +98,8 @@ struct platform_device_info { u64 dma_mask; const struct property_entry *properties; + + ANDROID_KABI_RESERVE(1); }; extern struct platform_device *platform_device_register_full( const struct platform_device_info *pdevinfo); @@ -206,6 +212,8 @@ struct platform_driver { struct device_driver driver; const struct platform_device_id *id_table; bool prevent_deferred_probe; + + ANDROID_KABI_RESERVE(1); }; #define to_platform_driver(drv) (container_of((drv), struct platform_driver, \ diff --git a/include/linux/pm.h b/include/linux/pm.h index 52d9724db9dc..8b174a8f9f04 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -15,6 +15,7 @@ #include #include #include +#include /* * Callbacks for platform drivers to implement. @@ -299,6 +300,8 @@ struct dev_pm_ops { int (*runtime_suspend)(struct device *dev); int (*runtime_resume)(struct device *dev); int (*runtime_idle)(struct device *dev); + + ANDROID_KABI_RESERVE(1); }; #ifdef CONFIG_PM_SLEEP @@ -621,6 +624,9 @@ struct dev_pm_info { struct pm_subsys_data *subsys_data; /* Owned by the subsystem. */ void (*set_latency_tolerance)(struct device *, s32); struct dev_pm_qos *qos; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; extern int dev_pm_get_subsys_data(struct device *dev); @@ -647,6 +653,8 @@ struct dev_pm_domain { int (*activate)(struct device *dev); void (*sync)(struct device *dev); void (*dismiss)(struct device *dev); + + ANDROID_KABI_RESERVE(1); }; /* diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index aefa0b0bc4c4..a8fa073a09b6 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -17,6 +17,7 @@ #include #include #include +#include /* * All voltages, currents, charges, energies, time and temperatures in uV, @@ -233,6 +234,8 @@ struct power_supply_config { char **supplied_to; size_t num_supplicants; + + ANDROID_KABI_RESERVE(1); }; /* Description of power supply */ @@ -274,6 +277,8 @@ struct power_supply_desc { bool no_thermal; /* For APM emulation, think legacy userspace. */ int use_for_apm; + + ANDROID_KABI_RESERVE(1); }; struct power_supply { @@ -315,6 +320,8 @@ struct power_supply { struct led_trigger *charging_blink_full_solid_trig; char *charging_blink_full_solid_trig_name; #endif + + ANDROID_KABI_RESERVE(1); }; /* @@ -382,6 +389,8 @@ struct power_supply_battery_info { int ocv_table_size[POWER_SUPPLY_OCV_TEMP_MAX]; struct power_supply_resistance_temp_table *resist_table; int resist_table_size; + + ANDROID_KABI_RESERVE(1); }; extern struct atomic_notifier_head power_supply_notifier; diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h index b95f3211566a..4b73f0f19ef5 100644 --- a/include/linux/psi_types.h +++ b/include/linux/psi_types.h @@ -157,6 +157,7 @@ struct psi_group { struct timer_list poll_timer; wait_queue_head_t poll_wait; atomic_t poll_wakeup; + atomic_t poll_scheduled; /* Protects data used by the monitor */ struct mutex trigger_lock; diff --git a/include/linux/pwm.h b/include/linux/pwm.h index f0a7329ba3b2..72fe459f0267 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -5,6 +5,7 @@ #include #include #include +#include struct pwm_capture; struct seq_file; @@ -97,6 +98,8 @@ struct pwm_device { struct pwm_args args; struct pwm_state state; struct pwm_state last; + + ANDROID_KABI_RESERVE(1); }; /** @@ -298,6 +301,8 @@ struct pwm_ops { enum pwm_polarity polarity); int (*enable)(struct pwm_chip *chip, struct pwm_device *pwm); void (*disable)(struct pwm_chip *chip, struct pwm_device *pwm); + + ANDROID_KABI_RESERVE(1); }; /** @@ -324,6 +329,8 @@ struct pwm_chip { /* only used internally by the PWM framework */ struct list_head list; struct pwm_device *pwms; + + ANDROID_KABI_RESERVE(1); }; /** diff --git a/include/linux/regmap.h b/include/linux/regmap.h index e7834d98207f..751ca3869307 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -19,6 +19,7 @@ #include #include #include +#include struct module; struct clk; @@ -401,6 +402,8 @@ struct regmap_config { unsigned int hwlock_mode; bool can_sleep; + + ANDROID_KABI_RESERVE(1); }; /** @@ -438,6 +441,8 @@ struct regmap_range_cfg { /* Data window (per each page) */ unsigned int window_start; unsigned int window_len; + + ANDROID_KABI_RESERVE(1); }; struct regmap_async; @@ -514,6 +519,8 @@ struct regmap_bus { enum regmap_endian val_format_endian_default; size_t max_raw_read; size_t max_raw_write; + + ANDROID_KABI_RESERVE(1); }; /* diff --git a/include/linux/regulator/driver.h b/include/linux/regulator/driver.h index d7c77ee370f3..4d73dc580d75 100644 --- a/include/linux/regulator/driver.h +++ b/include/linux/regulator/driver.h @@ -17,6 +17,7 @@ #include #include #include +#include struct gpio_desc; struct regmap; @@ -202,6 +203,8 @@ struct regulator_ops { int (*resume)(struct regulator_dev *rdev); int (*set_pull_down) (struct regulator_dev *); + + ANDROID_KABI_RESERVE(1); }; /* @@ -381,6 +384,8 @@ struct regulator_desc { unsigned int poll_enabled_time; unsigned int (*of_map_mode)(unsigned int mode); + + ANDROID_KABI_RESERVE(1); }; /** @@ -473,6 +478,8 @@ struct regulator_dev { /* time when this regulator was disabled last time */ unsigned long last_off_jiffy; + + ANDROID_KABI_RESERVE(1); }; struct regulator_dev * diff --git a/include/linux/rtc.h b/include/linux/rtc.h index 22d1575e4991..d51770334160 100644 --- a/include/linux/rtc.h +++ b/include/linux/rtc.h @@ -16,6 +16,7 @@ #include #include #include +#include #include extern int rtc_month_days(unsigned int month, unsigned int year); @@ -66,6 +67,8 @@ struct rtc_class_ops { int (*alarm_irq_enable)(struct device *, unsigned int enabled); int (*read_offset)(struct device *, long *offset); int (*set_offset)(struct device *, long offset); + + ANDROID_KABI_RESERVE(1); }; struct rtc_device; @@ -140,6 +143,8 @@ struct rtc_device { unsigned int uie_task_active:1; unsigned int uie_timer_active:1; #endif + + ANDROID_KABI_RESERVE(1); }; #define to_rtc_device(d) container_of(d, struct rtc_device, dev) diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 452158253764..173463340280 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -3,6 +3,7 @@ #define _LINUX_SCHED_TOPOLOGY_H #include +#include #include #include @@ -146,6 +147,10 @@ struct sched_domain { struct sched_domain_shared *shared; unsigned int span_weight; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + /* * Span of all CPUs in this domain. * diff --git a/include/linux/sched/user.h b/include/linux/sched/user.h index dd6a413d9565..6d63a5260130 100644 --- a/include/linux/sched/user.h +++ b/include/linux/sched/user.h @@ -6,6 +6,7 @@ #include #include #include +#include /* * Some day this will be a full-fledged user tracking system.. @@ -42,6 +43,9 @@ struct user_struct { /* Miscellaneous per-user rate limit */ struct ratelimit_state ratelimit; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); ANDROID_OEM_DATA_ARRAY(1, 2); }; diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index fb8c8f16a49b..57d23fae3781 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -12,6 +12,7 @@ #include #include #include +#include #define SCMI_MAX_STR_SIZE 16 #define SCMI_MAX_NUM_RATES 16 @@ -82,6 +83,8 @@ struct scmi_clk_proto_ops { u64 rate); int (*enable)(const struct scmi_protocol_handle *ph, u32 clk_id); int (*disable)(const struct scmi_protocol_handle *ph, u32 clk_id); + + ANDROID_KABI_RESERVE(1); }; /** @@ -125,6 +128,8 @@ struct scmi_perf_proto_ops { bool (*fast_switch_possible)(const struct scmi_protocol_handle *ph, struct device *dev); bool (*power_scale_mw_get)(const struct scmi_protocol_handle *ph); + + ANDROID_KABI_RESERVE(1); }; /** @@ -150,6 +155,8 @@ struct scmi_power_proto_ops { u32 state); int (*state_get)(const struct scmi_protocol_handle *ph, u32 domain, u32 *state); + + ANDROID_KABI_RESERVE(1); }; /** @@ -326,6 +333,8 @@ struct scmi_sensor_info { unsigned int resolution; int exponent; struct scmi_range_attrs scalar_attrs; + + ANDROID_KABI_RESERVE(1); }; /* @@ -461,6 +470,8 @@ struct scmi_sensor_proto_ops { u32 sensor_id, u32 *sensor_config); int (*config_set)(const struct scmi_protocol_handle *ph, u32 sensor_id, u32 sensor_config); + + ANDROID_KABI_RESERVE(1); }; /** @@ -481,6 +492,8 @@ struct scmi_reset_proto_ops { int (*reset)(const struct scmi_protocol_handle *ph, u32 domain); int (*assert)(const struct scmi_protocol_handle *ph, u32 domain); int (*deassert)(const struct scmi_protocol_handle *ph, u32 domain); + + ANDROID_KABI_RESERVE(1); }; /** @@ -625,6 +638,8 @@ struct scmi_handle { void (*devm_put_protocol)(struct scmi_device *sdev, u8 proto); const struct scmi_notify_ops *notify_ops; + + ANDROID_KABI_RESERVE(1); }; enum scmi_std_protocol { @@ -653,6 +668,8 @@ struct scmi_device { const char *name; struct device dev; struct scmi_handle *handle; + + ANDROID_KABI_RESERVE(1); }; #define to_scmi_dev(d) container_of(d, struct scmi_device, dev) diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index ff63c2963359..0a361d4b58de 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #ifdef CONFIG_SERIAL_CORE_CONSOLE @@ -80,6 +81,9 @@ struct uart_ops { void (*poll_put_char)(struct uart_port *, unsigned char); int (*poll_get_char)(struct uart_port *); #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; #define NO_POLL_CHAR 0x00ff0000 @@ -256,6 +260,9 @@ struct uart_port { struct gpio_desc *rs485_term_gpio; /* enable RS485 bus termination */ struct serial_iso7816 iso7816; void *private_data; /* generic platform data pointer */ + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; static inline int serial_port_in(struct uart_port *up, int offset) @@ -318,6 +325,8 @@ struct uart_driver { */ struct uart_state *state; struct tty_driver *tty_driver; + + ANDROID_KABI_RESERVE(1); }; void uart_write_wakeup(struct uart_port *port); diff --git a/include/linux/serio.h b/include/linux/serio.h index 6c27d413da92..25d638f584a6 100644 --- a/include/linux/serio.h +++ b/include/linux/serio.h @@ -13,6 +13,7 @@ #include #include #include +#include #include extern struct bus_type serio_bus; @@ -61,6 +62,8 @@ struct serio { * may get indigestion when exposed to concurrent access (i8042). */ struct mutex *ps2_cmd_mutex; + + ANDROID_KABI_RESERVE(1); }; #define to_serio_port(d) container_of(d, struct serio, dev) @@ -79,6 +82,8 @@ struct serio_driver { void (*cleanup)(struct serio *); struct device_driver driver; + + ANDROID_KABI_RESERVE(1); }; #define to_serio_driver(d) container_of(d, struct serio_driver, driver) diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 51b1dcfb5022..1bf2615449fe 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -85,7 +85,7 @@ extern bool shmem_huge_enabled(struct vm_area_struct *vma); extern unsigned long shmem_swap_usage(struct vm_area_struct *vma); extern unsigned long shmem_partial_swap_usage(struct address_space *mapping, pgoff_t start, pgoff_t end); -extern void shmem_mark_page_lazyfree(struct page *page); +extern void shmem_mark_page_lazyfree(struct page *page, bool tail); /* Flag allocation requirements to shmem_getpage */ enum sgp_type { diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index b390fdac1587..f47b06683b36 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -14,6 +14,7 @@ #include #include #include +#include struct dma_chan; struct property_entry; @@ -197,6 +198,9 @@ struct spi_device { /* the statistics */ struct spi_statistics statistics; + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + /* * likely need more hooks for more protocol options affecting how * the controller talks to each chip, like: @@ -281,6 +285,8 @@ struct spi_driver { int (*remove)(struct spi_device *spi); void (*shutdown)(struct spi_device *spi); struct device_driver driver; + + ANDROID_KABI_RESERVE(1); }; static inline struct spi_driver *to_spi_driver(struct device_driver *drv) @@ -667,6 +673,9 @@ struct spi_controller { /* Interrupt enable state during PTP system timestamping */ unsigned long irq_flags; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; static inline void *spi_controller_get_devdata(struct spi_controller *ctlr) @@ -964,6 +973,8 @@ struct spi_transfer { #define SPI_TRANS_FAIL_NO_START BIT(0) u16 error; + + ANDROID_KABI_RESERVE(1); }; /** @@ -1030,6 +1041,8 @@ struct spi_message { /* list of spi_res reources when the spi message is processed */ struct list_head resources; + + ANDROID_KABI_RESERVE(1); }; static inline void spi_message_init_no_memset(struct spi_message *m) @@ -1468,6 +1481,8 @@ struct spi_board_info { */ u32 mode; + ANDROID_KABI_RESERVE(1); + /* ... may need additional spi_device chip config data here. * avoid stuff protocol drivers can set; but include stuff * needed to behave without being bound to a driver: diff --git a/include/linux/spmi.h b/include/linux/spmi.h index 394a3f68bad5..6744f7cfadaf 100644 --- a/include/linux/spmi.h +++ b/include/linux/spmi.h @@ -7,6 +7,7 @@ #include #include #include +#include /* Maximum slave identifier */ #define SPMI_MAX_SLAVE_ID 16 @@ -85,6 +86,7 @@ struct spmi_controller { u8 sid, u16 addr, u8 *buf, size_t len); int (*write_cmd)(struct spmi_controller *ctrl, u8 opcode, u8 sid, u16 addr, const u8 *buf, size_t len); + ANDROID_KABI_RESERVE(1); }; static inline struct spmi_controller *to_spmi_controller(struct device *d) @@ -138,6 +140,7 @@ struct spmi_driver { struct device_driver driver; int (*probe)(struct spmi_device *sdev); void (*remove)(struct spmi_device *sdev); + ANDROID_KABI_RESERVE(1); }; static inline struct spmi_driver *to_spmi_driver(struct device_driver *d) diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 04b337d3a3cb..fa2ea36a7bd7 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #ifdef CONFIG_VT @@ -185,6 +186,8 @@ struct platform_suspend_ops { bool (*suspend_again)(void); void (*end)(void); void (*recover)(void); + + ANDROID_KABI_RESERVE(1); }; struct platform_s2idle_ops { @@ -195,6 +198,8 @@ struct platform_s2idle_ops { void (*restore_early)(void); void (*restore)(void); void (*end)(void); + + ANDROID_KABI_RESERVE(1); }; #ifdef CONFIG_SUSPEND @@ -426,6 +431,8 @@ struct platform_hibernation_ops { int (*pre_restore)(void); void (*restore_cleanup)(void); void (*recover)(void); + + ANDROID_KABI_RESERVE(1); }; #ifdef CONFIG_HIBERNATION diff --git a/include/linux/swap.h b/include/linux/swap.h index aabda9cb7bcd..f22b2ee01ec9 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -342,19 +342,9 @@ extern void lru_add_page_tail(struct page *page, struct page *page_tail, struct lruvec *lruvec, struct list_head *head); extern void mark_page_accessed(struct page *); -extern atomic_t lru_disable_count; - -static inline bool lru_cache_disabled(void) -{ - return atomic_read(&lru_disable_count); -} - -static inline void lru_cache_enable(void) -{ - atomic_dec(&lru_disable_count); -} - +extern bool lru_cache_disabled(void); extern void lru_cache_disable(void); +extern void lru_cache_enable(void); extern void lru_add_drain(void); extern void lru_add_drain_cpu(int cpu); extern void lru_add_drain_cpu_zone(struct zone *zone); @@ -363,7 +353,7 @@ extern void rotate_reclaimable_page(struct page *page); extern void deactivate_file_page(struct page *page); extern void deactivate_page(struct page *page); extern void mark_page_lazyfree(struct page *page); -extern void mark_page_lazyfree_movetail(struct page *page); +extern void mark_page_lazyfree_movetail(struct page *page, bool tail); extern void swap_setup(void); extern void __lru_cache_add_inactive_or_unevictable(struct page *page, diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 2f87377e9af7..b163dbd6875c 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -412,6 +413,8 @@ struct tcp_sock { */ struct request_sock __rcu *fastopen_rsk; struct saved_syn *saved_syn; + + ANDROID_KABI_RESERVE(1); }; enum tsq_enum { diff --git a/include/linux/thermal.h b/include/linux/thermal.h index fa9c9eae22a0..a7c121d53960 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #define THERMAL_TRIPS_NONE -1 @@ -79,6 +80,7 @@ struct thermal_zone_device_ops { enum thermal_trend *); int (*notify) (struct thermal_zone_device *, int, enum thermal_trip_type); + ANDROID_KABI_RESERVE(1); }; struct thermal_cooling_device_ops { @@ -88,6 +90,8 @@ struct thermal_cooling_device_ops { int (*get_requested_power)(struct thermal_cooling_device *, u32 *); int (*state2power)(struct thermal_cooling_device *, unsigned long, u32 *); int (*power2state)(struct thermal_cooling_device *, u32, unsigned long *); + + ANDROID_KABI_RESERVE(1); }; struct thermal_cooling_device { @@ -102,6 +106,8 @@ struct thermal_cooling_device { struct mutex lock; /* protect thermal_instances list */ struct list_head thermal_instances; struct list_head node; + + ANDROID_KABI_RESERVE(1); }; /** @@ -179,6 +185,8 @@ struct thermal_zone_device { struct list_head node; struct delayed_work poll_queue; enum thermal_notify_event notify_event; + + ANDROID_KABI_RESERVE(1); }; /** @@ -199,6 +207,8 @@ struct thermal_governor { void (*unbind_from_tz)(struct thermal_zone_device *tz); int (*throttle)(struct thermal_zone_device *tz, int trip); struct list_head governor_list; + + ANDROID_KABI_RESERVE(1); }; /* Structure that holds binding parameters for a zone */ @@ -234,6 +244,8 @@ struct thermal_bind_params { unsigned long *binding_limits; int (*match) (struct thermal_zone_device *tz, struct thermal_cooling_device *cdev); + + ANDROID_KABI_RESERVE(1); }; /* Structure to define Thermal Zone parameters */ @@ -287,6 +299,8 @@ struct thermal_zone_params { * Used by thermal zone drivers (default 0). */ int offset; + + ANDROID_KABI_RESERVE(1); }; /** @@ -311,6 +325,8 @@ struct thermal_zone_of_device_ops { int (*set_trips)(void *, int, int); int (*set_emul_temp)(void *, int); int (*set_trip_temp)(void *, int, int); + + ANDROID_KABI_RESERVE(1); }; /* Function declarations */ diff --git a/include/linux/tty.h b/include/linux/tty.h index 94fa384a650b..bd76f2f1cde0 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -14,6 +14,7 @@ #include #include #include +#include /* @@ -218,6 +219,8 @@ struct tty_port_operations { int (*activate)(struct tty_port *port, struct tty_struct *tty); /* Called on the final put of a port */ void (*destruct)(struct tty_port *port); + + ANDROID_KABI_RESERVE(1); }; struct tty_port_client_operations { @@ -252,6 +255,8 @@ struct tty_port { set to size of fifo */ struct kref kref; /* Ref counter */ void *client_data; + + ANDROID_KABI_RESERVE(1); }; /* tty_port::iflags bits -- use atomic bit ops */ @@ -345,6 +350,9 @@ struct tty_struct { /* If the tty has a pending do_SAK, queue it here - akpm */ struct work_struct SAK_work; struct tty_port *port; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); } __randomize_layout; /* Each of a tty's open files has private_data pointing to tty_file_private */ diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h index 5d509c125171..61f29370e1c3 100644 --- a/include/linux/tty_driver.h +++ b/include/linux/tty_driver.h @@ -237,6 +237,7 @@ #include #include #include +#include struct tty_struct; struct tty_driver; @@ -292,6 +293,9 @@ struct tty_operations { void (*poll_put_char)(struct tty_driver *driver, int line, char ch); #endif int (*proc_show)(struct seq_file *, void *); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); } __randomize_layout; struct tty_driver { @@ -326,6 +330,9 @@ struct tty_driver { const struct tty_operations *ops; struct list_head tty_drivers; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); } __randomize_layout; extern struct list_head tty_drivers; diff --git a/include/linux/tty_ldisc.h b/include/linux/tty_ldisc.h index 572a07976116..689ba487e752 100644 --- a/include/linux/tty_ldisc.h +++ b/include/linux/tty_ldisc.h @@ -126,6 +126,7 @@ #include #include #include +#include /* * the semaphore definition @@ -211,6 +212,9 @@ struct tty_ldisc_ops { struct module *owner; int refcount; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct tty_ldisc { diff --git a/include/linux/uio_driver.h b/include/linux/uio_driver.h index 54bf6b118401..77131e8fefcc 100644 --- a/include/linux/uio_driver.h +++ b/include/linux/uio_driver.h @@ -16,6 +16,7 @@ #include #include #include +#include struct module; struct uio_map; @@ -77,6 +78,8 @@ struct uio_device { struct mutex info_lock; struct kobject *map_dir; struct kobject *portio_dir; + + ANDROID_KABI_RESERVE(1); }; /** @@ -109,6 +112,7 @@ struct uio_info { int (*open)(struct uio_info *info, struct inode *inode); int (*release)(struct uio_info *info, struct inode *inode); int (*irqcontrol)(struct uio_info *info, s32 irq_on); + ANDROID_KABI_RESERVE(1); }; extern int __must_check diff --git a/include/linux/usb/gadget.h b/include/linux/usb/gadget.h index ea02847923bf..2286ff72ab7e 100644 --- a/include/linux/usb/gadget.h +++ b/include/linux/usb/gadget.h @@ -123,6 +123,8 @@ struct usb_request { int status; unsigned actual; + + ANDROID_KABI_RESERVE(1); }; /*-------------------------------------------------------------------------*/ @@ -153,6 +155,8 @@ struct usb_ep_ops { int (*fifo_status) (struct usb_ep *ep); void (*fifo_flush) (struct usb_ep *ep); + + ANDROID_KABI_RESERVE(1); }; /** @@ -241,6 +245,8 @@ struct usb_ep { u8 address; const struct usb_endpoint_descriptor *desc; const struct usb_ss_ep_comp_descriptor *comp_desc; + + ANDROID_KABI_RESERVE(1); }; /*-------------------------------------------------------------------------*/ @@ -326,9 +332,16 @@ struct usb_gadget_ops { void (*udc_set_speed)(struct usb_gadget *, enum usb_device_speed); void (*udc_set_ssp_rate)(struct usb_gadget *gadget, enum usb_ssp_rate rate); + void (*udc_async_callbacks)(struct usb_gadget *gadget, bool enable); struct usb_ep *(*match_ep)(struct usb_gadget *, struct usb_endpoint_descriptor *, struct usb_ss_ep_comp_descriptor *); + int (*check_config)(struct usb_gadget *gadget); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /** @@ -613,6 +626,7 @@ int usb_gadget_connect(struct usb_gadget *gadget); int usb_gadget_disconnect(struct usb_gadget *gadget); int usb_gadget_deactivate(struct usb_gadget *gadget); int usb_gadget_activate(struct usb_gadget *gadget); +int usb_gadget_check_config(struct usb_gadget *gadget); #else static inline int usb_gadget_frame_number(struct usb_gadget *gadget) { return 0; } @@ -636,6 +650,8 @@ static inline int usb_gadget_deactivate(struct usb_gadget *gadget) { return 0; } static inline int usb_gadget_activate(struct usb_gadget *gadget) { return 0; } +static inline int usb_gadget_check_config(struct usb_gadget *gadget) +{ return 0; } #endif /* CONFIG_USB_GADGET */ /*-------------------------------------------------------------------------*/ diff --git a/include/linux/usb/otg.h b/include/linux/usb/otg.h index 69f1b6328532..10cb3d193d77 100644 --- a/include/linux/usb/otg.h +++ b/include/linux/usb/otg.h @@ -12,6 +12,7 @@ #include #include +#include struct usb_otg { u8 default_a; @@ -40,6 +41,7 @@ struct usb_otg { /* start or continue HNP role switch */ int (*start_hnp)(struct usb_otg *otg); + ANDROID_KABI_RESERVE(1); }; /** diff --git a/include/linux/usb/phy.h b/include/linux/usb/phy.h index e4de6bc1f69b..888d2fdb662f 100644 --- a/include/linux/usb/phy.h +++ b/include/linux/usb/phy.h @@ -13,6 +13,7 @@ #include #include #include +#include #include enum usb_phy_interface { @@ -155,6 +156,8 @@ struct usb_phy { * manually detect the charger type. */ enum usb_charger_type (*charger_detect)(struct usb_phy *x); + + ANDROID_KABI_RESERVE(1); }; /* for board-specific init logic */ diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index 91b4303ca305..b2e2c302caa6 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -4,6 +4,7 @@ #define __LINUX_USB_TYPEC_H #include +#include /* USB Type-C Specification releases */ #define USB_TYPEC_REV_1_0 0x100 /* 1.0 */ @@ -215,6 +216,7 @@ struct typec_operations { int (*vconn_set)(struct typec_port *port, enum typec_role role); int (*port_type_set)(struct typec_port *port, enum typec_port_type type); + ANDROID_KABI_RESERVE(1); }; enum usb_pd_svdm_ver { @@ -252,6 +254,7 @@ struct typec_capability { void *driver_data; const struct typec_operations *ops; + ANDROID_KABI_RESERVE(1); }; /* Specific to try_role(). Indicates the user want's to clear the preference. */ diff --git a/include/linux/usb/typec_altmode.h b/include/linux/usb/typec_altmode.h index 65933cbe9129..60531284df7c 100644 --- a/include/linux/usb/typec_altmode.h +++ b/include/linux/usb/typec_altmode.h @@ -30,6 +30,7 @@ struct typec_altmode { char *desc; const struct typec_altmode_ops *ops; + ANDROID_KABI_RESERVE(1); }; #define to_typec_altmode(d) container_of(d, struct typec_altmode, dev) @@ -63,6 +64,7 @@ struct typec_altmode_ops { int (*notify)(struct typec_altmode *altmode, unsigned long conf, void *data); int (*activate)(struct typec_altmode *altmode, int activate); + ANDROID_KABI_RESERVE(1); }; int typec_altmode_enter(struct typec_altmode *altmode, u32 *vdo); diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index 26de0cae2a0a..e98cc6dea2b9 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -14,6 +14,7 @@ #include #include #include +#include struct workqueue_struct; @@ -106,6 +107,8 @@ struct work_struct { #ifdef CONFIG_LOCKDEP struct lockdep_map lockdep_map; #endif + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; #define WORK_DATA_INIT() ATOMIC_LONG_INIT((unsigned long)WORK_STRUCT_NO_POOL) @@ -119,6 +122,9 @@ struct delayed_work { /* target workqueue and CPU ->timer uses to queue ->work */ struct workqueue_struct *wq; int cpu; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct rcu_work { diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h index ead627c00655..113d39905b8c 100644 --- a/include/media/v4l2-ctrls.h +++ b/include/media/v4l2-ctrls.h @@ -11,6 +11,7 @@ #include #include #include +#include #include /* @@ -87,6 +88,9 @@ union v4l2_ctrl_ptr { struct v4l2_area *p_area; void *p; const void *p_const; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /** @@ -119,6 +123,8 @@ struct v4l2_ctrl_ops { int (*g_volatile_ctrl)(struct v4l2_ctrl *ctrl); int (*try_ctrl)(struct v4l2_ctrl *ctrl); int (*s_ctrl)(struct v4l2_ctrl *ctrl); + + ANDROID_KABI_RESERVE(1); }; /** @@ -140,6 +146,8 @@ struct v4l2_ctrl_type_ops { void (*log)(const struct v4l2_ctrl *ctrl); int (*validate)(const struct v4l2_ctrl *ctrl, u32 idx, union v4l2_ctrl_ptr ptr); + + ANDROID_KABI_RESERVE(1); }; /** @@ -290,6 +298,8 @@ struct v4l2_ctrl { union v4l2_ctrl_ptr p_def; union v4l2_ctrl_ptr p_new; union v4l2_ctrl_ptr p_cur; + + ANDROID_KABI_RESERVE(1); }; /** @@ -329,6 +339,8 @@ struct v4l2_ctrl_ref { bool req_done; bool valid_p_req; union v4l2_ctrl_ptr p_req; + + ANDROID_KABI_RESERVE(1); }; /** @@ -379,6 +391,8 @@ struct v4l2_ctrl_handler { struct list_head requests; struct list_head requests_queued; struct media_request_object req_obj; + + ANDROID_KABI_RESERVE(1); }; /** @@ -430,6 +444,8 @@ struct v4l2_ctrl_config { const char * const *qmenu; const s64 *qmenu_int; unsigned int is_private:1; + + ANDROID_KABI_RESERVE(1); }; /** diff --git a/include/media/v4l2-dev.h b/include/media/v4l2-dev.h index ad2d41952442..3a66e78e8e3b 100644 --- a/include/media/v4l2-dev.h +++ b/include/media/v4l2-dev.h @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -211,6 +212,8 @@ struct v4l2_file_operations { int (*mmap) (struct file *, struct vm_area_struct *); int (*open) (struct file *); int (*release) (struct file *); + + ANDROID_KABI_RESERVE(1); }; /* @@ -307,6 +310,9 @@ struct video_device DECLARE_BITMAP(valid_ioctls, BASE_VIDIOC_PRIVATE); struct mutex *lock; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /** diff --git a/include/media/videobuf2-v4l2.h b/include/media/videobuf2-v4l2.h index c203047eb834..1c59f0951b3b 100644 --- a/include/media/videobuf2-v4l2.h +++ b/include/media/videobuf2-v4l2.h @@ -13,6 +13,7 @@ #define _MEDIA_VIDEOBUF2_V4L2_H #include +#include #include #if VB2_MAX_FRAME != VIDEO_MAX_FRAME @@ -51,6 +52,8 @@ struct vb2_v4l2_buffer { __s32 request_fd; bool is_held; struct vb2_plane planes[VB2_MAX_PLANES]; + + ANDROID_KABI_RESERVE(1); }; /* VB2 V4L2 flags as set in vb2_queue.subsystem_flags */ diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index df611c8b6b59..36a6f84c5c49 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -564,6 +565,11 @@ struct hci_dev { int (*set_bdaddr)(struct hci_dev *hdev, const bdaddr_t *bdaddr); void (*cmd_timeout)(struct hci_dev *hdev); bool (*prevent_wake)(struct hci_dev *hdev); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #define HCI_PHY_HANDLE(handle) (handle & 0xff) @@ -660,6 +666,11 @@ struct hci_conn { void (*connect_cfm_cb) (struct hci_conn *conn, u8 status); void (*security_cfm_cb) (struct hci_conn *conn, u8 status); void (*disconn_cfm_cb) (struct hci_conn *conn, u8 reason); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; struct hci_chan { @@ -670,6 +681,8 @@ struct hci_chan { unsigned int sent; __u8 state; bool amp; + + ANDROID_KABI_RESERVE(1); }; struct hci_conn_params { @@ -696,6 +709,8 @@ struct hci_conn_params { struct hci_conn *conn; bool explicit_connect; u32 current_flags; + + ANDROID_KABI_RESERVE(1); }; extern struct list_head hci_dev_list; @@ -1415,6 +1430,8 @@ struct hci_cb { __u8 encrypt); void (*key_change_cfm) (struct hci_conn *conn, __u8 status); void (*role_switch_cfm) (struct hci_conn *conn, __u8 status, __u8 role); + + ANDROID_KABI_RESERVE(1); }; static inline void hci_connect_cfm(struct hci_conn *conn, __u8 status) @@ -1675,6 +1692,8 @@ struct hci_mgmt_chan { size_t handler_count; const struct hci_mgmt_handler *handlers; void (*hdev_init) (struct sock *sk, struct hci_dev *hdev); + + ANDROID_KABI_RESERVE(1); }; int hci_mgmt_chan_register(struct hci_mgmt_chan *c); diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 1d1232917de7..8c99677077b6 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -29,6 +29,7 @@ #include #include +#include /* L2CAP defaults */ #define L2CAP_DEFAULT_MTU 672 @@ -643,6 +644,9 @@ struct l2cap_chan { void *data; const struct l2cap_ops *ops; struct mutex lock; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct l2cap_ops { @@ -667,6 +671,9 @@ struct l2cap_ops { unsigned long len, int nb); int (*filter) (struct l2cap_chan * chan, struct sk_buff *skb); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct l2cap_conn { @@ -702,6 +709,9 @@ struct l2cap_conn { struct mutex chan_lock; struct kref ref; struct list_head users; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; struct l2cap_user { diff --git a/include/net/bluetooth/rfcomm.h b/include/net/bluetooth/rfcomm.h index 99d26879b02a..d8f4e835ede8 100644 --- a/include/net/bluetooth/rfcomm.h +++ b/include/net/bluetooth/rfcomm.h @@ -22,6 +22,7 @@ */ #include +#include #ifndef __RFCOMM_H #define __RFCOMM_H @@ -164,6 +165,8 @@ struct rfcomm_session { uint mtu; struct list_head dlcs; + + ANDROID_KABI_RESERVE(1); }; struct rfcomm_dlc { @@ -197,6 +200,9 @@ struct rfcomm_dlc { void (*data_ready)(struct rfcomm_dlc *d, struct sk_buff *skb); void (*state_change)(struct rfcomm_dlc *d, int err); void (*modem_status)(struct rfcomm_dlc *d, u8 v24_sig); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; /* DLC and session flags */ diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index fab23df83050..81d5b2486c04 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -20,6 +20,7 @@ #include #include #include +#include #include /** @@ -1042,6 +1043,8 @@ struct cfg80211_crypto_settings { const u8 *sae_pwd; u8 sae_pwd_len; enum nl80211_sae_pwe_mechanism sae_pwe; + + ANDROID_KABI_RESERVE(1); }; /** @@ -1088,6 +1091,8 @@ struct cfg80211_beacon_data { size_t probe_resp_len; size_t lci_len; size_t civicloc_len; + + ANDROID_KABI_RESERVE(1); }; struct mac_address { @@ -1224,6 +1229,8 @@ struct cfg80211_ap_settings { struct cfg80211_he_bss_color he_bss_color; struct cfg80211_fils_discovery fils_discovery; struct cfg80211_unsol_bcast_probe_resp unsol_bcast_probe_resp; + + ANDROID_KABI_RESERVE(1); }; /** @@ -1253,6 +1260,8 @@ struct cfg80211_csa_settings { bool radar_required; bool block_tx; u8 count; + + ANDROID_KABI_RESERVE(1); }; #define CFG80211_MAX_NUM_DIFFERENT_CHANNELS 10 @@ -1395,6 +1404,8 @@ struct station_parameters { u16 airtime_weight; struct sta_txpwr txpwr; const struct ieee80211_he_6ghz_capa *he_6ghz_capa; + + ANDROID_KABI_RESERVE(1); }; /** @@ -1741,6 +1752,8 @@ struct station_info { u32 airtime_link_metric; u8 connected_to_as; + + ANDROID_KABI_RESERVE(1); }; #if IS_ENABLED(CONFIG_CFG80211) @@ -1995,6 +2008,8 @@ struct mesh_config { u16 dot11MeshAwakeWindowDuration; u32 plink_timeout; bool dot11MeshNolearn; + + ANDROID_KABI_RESERVE(1); }; /** @@ -2044,6 +2059,8 @@ struct mesh_setup { struct cfg80211_bitrate_mask beacon_rate; bool userspace_handles_dfs; bool control_port_over_nl80211; + + ANDROID_KABI_RESERVE(1); }; /** @@ -2205,6 +2222,8 @@ struct cfg80211_scan_request { u32 n_6ghz_params; struct cfg80211_scan_6ghz_params *scan_6ghz_params; + ANDROID_KABI_RESERVE(1); + /* keep last */ struct ieee80211_channel *channels[]; }; @@ -2351,6 +2370,8 @@ struct cfg80211_sched_scan_request { bool nl_owner_dead; struct list_head list; + ANDROID_KABI_RESERVE(1); + /* keep last */ struct ieee80211_channel *channels[]; }; @@ -2474,6 +2495,8 @@ struct cfg80211_bss { u8 bssid_index; u8 max_bssid_indicator; + ANDROID_KABI_RESERVE(1); + u8 priv[] __aligned(sizeof(void *)); }; @@ -2600,6 +2623,8 @@ struct cfg80211_assoc_request { size_t fils_kek_len; const u8 *fils_nonces; struct ieee80211_s1g_cap s1g_capa, s1g_capa_mask; + + ANDROID_KABI_RESERVE(1); }; /** @@ -2698,6 +2723,8 @@ struct cfg80211_ibss_params { struct ieee80211_ht_cap ht_capa_mask; struct key_params *wep_keys; int wep_tx_key; + + ANDROID_KABI_RESERVE(1); }; /** @@ -2812,6 +2839,8 @@ struct cfg80211_connect_params { size_t fils_erp_rrk_len; bool want_1x; struct ieee80211_edmg edmg; + + ANDROID_KABI_RESERVE(1); }; /** @@ -3269,6 +3298,8 @@ struct cfg80211_nan_func { u8 num_rx_filters; u8 instance_id; u64 cookie; + + ANDROID_KABI_RESERVE(1); }; /** @@ -3429,6 +3460,8 @@ struct cfg80211_pmsr_ftm_result { dist_avg_valid:1, dist_variance_valid:1, dist_spread_valid:1; + + ANDROID_KABI_RESERVE(1); }; /** @@ -4265,6 +4298,10 @@ struct cfg80211_ops { struct cfg80211_tid_config *tid_conf); int (*reset_tid_config)(struct wiphy *wiphy, struct net_device *dev, const u8 *peer, u8 tids); + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /* @@ -4628,6 +4665,8 @@ struct wiphy_vendor_command { unsigned long *storage); const struct nla_policy *policy; unsigned int maxattr; + + ANDROID_KABI_RESERVE(1); }; /** @@ -5033,6 +5072,8 @@ struct wiphy { u8 max_data_retry_count; + ANDROID_KABI_RESERVE(1); + char priv[] __aligned(NETDEV_ALIGN); }; @@ -5332,6 +5373,9 @@ struct wireless_dev { struct work_struct pmsr_free_wk; unsigned long unprot_beacon_reported; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; static inline u8 *wdev_address(struct wireless_dev *wdev) diff --git a/include/net/genetlink.h b/include/net/genetlink.h index e55ec1597ce7..d66f0f397f2e 100644 --- a/include/net/genetlink.h +++ b/include/net/genetlink.h @@ -3,6 +3,7 @@ #define __NET_GENERIC_NETLINK_H #include +#include #include #include @@ -67,6 +68,8 @@ struct genl_family { const struct genl_small_ops *small_ops; const struct genl_multicast_group *mcgrps; struct module *module; + + ANDROID_KABI_RESERVE(1); }; /** @@ -158,6 +161,8 @@ struct genl_ops { u8 internal_flags; u8 flags; u8 validate; + + ANDROID_KABI_RESERVE(1); }; /** diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index aa92af3dd444..8177cb94958c 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -51,6 +52,8 @@ struct inet_connection_sock_af_ops { char __user *optval, int __user *optlen); void (*addr2sockaddr)(struct sock *sk, struct sockaddr *); void (*mtu_reduced)(struct sock *sk); + + ANDROID_KABI_RESERVE(1); }; /** inet_connection_sock - INET connection oriented sock @@ -134,6 +137,8 @@ struct inet_connection_sock { u32 icsk_probes_tstamp; u32 icsk_user_timeout; + ANDROID_KABI_RESERVE(1); + u64 icsk_ca_priv[104 / sizeof(u64)]; #define ICSK_CA_PRIV_SIZE (13 * sizeof(u64)) }; diff --git a/include/net/ip6_fib.h b/include/net/ip6_fib.h index ac5ff3c3afb1..bd4bfdd30e28 100644 --- a/include/net/ip6_fib.h +++ b/include/net/ip6_fib.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -67,6 +68,8 @@ struct fib6_config { struct nlattr *fc_encap; u16 fc_encap_type; bool fc_is_fdb; + + ANDROID_KABI_RESERVE(1); }; struct fib6_node { @@ -83,6 +86,8 @@ struct fib6_node { int fn_sernum; struct fib6_info __rcu *rr_ptr; struct rcu_head rcu; + + ANDROID_KABI_RESERVE(1); }; struct fib6_gc_args { @@ -199,6 +204,9 @@ struct fib6_info { struct rcu_head rcu; struct nexthop *nh; + + ANDROID_KABI_RESERVE(1); + struct fib6_nh fib6_nh[]; }; @@ -218,6 +226,8 @@ struct rt6_info { /* more non-fragment space at head required */ unsigned short rt6i_nfheader_len; + + ANDROID_KABI_RESERVE(1); }; struct fib6_result { diff --git a/include/net/mac80211.h b/include/net/mac80211.h index 6ff49c13717b..02cd47eac042 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -701,6 +702,8 @@ struct ieee80211_bss_conf { u32 unsol_bcast_probe_resp_interval; bool s1g; struct cfg80211_bitrate_mask beacon_tx_rate; + + ANDROID_KABI_RESERVE(1); }; /** @@ -1097,6 +1100,9 @@ struct ieee80211_tx_info { void *rate_driver_data[ IEEE80211_TX_INFO_RATE_DRIVER_DATA_SIZE / sizeof(void *)]; }; + + ANDROID_KABI_RESERVE(1); + void *driver_data[ IEEE80211_TX_INFO_DRIVER_DATA_SIZE / sizeof(void *)]; }; @@ -1583,6 +1589,8 @@ struct ieee80211_conf { struct cfg80211_chan_def chandef; bool radar_enabled; enum ieee80211_smps_mode smps_mode; + + ANDROID_KABI_RESERVE(1); }; /** @@ -1727,6 +1735,8 @@ struct ieee80211_vif { bool txqs_stopped[IEEE80211_NUM_ACS]; + ANDROID_KABI_RESERVE(1); + /* must be last */ u8 drv_priv[] __aligned(sizeof(void *)); }; @@ -2112,6 +2122,8 @@ struct ieee80211_sta { struct ieee80211_txq *txq[IEEE80211_NUM_TIDS + 1]; + ANDROID_KABI_RESERVE(1); + /* must be last */ u8 drv_priv[] __aligned(sizeof(void *)); }; @@ -2606,6 +2618,8 @@ struct ieee80211_hw { u8 tx_sk_pacing_shift; u8 weight_multiplier; u32 max_mtu; + + ANDROID_KABI_RESERVE(1); }; static inline bool _ieee80211_hw_check(struct ieee80211_hw *hw, @@ -4192,6 +4206,11 @@ struct ieee80211_ops { struct ieee80211_vif *vif); void (*sta_set_4addr)(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta, bool enabled); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /** @@ -6161,6 +6180,11 @@ struct rate_control_ops { struct dentry *dir); u32 (*get_expected_throughput)(void *priv_sta); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; static inline int rate_supported(struct ieee80211_sta *sta, diff --git a/include/net/neighbour.h b/include/net/neighbour.h index 22ced1381ede..8a26efdadbbc 100644 --- a/include/net/neighbour.h +++ b/include/net/neighbour.h @@ -28,6 +28,7 @@ #include #include #include +#include #include /* @@ -83,6 +84,8 @@ struct neigh_parms { int reachable_time; int data[NEIGH_VAR_DATA_MAX]; DECLARE_BITMAP(data_state, NEIGH_VAR_DATA_MAX); + + ANDROID_KABI_RESERVE(1); }; static inline void neigh_var_set(struct neigh_parms *p, int index, int val) @@ -157,6 +160,9 @@ struct neighbour { struct list_head gc_list; struct rcu_head rcu; struct net_device *dev; + + ANDROID_KABI_RESERVE(1); + u8 primary_key[0]; } __randomize_layout; @@ -226,6 +232,8 @@ struct neigh_table { struct neigh_statistics __percpu *stats; struct neigh_hash_table __rcu *nht; struct pneigh_entry **phash_buckets; + + ANDROID_KABI_RESERVE(1); }; enum { diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h index 7a44164f147c..878d0ce81321 100644 --- a/include/net/netfilter/nf_conntrack.h +++ b/include/net/netfilter/nf_conntrack.h @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -109,6 +110,8 @@ struct nf_conn { ANDROID_KABI_RESERVE(1); ANDROID_KABI_RESERVE(2); + + ANDROID_OEM_DATA(1); }; static inline struct nf_conn * diff --git a/include/net/netns/can.h b/include/net/netns/can.h index 52fbd8291a96..d075f610e647 100644 --- a/include/net/netns/can.h +++ b/include/net/netns/can.h @@ -7,6 +7,7 @@ #define __NETNS_CAN_H__ #include +#include struct can_dev_rcv_lists; struct can_pkg_stats; @@ -35,6 +36,8 @@ struct netns_can { /* CAN GW per-net gateway jobs */ struct hlist_head cgw_list; + + ANDROID_KABI_RESERVE(1); }; #endif /* __NETNS_CAN_H__ */ diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index 175206956080..13b3defd5cb0 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -10,6 +10,7 @@ #include #include #include +#include struct tcpm_hash_bucket; struct ctl_table_header; @@ -228,5 +229,7 @@ struct netns_ipv4 { atomic_t rt_genid; siphash_key_t ip_id_key; + + ANDROID_KABI_RESERVE(1); }; #endif diff --git a/include/net/netns/ipv6.h b/include/net/netns/ipv6.h index 5ec054473d81..213e96241e76 100644 --- a/include/net/netns/ipv6.h +++ b/include/net/netns/ipv6.h @@ -7,6 +7,7 @@ #ifndef __NETNS_IPV6_H__ #define __NETNS_IPV6_H__ +#include #include #include @@ -115,6 +116,8 @@ struct netns_ipv6 { spinlock_t lock; u32 seq; } ip6addrlbl_table; + + ANDROID_KABI_RESERVE(1); }; #if IS_ENABLED(CONFIG_NF_DEFRAG_IPV6) diff --git a/include/net/netns/netfilter.h b/include/net/netns/netfilter.h index ca043342c0eb..fa2663747a85 100644 --- a/include/net/netns/netfilter.h +++ b/include/net/netns/netfilter.h @@ -3,6 +3,7 @@ #define __NETNS_NETFILTER_H #include +#include struct proc_dir_entry; struct nf_logger; @@ -34,5 +35,7 @@ struct netns_nf { #if IS_ENABLED(CONFIG_NF_DEFRAG_IPV6) bool defrag_ipv6; #endif + + ANDROID_KABI_RESERVE(1); }; #endif diff --git a/include/net/netns/nftables.h b/include/net/netns/nftables.h index 6c0806bd8d1e..ccb84b7dadce 100644 --- a/include/net/netns/nftables.h +++ b/include/net/netns/nftables.h @@ -3,6 +3,7 @@ #define _NETNS_NFTABLES_H_ #include +#include struct netns_nftables { struct list_head tables; @@ -13,6 +14,8 @@ struct netns_nftables { unsigned int base_seq; u8 gencursor; u8 validate_state; + + ANDROID_KABI_RESERVE(1); }; #endif diff --git a/include/net/netns/xfrm.h b/include/net/netns/xfrm.h index b59d73d529ba..93d74c68d87e 100644 --- a/include/net/netns/xfrm.h +++ b/include/net/netns/xfrm.h @@ -7,6 +7,7 @@ #include #include #include +#include #include struct ctl_table_header; @@ -77,6 +78,8 @@ struct netns_xfrm { spinlock_t xfrm_policy_lock; struct mutex xfrm_cfg_mutex; + + ANDROID_KABI_RESERVE(1); }; #endif diff --git a/include/net/page_pool.h b/include/net/page_pool.h index b139e7bf45fe..6c469951e589 100644 --- a/include/net/page_pool.h +++ b/include/net/page_pool.h @@ -33,6 +33,7 @@ #include /* Needed by ptr_ring */ #include #include +#include #define PP_FLAG_DMA_MAP BIT(0) /* Should page_pool do the DMA * map/unmap @@ -126,6 +127,8 @@ struct page_pool { refcount_t user_cnt; u64 destroy_cnt; + + ANDROID_KABI_RESERVE(1); }; struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp); diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index f8631ad3c868..2faae277f042 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -114,6 +115,8 @@ struct Qdisc { bool empty; struct rcu_head rcu; + ANDROID_KABI_RESERVE(1); + /* private data */ long privdata[] ____cacheline_aligned; }; @@ -271,6 +274,8 @@ struct Qdisc_class_ops { struct sk_buff *skb, struct tcmsg*); int (*dump_stats)(struct Qdisc *, unsigned long, struct gnet_dump *); + + ANDROID_KABI_RESERVE(1); }; /* Qdisc_class_ops flag values */ @@ -314,6 +319,8 @@ struct Qdisc_ops { u32 (*egress_block_get)(struct Qdisc *sch); struct module *owner; + + ANDROID_KABI_RESERVE(1); }; diff --git a/include/net/sock.h b/include/net/sock.h index 912c295cf7bb..076eaef817ff 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -69,6 +69,7 @@ #include #include #include +#include /* * This structure really needs to be cleaned up. @@ -530,6 +531,8 @@ struct sock { ANDROID_KABI_RESERVE(6); ANDROID_KABI_RESERVE(7); ANDROID_KABI_RESERVE(8); + + ANDROID_OEM_DATA(1); }; enum sk_pacing { diff --git a/include/net/tls.h b/include/net/tls.h index 624864c2dcfb..b0d953be977e 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -116,6 +116,9 @@ struct tls_rec { char aad_space[TLS_AAD_SPACE_SIZE]; u8 iv_data[MAX_IV_SIZE]; struct aead_request aead_req; + + ANDROID_KABI_RESERVE(1); + u8 aead_req_ctx[]; }; @@ -144,6 +147,8 @@ struct tls_sw_context_tx { #define BIT_TX_SCHEDULED 0 #define BIT_TX_CLOSING 1 unsigned long tx_bitmask; + + ANDROID_KABI_RESERVE(1); }; struct tls_sw_context_rx { @@ -161,6 +166,8 @@ struct tls_sw_context_rx { /* protect crypto_wait with decrypt_pending*/ spinlock_t decrypt_compl_lock; bool async_notify; + + ANDROID_KABI_RESERVE(1); }; struct tls_record_info { diff --git a/include/scsi/scsi_cmnd.h b/include/scsi/scsi_cmnd.h index 69ade4fb71aa..b6398c9d2c5c 100644 --- a/include/scsi/scsi_cmnd.h +++ b/include/scsi/scsi_cmnd.h @@ -11,6 +11,7 @@ #include #include #include +#include struct Scsi_Host; struct scsi_driver; @@ -142,6 +143,11 @@ struct scsi_cmnd { unsigned char tag; /* SCSI-II queued command tag */ unsigned int extra_len; /* length of alignment and padding */ + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /* diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h index 1a5c9a3df6d6..158423bcdde9 100644 --- a/include/scsi/scsi_device.h +++ b/include/scsi/scsi_device.h @@ -8,6 +8,7 @@ #include #include #include +#include struct device; struct request_queue; @@ -237,6 +238,12 @@ struct scsi_device { struct mutex state_mutex; enum scsi_device_state sdev_state; struct task_struct *quiesced_by; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); + unsigned long sdev_data[]; } __attribute__((aligned(sizeof(unsigned long)))); diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index 701f178b20ae..8c0d37f389df 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -10,6 +10,7 @@ #include #include #include +#include struct block_device; struct completion; @@ -486,6 +487,11 @@ struct scsi_host_template { /* Delay for runtime autosuspend */ int rpm_autosuspend_delay; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; /* @@ -692,6 +698,8 @@ struct Scsi_Host { */ struct device *dma_dev; + ANDROID_KABI_RESERVE(1); + /* * We should ensure that this is aligned, both for better performance * and also because some compilers (m68k) don't automatically force diff --git a/include/sound/compress_driver.h b/include/sound/compress_driver.h index 70cbc5095e72..82417ce70d62 100644 --- a/include/sound/compress_driver.h +++ b/include/sound/compress_driver.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -54,6 +55,8 @@ struct snd_compr_runtime { dma_addr_t dma_addr; size_t dma_bytes; struct snd_dma_buffer *dma_buffer_p; + + ANDROID_KABI_RESERVE(1); }; /** @@ -82,6 +85,8 @@ struct snd_compr_stream { bool partial_drain; void *private_data; struct snd_dma_buffer dma_buffer; + + ANDROID_KABI_RESERVE(1); }; /** @@ -130,6 +135,8 @@ struct snd_compr_ops { struct snd_compr_caps *caps); int (*get_codec_caps) (struct snd_compr_stream *stream, struct snd_compr_codec_caps *codec); + + ANDROID_KABI_RESERVE(1); }; /** @@ -158,6 +165,7 @@ struct snd_compr { struct snd_info_entry *proc_root; struct snd_info_entry *proc_info_entry; #endif + ANDROID_KABI_RESERVE(1); }; /* compress device register APIs */ diff --git a/include/sound/core.h b/include/sound/core.h index 0462c577d7a3..b5ecbf936ffb 100644 --- a/include/sound/core.h +++ b/include/sound/core.h @@ -14,6 +14,7 @@ #include /* pm_message_t */ #include #include +#include /* number of supported soundcards */ #ifdef CONFIG_SND_DYNAMIC_MINORS @@ -61,6 +62,8 @@ struct snd_device_ops { int (*dev_free)(struct snd_device *dev); int (*dev_register)(struct snd_device *dev); int (*dev_disconnect)(struct snd_device *dev); + + ANDROID_KABI_RESERVE(1); }; struct snd_device { @@ -70,6 +73,8 @@ struct snd_device { enum snd_device_type type; /* device type */ void *device_data; /* device structure */ const struct snd_device_ops *ops; /* operations */ + + ANDROID_KABI_RESERVE(1); }; #define snd_device(n) list_entry(n, struct snd_device, list) @@ -132,6 +137,9 @@ struct snd_card { struct snd_mixer_oss *mixer_oss; int mixer_oss_change_count; #endif + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; #define dev_to_snd_card(p) container_of(p, struct snd_card, card_dev) @@ -167,6 +175,8 @@ struct snd_minor { void *private_data; /* private data for f_ops->open */ struct device *dev; /* device for sysfs */ struct snd_card *card_ptr; /* assigned card instance */ + + ANDROID_KABI_RESERVE(1); }; /* return a device pointer linked to each sound device as a parent */ diff --git a/include/sound/hwdep.h b/include/sound/hwdep.h index 8d6cdb254039..d99a53025ff0 100644 --- a/include/sound/hwdep.h +++ b/include/sound/hwdep.h @@ -9,6 +9,7 @@ #include #include +#include struct snd_hwdep; @@ -34,6 +35,8 @@ struct snd_hwdep_ops { struct snd_hwdep_dsp_status *status); int (*dsp_load)(struct snd_hwdep *hw, struct snd_hwdep_dsp_image *image); + + ANDROID_KABI_RESERVE(1); }; struct snd_hwdep { @@ -59,6 +62,8 @@ struct snd_hwdep { int used; /* reference counter */ unsigned int dsp_loaded; /* bit fields of loaded dsp indices */ unsigned int exclusive:1; /* exclusive access mode */ + + ANDROID_KABI_RESERVE(1); }; extern int snd_hwdep_new(struct snd_card *card, char *id, int device, diff --git a/include/sound/info.h b/include/sound/info.h index 7c13bf52cc81..25af5d8d1527 100644 --- a/include/sound/info.h +++ b/include/sound/info.h @@ -9,6 +9,7 @@ #include #include +#include #include /* buffer for information */ @@ -55,6 +56,8 @@ struct snd_info_entry_ops { int (*mmap)(struct snd_info_entry *entry, void *file_private_data, struct inode *inode, struct file *file, struct vm_area_struct *vma); + + ANDROID_KABI_RESERVE(1); }; struct snd_info_entry { @@ -74,6 +77,8 @@ struct snd_info_entry { struct mutex access; struct list_head children; struct list_head list; + + ANDROID_KABI_RESERVE(1); }; #if defined(CONFIG_SND_OSSEMUL) && defined(CONFIG_SND_PROC_FS) diff --git a/include/sound/jack.h b/include/sound/jack.h index 9eb2b5ec1ec4..70400b20e952 100644 --- a/include/sound/jack.h +++ b/include/sound/jack.h @@ -9,6 +9,7 @@ */ #include +#include struct input_dev; @@ -69,6 +70,8 @@ struct snd_jack { #endif /* CONFIG_SND_JACK_INPUT_DEV */ void *private_data; void (*private_free)(struct snd_jack *); + + ANDROID_KABI_RESERVE(1); }; #ifdef CONFIG_SND_JACK diff --git a/include/sound/soc.h b/include/sound/soc.h index f7fabb9d1c83..63338f68de48 100644 --- a/include/sound/soc.h +++ b/include/sound/soc.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -846,6 +847,8 @@ struct snd_soc_dai_link { #ifdef CONFIG_SND_SOC_TOPOLOGY struct snd_soc_dobj dobj; /* For topology */ #endif + + ANDROID_KABI_RESERVE(1); }; #define for_each_link_codecs(link, i, codec) \ for ((i) = 0; \ @@ -1096,6 +1099,11 @@ struct snd_soc_card { unsigned int probed:1; void *drvdata; + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #define for_each_card_prelinks(card, i, link) \ for ((i) = 0; \ @@ -1177,6 +1185,9 @@ struct snd_soc_pcm_runtime { unsigned int fe_compr:1; /* for Dynamic PCM */ int num_components; + + ANDROID_KABI_RESERVE(1); + struct snd_soc_component *components[]; /* CPU/Codec/Platform */ }; /* see soc_new_pcm_runtime() */ @@ -1222,6 +1233,8 @@ struct soc_mixer_control { #ifdef CONFIG_SND_SOC_TOPOLOGY struct snd_soc_dobj dobj; #endif + + ANDROID_KABI_RESERVE(1); }; struct soc_bytes { @@ -1266,6 +1279,8 @@ struct soc_enum { #ifdef CONFIG_SND_SOC_TOPOLOGY struct snd_soc_dobj dobj; #endif + + ANDROID_KABI_RESERVE(1); }; static inline bool snd_soc_volsw_is_stereo(struct soc_mixer_control *mc) diff --git a/include/sound/timer.h b/include/sound/timer.h index 760e132cc0cd..b7695e4dcca0 100644 --- a/include/sound/timer.h +++ b/include/sound/timer.h @@ -10,6 +10,7 @@ #include #include +#include #define snd_timer_chip(timer) ((timer)->private_data) @@ -52,6 +53,8 @@ struct snd_timer_hardware { int (*stop) (struct snd_timer * timer); int (*set_period) (struct snd_timer * timer, unsigned long period_num, unsigned long period_den); int (*precise_resolution) (struct snd_timer * timer, unsigned long *num, unsigned long *den); + + ANDROID_KABI_RESERVE(1); }; struct snd_timer { @@ -77,6 +80,8 @@ struct snd_timer { struct work_struct task_work; int max_instances; /* upper limit of timer instances */ int num_instances; /* current number of timer instances */ + + ANDROID_KABI_RESERVE(1); }; struct snd_timer_instance { @@ -106,6 +111,8 @@ struct snd_timer_instance { struct list_head slave_list_head; struct list_head slave_active_head; struct snd_timer_instance *master; + + ANDROID_KABI_RESERVE(1); }; /* diff --git a/include/trace/events/cma.h b/include/trace/events/cma.h index 5cf385ae7c08..9c134325d6b2 100644 --- a/include/trace/events/cma.h +++ b/include/trace/events/cma.h @@ -91,6 +91,51 @@ TRACE_EVENT(cma_alloc_start, __entry->align) ); +TRACE_EVENT(cma_alloc_info, + + TP_PROTO(const char *name, const struct page *page, unsigned int count, unsigned int align, struct cma_alloc_info *info), + + TP_ARGS(name, page, count, align, info), + + TP_STRUCT__entry( + __string(name, name) + __field(unsigned long, pfn) + __field(unsigned int, count) + __field(unsigned int, align) + __field(unsigned long, nr_migrated) + __field(unsigned long, nr_reclaimed) + __field(unsigned long, nr_mapped) + __field(unsigned int, err_iso) + __field(unsigned int, err_mig) + __field(unsigned int, err_test) + ), + + TP_fast_assign( + __assign_str(name, name); + __entry->pfn = page ? page_to_pfn(page) : -1; + __entry->count = count; + __entry->align = align; + __entry->nr_migrated = info->nr_migrated; + __entry->nr_reclaimed = info->nr_reclaimed; + __entry->nr_mapped = info->nr_mapped; + __entry->err_iso = info->nr_isolate_fail; + __entry->err_mig = info->nr_migrate_fail; + __entry->err_test = info->nr_test_fail; + ), + + TP_printk("name=%s pfn=0x%lx count=%u align=%u nr_migrated=%lu nr_reclaimed=%lu nr_mapped=%lu err_iso=%u err_mig=%u err_test=%u", + __get_str(name), + __entry->pfn, + __entry->count, + __entry->align, + __entry->nr_migrated, + __entry->nr_reclaimed, + __entry->nr_mapped, + __entry->err_iso, + __entry->err_mig, + __entry->err_test) +); + DEFINE_EVENT(cma_alloc_class, cma_alloc_finish, TP_PROTO(const char *name, unsigned long pfn, const struct page *page, diff --git a/include/trace/hooks/block.h b/include/trace/hooks/block.h new file mode 100644 index 000000000000..0d9d7db39968 --- /dev/null +++ b/include/trace/hooks/block.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM block + +#define TRACE_INCLUDE_PATH trace/hooks + +#if !defined(_TRACE_HOOK_BLOCK_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_BLOCK_H + +#include +#include + +struct blk_mq_tag_set; +struct blk_mq_tags; +struct blk_mq_alloc_data; + +DECLARE_HOOK(android_vh_blk_alloc_rqs, + TP_PROTO(size_t *rq_size, struct blk_mq_tag_set *set, + struct blk_mq_tags *tags), + TP_ARGS(rq_size, set, tags)); + +DECLARE_HOOK(android_vh_blk_rq_ctx_init, + TP_PROTO(struct request *rq, struct blk_mq_tags *tags, + struct blk_mq_alloc_data *data, u64 alloc_time_ns), + TP_ARGS(rq, tags, data, alloc_time_ns)); + +#endif /* _TRACE_HOOK_BLOCK_H */ + +/* This part must be outside protection */ +#include diff --git a/include/trace/hooks/ipv6.h b/include/trace/hooks/ipv6.h new file mode 100644 index 000000000000..c1998af0c5a0 --- /dev/null +++ b/include/trace/hooks/ipv6.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM ipv6 +#define TRACE_INCLUDE_PATH trace/hooks + +#if !defined(_TRACE_HOOK_IPV6_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_IPV6_H +#include +#include +/* + * Following tracepoints are not exported in tracefs and provide a + * mechanism for vendor modules to hook and extend functionality + */ +DECLARE_HOOK(android_vh_ipv6_gen_linklocal_addr, + TP_PROTO(struct net_device *dev, bool *ret), + TP_ARGS(dev, ret)); + +/* macro versions of hooks are no longer required */ + +#endif /* _TRACE_HOOK_IPV6_H */ +/* This part must be outside protection */ +#include diff --git a/include/trace/hooks/net.h b/include/trace/hooks/net.h index 31e0f21373e8..6715aa4eb668 100644 --- a/include/trace/hooks/net.h +++ b/include/trace/hooks/net.h @@ -18,6 +18,17 @@ DECLARE_HOOK(android_vh_ptype_head, DECLARE_HOOK(android_vh_kfree_skb, TP_PROTO(struct sk_buff *skb), TP_ARGS(skb)); +struct nf_conn; +struct sock; +DECLARE_RESTRICTED_HOOK(android_rvh_nf_conn_alloc, + TP_PROTO(struct nf_conn *nf_conn), TP_ARGS(nf_conn), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_nf_conn_free, + TP_PROTO(struct nf_conn *nf_conn), TP_ARGS(nf_conn), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_sk_alloc, + TP_PROTO(struct sock *sock), TP_ARGS(sock), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_sk_free, + TP_PROTO(struct sock *sock), TP_ARGS(sock), 1); + /* macro versions of hooks are no longer required */ #endif /* _TRACE_HOOK_NET_VH_H */ diff --git a/include/trace/hooks/snd_compr.h b/include/trace/hooks/snd_compr.h new file mode 100644 index 000000000000..c42ed8d6bc30 --- /dev/null +++ b/include/trace/hooks/snd_compr.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM snd_compr + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH trace/hooks + +#if !defined(_TRACE_HOOK_SND_COMPR_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_SND_COMPR_H + +#include +#include +/* + * Following tracepoints are not exported in tracefs and provide a + * mechanism for vendor modules to hook and extend functionality + */ + +DECLARE_HOOK(android_vh_snd_compr_use_pause_in_drain, + TP_PROTO(bool *use_pause_in_drain, bool *leave_draining), + TP_ARGS(use_pause_in_drain, leave_draining)); + +#endif /* _TRACE_HOOK_SND_COMPR_H */ +/* This part must be outside protection */ +#include + diff --git a/include/trace/hooks/sound.h b/include/trace/hooks/sound.h new file mode 100644 index 000000000000..2bf018edf392 --- /dev/null +++ b/include/trace/hooks/sound.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM sound +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH trace/hooks +#if !defined(_TRACE_HOOK_SOUND_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_SOUND_H +#include +#include +#include + +DECLARE_HOOK(android_vh_sound_usb_support_cpu_suspend, + TP_PROTO(struct usb_device *udev, + int direction, + bool *is_support), + TP_ARGS(udev, direction, is_support)); + +#endif /* _TRACE_HOOK_SOUND_H */ +/* This part must be outside protection */ +#include diff --git a/include/uapi/linux/elf.h b/include/uapi/linux/elf.h index 30f68b42eeb5..61bf4774b8f2 100644 --- a/include/uapi/linux/elf.h +++ b/include/uapi/linux/elf.h @@ -426,6 +426,7 @@ typedef struct elf64_shdr { #define NT_ARM_PACA_KEYS 0x407 /* ARM pointer authentication address keys */ #define NT_ARM_PACG_KEYS 0x408 /* ARM pointer authentication generic key */ #define NT_ARM_TAGGED_ADDR_CTRL 0x409 /* arm64 tagged address control (prctl()) */ +#define NT_ARM_PAC_ENABLED_KEYS 0x40a /* arm64 ptr auth enabled keys (prctl()) */ #define NT_ARC_V2 0x600 /* ARCv2 accumulator/extra registers */ #define NT_VMCOREDD 0x700 /* Vmcore Device Dump Note */ #define NT_MIPS_DSP 0x800 /* MIPS DSP ASE registers */ diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index c190e220ff67..58b37b12e745 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -234,14 +234,15 @@ struct prctl_mm_map { #define PR_GET_TAGGED_ADDR_CTRL 56 # define PR_TAGGED_ADDR_ENABLE (1UL << 0) /* MTE tag check fault modes */ -# define PR_MTE_TCF_SHIFT 1 -# define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT) -# define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT) -# define PR_MTE_TCF_ASYNC (2UL << PR_MTE_TCF_SHIFT) -# define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT) +# define PR_MTE_TCF_NONE 0 +# define PR_MTE_TCF_SYNC (1UL << 1) +# define PR_MTE_TCF_ASYNC (1UL << 2) +# define PR_MTE_TCF_MASK (PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC) /* MTE tag inclusion mask */ # define PR_MTE_TAG_SHIFT 3 # define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT) +/* Unused; kept only for source compatibility */ +# define PR_MTE_TCF_SHIFT 1 /* Control reclaim behavior when allocating memory */ #define PR_SET_IO_FLUSHER 57 @@ -250,4 +251,8 @@ struct prctl_mm_map { #define PR_SET_VMA 0x53564d41 # define PR_SET_VMA_ANON_NAME 0 +/* Set/get enabled arm64 pointer authentication keys */ +#define PR_PAC_SET_ENABLED_KEYS 60 +#define PR_PAC_GET_ENABLED_KEYS 61 + #endif /* _LINUX_PRCTL_H */ diff --git a/kernel/fork.c b/kernel/fork.c index 9e6beba23b98..c1771f539901 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -443,6 +443,7 @@ void put_task_stack(struct task_struct *tsk) if (refcount_dec_and_test(&tsk->stack_refcount)) release_task_stack(tsk); } +EXPORT_SYMBOL_GPL(put_task_stack); #endif void free_task(struct task_struct *tsk) diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 5a7b1428cdea..5518c070d1d9 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -195,6 +195,7 @@ static void group_init(struct psi_group *group) INIT_DELAYED_WORK(&group->avgs_work, psi_avgs_work); mutex_init(&group->avgs_lock); /* Init trigger-related members */ + atomic_set(&group->poll_scheduled, 0); mutex_init(&group->trigger_lock); INIT_LIST_HEAD(&group->triggers); memset(group->nr_triggers, 0, sizeof(group->nr_triggers)); @@ -560,18 +561,17 @@ static u64 update_triggers(struct psi_group *group, u64 now) return now + group->poll_min_period; } -/* Schedule polling if it's not already scheduled. */ -static void psi_schedule_poll_work(struct psi_group *group, unsigned long delay) +/* Schedule polling if it's not already scheduled or forced. */ +static void psi_schedule_poll_work(struct psi_group *group, unsigned long delay, + bool force) { struct task_struct *task; /* - * Do not reschedule if already scheduled. - * Possible race with a timer scheduled after this check but before - * mod_timer below can be tolerated because group->polling_next_update - * will keep updates on schedule. + * atomic_xchg should be called even when !force to provide a + * full memory barrier (see the comment inside psi_poll_work). */ - if (timer_pending(&group->poll_timer)) + if (atomic_xchg(&group->poll_scheduled, 1) && !force) return; rcu_read_lock(); @@ -583,12 +583,15 @@ static void psi_schedule_poll_work(struct psi_group *group, unsigned long delay) */ if (likely(task)) mod_timer(&group->poll_timer, jiffies + delay); + else + atomic_set(&group->poll_scheduled, 0); rcu_read_unlock(); } static void psi_poll_work(struct psi_group *group) { + bool force_reschedule = false; u32 changed_states; u64 now; @@ -596,6 +599,43 @@ static void psi_poll_work(struct psi_group *group) now = sched_clock(); + if (now > group->polling_until) { + /* + * We are either about to start or might stop polling if no + * state change was recorded. Resetting poll_scheduled leaves + * a small window for psi_group_change to sneak in and schedule + * an immegiate poll_work before we get to rescheduling. One + * potential extra wakeup at the end of the polling window + * should be negligible and polling_next_update still keeps + * updates correctly on schedule. + */ + atomic_set(&group->poll_scheduled, 0); + /* + * A task change can race with the poll worker that is supposed to + * report on it. To avoid missing events, ensure ordering between + * poll_scheduled and the task state accesses, such that if the poll + * worker misses the state update, the task change is guaranteed to + * reschedule the poll worker: + * + * poll worker: + * atomic_set(poll_scheduled, 0) + * smp_mb() + * LOAD states + * + * task change: + * STORE states + * if atomic_xchg(poll_scheduled, 1) == 0: + * schedule poll worker + * + * The atomic_xchg() implies a full barrier. + */ + smp_mb(); + } else { + /* Polling window is not over, keep rescheduling */ + force_reschedule = true; + } + + collect_percpu_times(group, PSI_POLL, &changed_states); if (changed_states & group->poll_states) { @@ -621,7 +661,8 @@ static void psi_poll_work(struct psi_group *group) group->polling_next_update = update_triggers(group, now); psi_schedule_poll_work(group, - nsecs_to_jiffies(group->polling_next_update - now) + 1); + nsecs_to_jiffies(group->polling_next_update - now) + 1, + force_reschedule); out: mutex_unlock(&group->trigger_lock); @@ -750,7 +791,7 @@ static void psi_group_change(struct psi_group *group, int cpu, write_seqcount_end(&groupc->seq); if (state_mask & group->poll_states) - psi_schedule_poll_work(group, 1); + psi_schedule_poll_work(group, 1, false); if (wake_clock && !delayed_work_pending(&group->avgs_work)) schedule_delayed_work(&group->avgs_work, PSI_FREQ); @@ -1214,6 +1255,7 @@ static void psi_trigger_destroy(struct kref *ref) * can no longer be found through group->poll_task. */ kthread_stop(task_to_destroy); + atomic_set(&group->poll_scheduled, 0); } kfree(t); } diff --git a/kernel/sys.c b/kernel/sys.c index 452ac56bc628..e000e9b38e89 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -123,6 +123,12 @@ #ifndef PAC_RESET_KEYS # define PAC_RESET_KEYS(a, b) (-EINVAL) #endif +#ifndef PAC_SET_ENABLED_KEYS +# define PAC_SET_ENABLED_KEYS(a, b, c) (-EINVAL) +#endif +#ifndef PAC_GET_ENABLED_KEYS +# define PAC_GET_ENABLED_KEYS(a) (-EINVAL) +#endif #ifndef SET_TAGGED_ADDR_CTRL # define SET_TAGGED_ADDR_CTRL(a) (-EINVAL) #endif @@ -2651,6 +2657,16 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, return -EINVAL; error = PAC_RESET_KEYS(me, arg2); break; + case PR_PAC_SET_ENABLED_KEYS: + if (arg4 || arg5) + return -EINVAL; + error = PAC_SET_ENABLED_KEYS(me, arg2, arg3); + break; + case PR_PAC_GET_ENABLED_KEYS: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + error = PAC_GET_ENABLED_KEYS(me); + break; case PR_SET_TAGGED_ADDR_CTRL: if (arg3 || arg4 || arg5) return -EINVAL; diff --git a/mm/cma.c b/mm/cma.c index 7bbc98a90e29..47538607045e 100644 --- a/mm/cma.c +++ b/mm/cma.c @@ -443,6 +443,7 @@ struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, int num_attempts = 0; int max_retries = 5; s64 ts; + struct cma_alloc_info cma_info = {0}; trace_android_vh_cma_alloc_start(&ts); @@ -466,6 +467,8 @@ struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, goto out; for (;;) { + struct acr_info info = {0}; + mutex_lock(&cma->lock); bitmap_no = bitmap_find_next_zero_area_off(cma->bitmap, bitmap_maxno, start, bitmap_count, mask, @@ -474,7 +477,8 @@ struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, if ((num_attempts < max_retries) && (ret == -EBUSY)) { mutex_unlock(&cma->lock); - if (fatal_signal_pending(current)) + if (fatal_signal_pending(current) || + (gfp_mask & __GFP_NORETRY)) break; /* @@ -503,7 +507,18 @@ struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, mutex_unlock(&cma->lock); pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit); - ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, gfp_mask); + ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, gfp_mask, &info); + cma_info.nr_migrated += info.nr_migrated; + cma_info.nr_reclaimed += info.nr_reclaimed; + cma_info.nr_mapped += info.nr_mapped; + if (info.err) { + if (info.err & ACR_ERR_ISOLATE) + cma_info.nr_isolate_fail++; + if (info.err & ACR_ERR_MIGRATE) + cma_info.nr_migrate_fail++; + if (info.err & ACR_ERR_TEST) + cma_info.nr_test_fail++; + } if (ret == 0) { page = pfn_to_page(pfn); break; @@ -518,11 +533,20 @@ struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align, trace_cma_alloc_busy_retry(cma->name, pfn, pfn_to_page(pfn), count, align); - /* try again with a bit different memory target */ - start = bitmap_no + mask + 1; + + if (info.failed_pfn && gfp_mask & __GFP_NORETRY) { + /* try again from following failed page */ + start = (pfn_max_align_up(info.failed_pfn + 1) - + cma->base_pfn) >> cma->order_per_bit; + + } else { + /* try again with a bit different memory target */ + start = bitmap_no + mask + 1; + } } trace_cma_alloc_finish(cma->name, pfn, page, count, align); + trace_cma_alloc_info(cma->name, page, count, align, &cma_info); /* * CMA can allocate multiple page blocks, which results in different diff --git a/mm/memory.c b/mm/memory.c index 244da70a3db2..33da187c3185 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1302,7 +1302,8 @@ again: page_remove_rmap(page, false); if (unlikely(page_mapcount(page) < 0)) print_bad_pte(vma, addr, ptent, page); - if (unlikely(__tlb_remove_page(tlb, page))) { + if (unlikely(__tlb_remove_page(tlb, page)) || + lru_cache_disabled()) { force_flush = 1; addr += PAGE_SIZE; break; diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index f1b193096ffa..41ac69bc0299 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1541,12 +1541,14 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages) /* set above range as isolated */ ret = start_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE, - MEMORY_OFFLINE | REPORT_FAILURE); + MEMORY_OFFLINE | REPORT_FAILURE, NULL); if (ret) { reason = "failure to isolate range"; goto failed_removal; } + drain_all_pages(zone); + arg.start_pfn = start_pfn; arg.nr_pages = nr_pages; node_states_check_changes_offline(nr_pages, zone, &arg); @@ -1596,18 +1598,17 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages) } /* - * per-cpu pages are drained in start_isolate_page_range, but if - * there are still pages that are not free, make sure that we - * drain again, because when we isolated range we might - * have raced with another thread that was adding pages to pcp - * list. + * per-cpu pages are drained after start_isolate_page_range, but + * if there are still pages that are not free, make sure that we + * drain again, because when we isolated range we might have + * raced with another thread that was adding pages to pcp list. * * Forward progress should be still guaranteed because * pages on the pcp list can only belong to MOVABLE_ZONE * because has_unmovable_pages explicitly checks for * PageBuddy on freed pages on other zones. */ - ret = test_pages_isolated(start_pfn, end_pfn, MEMORY_OFFLINE); + ret = test_pages_isolated(start_pfn, end_pfn, MEMORY_OFFLINE, NULL); if (ret) drain_all_pages(zone); } while (ret); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 2eaa7778733d..d1cc0dea6f0d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -8540,7 +8540,7 @@ static unsigned long pfn_max_align_down(unsigned long pfn) pageblock_nr_pages) - 1); } -static unsigned long pfn_max_align_up(unsigned long pfn) +unsigned long pfn_max_align_up(unsigned long pfn) { return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, pageblock_nr_pages)); @@ -8579,7 +8579,8 @@ static inline void alloc_contig_dump_pages(struct list_head *page_list) /* [start, end) must belong to a single zone. */ static int __alloc_contig_migrate_range(struct compact_control *cc, - unsigned long start, unsigned long end) + unsigned long start, unsigned long end, + struct acr_info *info) { /* This function is based on compact_zone() from compaction.c. */ unsigned int nr_reclaimed; @@ -8587,6 +8588,7 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, unsigned int tries = 0; unsigned int max_tries = 5; int ret = 0; + struct page *page; struct migration_target_control mtc = { .nid = zone_to_nid(cc->zone), .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, @@ -8618,10 +8620,16 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, &cc->migratepages); + info->nr_reclaimed += nr_reclaimed; cc->nr_migratepages -= nr_reclaimed; + list_for_each_entry(page, &cc->migratepages, lru) + info->nr_mapped += page_mapcount(page); + ret = migrate_pages(&cc->migratepages, alloc_migration_target, NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE); + if (!ret) + info->nr_migrated += cc->nr_migratepages; } lru_cache_enable(); @@ -8630,7 +8638,14 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, alloc_contig_dump_pages(&cc->migratepages); page_pinner_mark_migration_failed_pages(&cc->migratepages); } + + if (!list_empty(&cc->migratepages)) { + page = list_first_entry(&cc->migratepages, struct page , lru); + info->failed_pfn = page_to_pfn(page); + } + putback_movable_pages(&cc->migratepages); + info->err |= ACR_ERR_MIGRATE; return ret; } return 0; @@ -8658,7 +8673,8 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, * need to be freed with free_contig_range(). */ int alloc_contig_range(unsigned long start, unsigned long end, - unsigned migratetype, gfp_t gfp_mask) + unsigned migratetype, gfp_t gfp_mask, + struct acr_info *info) { unsigned long outer_start, outer_end; unsigned int order; @@ -8701,9 +8717,14 @@ int alloc_contig_range(unsigned long start, unsigned long end, */ ret = start_isolate_page_range(pfn_max_align_down(start), - pfn_max_align_up(end), migratetype, 0); - if (ret) + pfn_max_align_up(end), migratetype, 0, + &info->failed_pfn); + if (ret) { + info->err |= ACR_ERR_ISOLATE; return ret; + } + + drain_all_pages(cc.zone); /* * In case of -EBUSY, we'd like to know which page causes problem. @@ -8715,8 +8736,8 @@ int alloc_contig_range(unsigned long start, unsigned long end, * allocated. So, if we fall through be sure to clear ret so that * -EBUSY is not accidentally used or returned to caller. */ - ret = __alloc_contig_migrate_range(&cc, start, end); - if (ret && ret != -EBUSY) + ret = __alloc_contig_migrate_range(&cc, start, end, info); + if (ret && (ret != -EBUSY || (gfp_mask & __GFP_NORETRY))) goto done; ret =0; @@ -8761,10 +8782,11 @@ int alloc_contig_range(unsigned long start, unsigned long end, } /* Make sure the range is really isolated. */ - if (test_pages_isolated(outer_start, end, 0)) { + if (test_pages_isolated(outer_start, end, 0, &info->failed_pfn)) { pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n", __func__, outer_start, end); ret = -EBUSY; + info->err |= ACR_ERR_TEST; goto done; } @@ -8791,10 +8813,11 @@ EXPORT_SYMBOL(alloc_contig_range); static int __alloc_contig_pages(unsigned long start_pfn, unsigned long nr_pages, gfp_t gfp_mask) { + struct acr_info dummy; unsigned long end_pfn = start_pfn + nr_pages; return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE, - gfp_mask); + gfp_mask, &dummy); } static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn, diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 6ecd3f432336..b391091bfd5c 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -49,7 +49,6 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_ __mod_zone_freepage_state(zone, -nr_pages, mt); spin_unlock_irqrestore(&zone->lock, flags); - drain_all_pages(zone); return 0; } @@ -172,16 +171,18 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages) * * Please note that there is no strong synchronization with the page allocator * either. Pages might be freed while their page blocks are marked ISOLATED. - * In some cases pages might still end up on pcp lists and that would allow + * A call to drain_all_pages() after isolation can flush most of them. However + * in some cases pages might still end up on pcp lists and that would allow * for their allocation even when they are in fact isolated already. Depending - * on how strong of a guarantee the caller needs drain_all_pages might be needed - * (e.g. __offline_pages will need to call it after check for isolated range for - * a next retry). + * on how strong of a guarantee the caller needs, further drain_all_pages() + * might be needed (e.g. __offline_pages will need to call it after check for + * isolated range for a next retry). * * Return: 0 on success and -EBUSY if any part of range cannot be isolated. */ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, - unsigned migratetype, int flags) + unsigned migratetype, int flags, + unsigned long *failed_pfn) { unsigned long pfn; unsigned long undo_pfn; @@ -197,6 +198,8 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, if (page) { if (set_migratetype_isolate(page, migratetype, flags)) { undo_pfn = pfn; + if (failed_pfn) + *failed_pfn = page_to_pfn(page); goto undo; } } @@ -282,7 +285,7 @@ __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn, /* Caller should ensure that requested range is in a single zone */ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn, - int isol_flags) + int isol_flags, unsigned long *failed_pfn) { unsigned long pfn, flags; struct page *page; @@ -310,6 +313,8 @@ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn, trace_test_pages_isolated(start_pfn, end_pfn, pfn); if (pfn < end_pfn) { page_pinner_failure_detect(pfn_to_page(pfn)); + if (failed_pfn) + *failed_pfn = pfn; return -EBUSY; } diff --git a/mm/page_pinner.c b/mm/page_pinner.c index f4a141aafac6..3f5c48af3135 100644 --- a/mm/page_pinner.c +++ b/mm/page_pinner.c @@ -59,7 +59,7 @@ static bool page_pinner_enabled; DEFINE_STATIC_KEY_FALSE(page_pinner_inited); DEFINE_STATIC_KEY_TRUE(failure_tracking); -EXPORT_SYMBOL(failure_tracking); +EXPORT_SYMBOL_GPL(failure_tracking); static depot_stack_handle_t failure_handle; @@ -350,7 +350,7 @@ void __page_pinner_migration_failed(struct page *page) acf_pinner.pinner[idx] = record; spin_unlock_irqrestore(&acf_pinner.lock, flags); } -EXPORT_SYMBOL(__page_pinner_migration_failed); +EXPORT_SYMBOL_GPL(__page_pinner_migration_failed); void __page_pinner_mark_migration_failed_pages(struct list_head *page_list) { diff --git a/mm/shmem.c b/mm/shmem.c index 16da32afb2c9..3fcd9610311a 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -4306,9 +4306,9 @@ struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, } EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); -void shmem_mark_page_lazyfree(struct page *page) +void shmem_mark_page_lazyfree(struct page *page, bool tail) { - mark_page_lazyfree_movetail(page); + mark_page_lazyfree_movetail(page, tail); } EXPORT_SYMBOL_GPL(shmem_mark_page_lazyfree); diff --git a/mm/swap.c b/mm/swap.c index 171213a64202..8d5c61de5a6e 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -634,6 +634,8 @@ static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec, static void lru_lazyfree_movetail_fn(struct page *page, struct lruvec *lruvec, void *arg) { + bool *add_to_tail = (bool *)arg; + if (PageLRU(page) && !PageUnevictable(page) && PageSwapBacked(page) && !PageSwapCache(page)) { bool active = PageActive(page); @@ -642,7 +644,10 @@ static void lru_lazyfree_movetail_fn(struct page *page, struct lruvec *lruvec, LRU_INACTIVE_ANON + active); ClearPageActive(page); ClearPageReferenced(page); - add_page_to_lru_list_tail(page, lruvec, LRU_INACTIVE_FILE); + if (add_to_tail && *add_to_tail) + add_page_to_lru_list_tail(page, lruvec, LRU_INACTIVE_FILE); + else + add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE); } } @@ -769,7 +774,7 @@ void mark_page_lazyfree(struct page *page) * mark_page_lazyfree_movetail() moves @page to the tail of inactive file list. * This is done to accelerate the reclaim of @page. */ -void mark_page_lazyfree_movetail(struct page *page) +void mark_page_lazyfree_movetail(struct page *page, bool tail) { if (PageLRU(page) && !PageUnevictable(page) && PageSwapBacked(page) && !PageSwapCache(page)) { @@ -780,7 +785,7 @@ void mark_page_lazyfree_movetail(struct page *page) get_page(page); if (pagevec_add_and_need_flush(pvec, page)) pagevec_lru_move_fn(pvec, - lru_lazyfree_movetail_fn, NULL); + lru_lazyfree_movetail_fn, &tail); local_unlock(&lru_pvecs.lock); } } @@ -924,7 +929,18 @@ void lru_add_drain_all(void) } #endif /* CONFIG_SMP */ -atomic_t lru_disable_count = ATOMIC_INIT(0); +static atomic_t lru_disable_count = ATOMIC_INIT(0); + +bool lru_cache_disabled(void) +{ + return atomic_read(&lru_disable_count) != 0; +} + +void lru_cache_enable(void) +{ + atomic_dec(&lru_disable_count); +} +EXPORT_SYMBOL_GPL(lru_cache_enable); /* * lru_cache_disable() needs to be called before we start compiling @@ -936,7 +952,12 @@ atomic_t lru_disable_count = ATOMIC_INIT(0); */ void lru_cache_disable(void) { - atomic_inc(&lru_disable_count); + /* + * If someone is already disabled lru_cache, just return with + * increasing the lru_disable_count. + */ + if (atomic_inc_not_zero(&lru_disable_count)) + return; #ifdef CONFIG_SMP /* * lru_add_drain_all in the force mode will schedule draining on @@ -950,7 +971,9 @@ void lru_cache_disable(void) #else lru_add_drain(); #endif + atomic_inc(&lru_disable_count); } +EXPORT_SYMBOL_GPL(lru_cache_disable); /** * release_pages - batched put_page() diff --git a/net/core/sock.c b/net/core/sock.c index e640a68f34ba..ffc1e1e8ec73 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -136,6 +136,7 @@ #include #include +#include #include #include @@ -1678,6 +1679,8 @@ static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, if (security_sk_alloc(sk, family, priority)) goto out_free; + trace_android_rvh_sk_alloc(sk); + if (!try_module_get(prot->owner)) goto out_free_sec; sk_tx_queue_clear(sk); @@ -1687,6 +1690,7 @@ static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority, out_free_sec: security_sk_free(sk); + trace_android_rvh_sk_free(sk); out_free: if (slab != NULL) kmem_cache_free(slab, sk); @@ -1706,6 +1710,7 @@ static void sk_prot_free(struct proto *prot, struct sock *sk) cgroup_sk_free(&sk->sk_cgrp_data); mem_cgroup_sk_free(sk); security_sk_free(sk); + trace_android_rvh_sk_free(sk); if (slab != NULL) kmem_cache_free(slab, sk); else diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index a00f764083dd..df0d19a7e557 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -90,6 +90,8 @@ #include #include +#include + #define INFINITY_LIFE_TIME 0xFFFFFFFF #define IPV6_MAX_STRLEN \ @@ -3344,6 +3346,7 @@ static void addrconf_addr_gen(struct inet6_dev *idev, bool prefix_route) static void addrconf_dev_config(struct net_device *dev) { struct inet6_dev *idev; + bool ret = false; ASSERT_RTNL(); @@ -3371,6 +3374,10 @@ static void addrconf_dev_config(struct net_device *dev) if (IS_ERR(idev)) return; + trace_android_vh_ipv6_gen_linklocal_addr(dev, &ret); + if (ret) + return; + /* this device type has no EUI support */ if (dev->type == ARPHRD_NONE && idev->cnf.addr_gen_mode == IN6_ADDR_GEN_MODE_EUI64) diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index ff0168736f6e..573e0b817d05 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -1514,6 +1515,8 @@ __nf_conntrack_alloc(struct net *net, nf_ct_zone_add(ct, zone); + trace_android_rvh_nf_conn_alloc(ct); + /* Because we use RCU lookups, we set ct_general.use to zero before * this is inserted in any list. */ @@ -1546,6 +1549,7 @@ void nf_conntrack_free(struct nf_conn *ct) nf_ct_ext_destroy(ct); kmem_cache_free(nf_conntrack_cachep, ct); smp_mb__before_atomic(); + trace_android_rvh_nf_conn_free(ct); atomic_dec(&net->ct.count); } EXPORT_SYMBOL_GPL(nf_conntrack_free); diff --git a/sound/core/compress_offload.c b/sound/core/compress_offload.c index c1fec932c49d..f43a1dce3bd2 100644 --- a/sound/core/compress_offload.c +++ b/sound/core/compress_offload.c @@ -33,6 +33,10 @@ #include #include +#ifndef __GENKSYMS__ +#include +#endif + /* struct snd_compr_codec_caps overflows the ioctl bit size for some * architectures, so we need to disable the relevant ioctls. */ @@ -708,6 +712,20 @@ snd_compr_tstamp(struct snd_compr_stream *stream, unsigned long arg) static int snd_compr_pause(struct snd_compr_stream *stream) { int retval; + bool use_pause_in_drain = false; + bool leave_draining_state = false; + + trace_android_vh_snd_compr_use_pause_in_drain(&use_pause_in_drain, + &leave_draining_state); + + if (use_pause_in_drain && stream->runtime->state == SNDRV_PCM_STATE_DRAINING) { + retval = stream->ops->trigger(stream, SNDRV_PCM_TRIGGER_PAUSE_PUSH); + if (!retval && leave_draining_state) { + stream->runtime->state = SNDRV_PCM_STATE_PAUSED; + wake_up(&stream->runtime->sleep); + } + return retval; + } if (stream->runtime->state != SNDRV_PCM_STATE_RUNNING) return -EPERM; @@ -720,6 +738,14 @@ static int snd_compr_pause(struct snd_compr_stream *stream) static int snd_compr_resume(struct snd_compr_stream *stream) { int retval; + bool use_pause_in_drain = false; + bool leave_draining_state = false; + + trace_android_vh_snd_compr_use_pause_in_drain(&use_pause_in_drain, + &leave_draining_state); + + if (use_pause_in_drain && stream->runtime->state == SNDRV_PCM_STATE_DRAINING) + return stream->ops->trigger(stream, SNDRV_PCM_TRIGGER_PAUSE_RELEASE); if (stream->runtime->state != SNDRV_PCM_STATE_PAUSED) return -EPERM; diff --git a/sound/usb/card.h b/sound/usb/card.h index e3a59455f71c..3683f10ff542 100644 --- a/sound/usb/card.h +++ b/sound/usb/card.h @@ -178,6 +178,8 @@ struct snd_usb_substream { bool trigger_tstamp_pending_update; /* trigger timestamp being updated from initial estimate */ struct media_ctl *media_ctl; + + ANDROID_KABI_RESERVE(1); }; struct snd_usb_stream { diff --git a/sound/usb/pcm.c b/sound/usb/pcm.c index 0b90a4d04d7d..9e93a43ab353 100644 --- a/sound/usb/pcm.c +++ b/sound/usb/pcm.c @@ -14,6 +14,8 @@ #include #include +#include + #include "usbaudio.h" #include "card.h" #include "quirks.h" @@ -1508,6 +1510,7 @@ static int snd_usb_pcm_open(struct snd_pcm_substream *substream) struct snd_pcm_runtime *runtime = substream->runtime; struct snd_usb_substream *subs = &as->substream[direction]; int ret; + bool is_support = false; ret = snd_vendor_set_pcm_connection(subs->dev, SOUND_PCM_OPEN, direction); @@ -1532,6 +1535,11 @@ static int snd_usb_pcm_open(struct snd_pcm_substream *substream) if (ret) snd_usb_autosuspend(subs->stream->chip); } + + trace_android_vh_sound_usb_support_cpu_suspend(subs->dev, direction, &is_support); + if (!ret && is_support) + snd_usb_autosuspend(subs->stream->chip); + return ret; } @@ -1541,12 +1549,17 @@ static int snd_usb_pcm_close(struct snd_pcm_substream *substream) struct snd_usb_stream *as = snd_pcm_substream_chip(substream); struct snd_usb_substream *subs = &as->substream[direction]; int ret; + bool is_support = false; ret = snd_vendor_set_pcm_connection(subs->dev, SOUND_PCM_CLOSE, direction); if (ret) return ret; + trace_android_vh_sound_usb_support_cpu_suspend(subs->dev, direction, &is_support); + if (!ret && is_support) + snd_usb_autoresume(subs->stream->chip); + snd_media_stop_pipeline(subs); if (!as->chip->keep_iface && diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index 47f56e0354cc..6d40b46c3c13 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -12,6 +12,8 @@ #define USB_ID_VENDOR(id) ((id) >> 16) #define USB_ID_PRODUCT(id) ((u16)(id)) +#include + /* * */ @@ -64,6 +66,11 @@ struct snd_usb_audio { struct mutex dev_lock; /* to protect any race with disconnect */ int card_num; /* cache pcm card number to use upon disconnect */ void (*disconnect_cb)(struct snd_usb_audio *chip); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); + ANDROID_KABI_RESERVE(3); + ANDROID_KABI_RESERVE(4); }; #define USB_AUDIO_IFACE_UNUSED ((void *)-1L) @@ -176,6 +183,9 @@ struct snd_usb_audio_vendor_ops { struct audioformat *found, int *cur_attr, int *attr); int (*usb_add_ctls)(struct snd_usb_audio *chip); + + ANDROID_KABI_RESERVE(1); + ANDROID_KABI_RESERVE(2); }; #endif /* __USBAUDIO_H */ diff --git a/tools/crypto/gen_fips140_testvecs.py b/tools/crypto/gen_fips140_testvecs.py new file mode 100755 index 000000000000..25164d83389c --- /dev/null +++ b/tools/crypto/gen_fips140_testvecs.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright 2021 Google LLC +# +# Generate most of the test vectors for the FIPS 140 cryptographic self-tests. +# +# Usage: +# tools/crypto/gen_fips140_testvecs.py > crypto/fips140-generated-testvecs.h +# +# Prerequisites: +# Debian: apt-get install python3-pycryptodome python3-cryptography +# Arch Linux: pacman -S python-pycryptodomex python-cryptography + +import hashlib +import hmac +import os + +import Cryptodome.Cipher.AES +import Cryptodome.Util.Counter + +import cryptography.hazmat.primitives.ciphers +import cryptography.hazmat.primitives.ciphers.algorithms +import cryptography.hazmat.primitives.ciphers.modes + +scriptname = os.path.basename(__file__) + +message = bytes('This is a 32-byte test message.\0', 'ascii') +aes_key = bytes('128-bit AES key\0', 'ascii') +aes_xts_key = bytes('This is an AES-128-XTS key.\0\0\0\0\0', 'ascii') +aes_iv = bytes('ABCDEFGHIJKL\0\0\0\0', 'ascii') +assoc = bytes('associated data string', 'ascii') +hmac_key = bytes('128-bit HMAC key', 'ascii') + +def warn_generated(): + print(f'''/* + * This header was automatically generated by {scriptname}. + * Don't edit it directly. + */''') + +def is_string_value(value): + return (value.isascii() and + all(c == '\x00' or c.isprintable() for c in str(value, 'ascii'))) + +def format_value(value, is_string): + if is_string: + return value + hexstr = '' + for byte in value: + hexstr += f'\\x{byte:02x}' + return hexstr + +def print_value(name, value): + is_string = is_string_value(value) + hdr = f'static const u8 fips_{name}[{len(value)}] __initconst =' + print(hdr, end='') + if is_string: + value = str(value, 'ascii').rstrip('\x00') + chars_per_byte = 1 + else: + chars_per_byte = 4 + bytes_per_line = 64 // chars_per_byte + + if len(hdr) + (chars_per_byte * len(value)) + 4 <= 80: + print(f' "{format_value(value, is_string)}"', end='') + else: + for chunk in [value[i:i+bytes_per_line] + for i in range(0, len(value), bytes_per_line)]: + print(f'\n\t"{format_value(chunk, is_string)}"', end='') + print(';') + print('') + +def generate_aes_testvecs(): + print_value('aes_key', aes_key) + print_value('aes_iv', aes_iv) + + cbc = Cryptodome.Cipher.AES.new(aes_key, Cryptodome.Cipher.AES.MODE_CBC, + iv=aes_iv) + print_value('aes_cbc_ciphertext', cbc.encrypt(message)) + + ecb = Cryptodome.Cipher.AES.new(aes_key, Cryptodome.Cipher.AES.MODE_ECB) + print_value('aes_ecb_ciphertext', ecb.encrypt(message)) + + ctr = Cryptodome.Cipher.AES.new(aes_key, Cryptodome.Cipher.AES.MODE_CTR, + nonce=aes_iv[:12]) + print_value('aes_ctr_ciphertext', ctr.encrypt(message)) + + print_value('aes_gcm_assoc', assoc) + gcm = Cryptodome.Cipher.AES.new(aes_key, Cryptodome.Cipher.AES.MODE_GCM, + nonce=aes_iv[:12], mac_len=16) + gcm.update(assoc) + raw_ciphertext, tag = gcm.encrypt_and_digest(message) + print_value('aes_gcm_ciphertext', raw_ciphertext + tag) + + # Unfortunately, pycryptodome doesn't support XTS, so for it we need to use + # a different Python package (the "cryptography" package). + print_value('aes_xts_key', aes_xts_key) + xts = cryptography.hazmat.primitives.ciphers.Cipher( + cryptography.hazmat.primitives.ciphers.algorithms.AES(aes_xts_key), + cryptography.hazmat.primitives.ciphers.modes.XTS(aes_iv)).encryptor() + ciphertext = xts.update(message) + xts.finalize() + print_value('aes_xts_ciphertext', ciphertext) + +def generate_sha_testvecs(): + print_value('hmac_key', hmac_key) + for alg in ['sha1', 'sha256', 'hmac_sha256', 'sha512']: + if alg.startswith('hmac_'): + h = hmac.new(hmac_key, message, alg.removeprefix('hmac_')) + else: + h = hashlib.new(alg, message) + print_value(f'{alg}_digest', h.digest()) + +print('/* SPDX-License-Identifier: GPL-2.0-only */') +print('/* Copyright 2021 Google LLC */') +print('') +warn_generated() +print('') +print_value('message', message) +generate_aes_testvecs() +generate_sha_testvecs() +warn_generated()