diff --git a/qcom/opensource/graphics-kernel/Android.bp b/qcom/opensource/graphics-kernel/Android.bp new file mode 100644 index 0000000000..44160ae331 --- /dev/null +++ b/qcom/opensource/graphics-kernel/Android.bp @@ -0,0 +1,35 @@ +headers_src = [ + "include/uapi/linux/*.h", +] + +gfx_headers_out = [ + "linux/msm_kgsl.h", +] + +gfx_kernel_headers_verbose = "--verbose " +genrule { + name: "qti_generate_gfx_kernel_headers", + tools: ["headers_install.sh", + "unifdef" + ], + tool_files: [ + "gfx_kernel_headers.py", + ], + srcs: headers_src, + cmd: "python3 -u $(location gfx_kernel_headers.py) " + + gfx_kernel_headers_verbose + + "--header_arch arm64 " + + "--gen_dir $(genDir) " + + "--gfx_include_uapi $(locations include/uapi/linux/*.h) " + + "--unifdef $(location unifdef) " + + "--headers_install $(location headers_install.sh)", + out: gfx_headers_out, +} + +cc_library_headers { + name: "qti_gfx_kernel_uapi", + generated_headers: ["qti_generate_gfx_kernel_headers"], + export_generated_headers: ["qti_generate_gfx_kernel_headers"], + vendor: true, + recovery_available: true +} diff --git a/qcom/opensource/graphics-kernel/Android.mk b/qcom/opensource/graphics-kernel/Android.mk new file mode 100644 index 0000000000..1511d9f38f --- /dev/null +++ b/qcom/opensource/graphics-kernel/Android.mk @@ -0,0 +1,57 @@ +ifeq ($(TARGET_USES_QMAA),true) + KGSL_ENABLED := false + ifeq ($(TARGET_USES_QMAA_OVERRIDE_GFX),true) + KGSL_ENABLED := true + endif # TARGET_USES_QMAA_OVERRIDE_GFX +else + KGSL_ENABLED := true +endif # TARGET_USES_QMAA + +ifeq ($(ENABLE_HYP), true) + KGSL_ENABLED := false +endif + +LOCAL_MODULE_DDK_BUILD := true +LOCAL_MODULE_DDK_ALLOW_UNSAFE_HEADERS := true + +ifeq ($(KGSL_ENABLED),true) +KGSL_SELECT := CONFIG_QCOM_KGSL=m + +LOCAL_PATH := $(call my-dir) +include $(CLEAR_VARS) + +# This makefile is only for DLKM +ifneq ($(findstring vendor,$(LOCAL_PATH)),) + +ifeq ($(BOARD_COMMON_DIR),) + BOARD_COMMON_DIR := device/qcom/common +endif + +DLKM_DIR := $(BOARD_COMMON_DIR)/dlkm + +KBUILD_OPTIONS += BOARD_PLATFORM=$(TARGET_BOARD_PLATFORM) +KBUILD_OPTIONS += $(KGSL_SELECT) +KBUILD_OPTIONS += MODNAME=msm_kgsl +ifeq ($(TARGET_BOARD_PLATFORM), pineapple) + KBUILD_OPTIONS += KBUILD_EXTRA_SYMBOLS+=$(PWD)/$(call intermediates-dir-for,DLKM,hw-fence-module-symvers)/Module.symvers +endif + +include $(CLEAR_VARS) +# For incremental compilation +LOCAL_SRC_FILES := $(wildcard $(LOCAL_PATH)/**/*) $(wildcard $(LOCAL_PATH)/*) +LOCAL_MODULE := msm_kgsl.ko +LOCAL_MODULE_KBUILD_NAME := msm_kgsl.ko +LOCAL_MODULE_TAGS := optional +LOCAL_MODULE_DEBUG_ENABLE := true +LOCAL_MODULE_PATH := $(KERNEL_MODULES_OUT) + +ifeq ($(TARGET_BOARD_PLATFORM), pineapple) + LOCAL_REQUIRED_MODULES := hw-fence-module-symvers + LOCAL_ADDITIONAL_DEPENDENCIES := $(call intermediates-dir-for,DLKM,hw-fence-module-symvers)/Module.symvers +endif +# Include msm_kgsl.ko in the /vendor/lib/modules (vendor.img) +BOARD_VENDOR_KERNEL_MODULES += $(LOCAL_MODULE_PATH)/$(LOCAL_MODULE) +include $(DLKM_DIR)/Build_external_kernelmodule.mk + +endif # DLKM check +endif # KGSL_ENABLED diff --git a/qcom/opensource/graphics-kernel/BUILD.bazel b/qcom/opensource/graphics-kernel/BUILD.bazel new file mode 100644 index 0000000000..ace170700e --- /dev/null +++ b/qcom/opensource/graphics-kernel/BUILD.bazel @@ -0,0 +1,6 @@ +load(":build/kgsl_defs.bzl", "define_target_module") + +define_target_module("pineapple") +define_target_module("sun") +define_target_module("blair") +define_target_module("monaco") diff --git a/qcom/opensource/graphics-kernel/Kbuild b/qcom/opensource/graphics-kernel/Kbuild new file mode 100644 index 0000000000..3f49319076 --- /dev/null +++ b/qcom/opensource/graphics-kernel/Kbuild @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: GPL-2.0-only + +KDIR := $(TOP)/kernel_platform/common + +ifeq ($(KGSL_PATH),) +KGSL_PATH=$(src) +endif + +# If we're not GVM and not in an Android tree, select KGSL config +ifeq ($(CONFIG_QTI_QUIN_GVM),) + ifeq ($(ANDROID_BUILD_TOP),) + CONFIG_QCOM_KGSL = m + endif +endif + +ifeq ($(CONFIG_ARCH_WAIPIO), y) + include $(KGSL_PATH)/config/gki_waipiodisp.conf +endif +ifeq ($(CONFIG_ARCH_KALAMA), y) + include $(KGSL_PATH)/config/gki_kalama.conf +endif +ifeq ($(CONFIG_ARCH_PINEAPPLE), y) + include $(KGSL_PATH)/config/gki_pineapple.conf +endif +ifeq ($(CONFIG_ARCH_BLAIR), y) + include $(KGSL_PATH)/config/gki_blair.conf +endif +ifeq ($(CONFIG_ARCH_PITTI), y) + include $(KGSL_PATH)/config/gki_pitti.conf +endif +ifeq ($(CONFIG_ARCH_SA8155), y) + include $(KGSL_PATH)/config/gki_sa8155.conf +endif +ifeq ($(CONFIG_ARCH_KHAJE), y) + include $(KGSL_PATH)/config/gki_khajedisp.conf +endif +ifeq ($(CONFIG_ARCH_SA8195), y) + include $(KGSL_PATH)/config/gki_sa8155.conf +endif +ifeq ($(CONFIG_ARCH_SA6155), y) + include $(KGSL_PATH)/config/gki_sa8155.conf +endif +ifeq ($(CONFIG_ARCH_MONACO), y) + include $(KGSL_PATH)/config/gki_monaco.conf +endif +ifeq ($(CONFIG_ARCH_LEMANS), y) + include $(KGSL_PATH)/config/gki_lemans.conf +endif +ifeq ($(CONFIG_ARCH_KONA), y) + include $(KGSL_PATH)/config/gki_kona.conf +endif +ifeq ($(CONFIG_ARCH_TRINKET), y) + include $(KGSL_PATH)/config/gki_trinket.conf +endif +ifeq ($(CONFIG_ARCH_QCS405), y) + include $(KGSL_PATH)/config/gki_qcs405.conf +endif +ifeq ($(CONFIG_ARCH_HOLI), y) + include $(KGSL_PATH)/config/gki_blair.conf +endif + +ccflags-y += -I$(KGSL_PATH) -I$(KGSL_PATH)/include/linux -I$(KGSL_PATH)/include -I$(KERNEL_SRC)/drivers/devfreq + +obj-$(CONFIG_QCOM_KGSL) += msm_kgsl.o + +msm_kgsl-y = \ + kgsl.o \ + kgsl_bus.o \ + kgsl_drawobj.o \ + kgsl_events.o \ + kgsl_eventlog.o \ + kgsl_gmu_core.o \ + kgsl_ioctl.o \ + kgsl_mmu.o \ + kgsl_pwrctrl.o \ + kgsl_pwrscale.o \ + kgsl_regmap.o \ + kgsl_sharedmem.o \ + kgsl_snapshot.o \ + kgsl_timeline.o \ + kgsl_trace.o \ + kgsl_util.o \ + kgsl_vbo.o + +msm_kgsl-$(CONFIG_COMPAT) += kgsl_compat.o +msm_kgsl-$(CONFIG_DEBUG_FS) += kgsl_debugfs.o +msm_kgsl-$(CONFIG_ARM_SMMU) += kgsl_iommu.o +msm_kgsl-$(CONFIG_SYNC_FILE) += kgsl_sync.o +msm_kgsl-$(CONFIG_QCOM_KGSL_PROCESS_RECLAIM) += kgsl_reclaim.o + +ifndef CONFIG_QCOM_KGSL_USE_SHMEM + msm_kgsl-y += kgsl_pool.o +endif + +msm_kgsl-y += \ + adreno.o \ + adreno_a3xx.o \ + adreno_a3xx_perfcounter.o \ + adreno_a3xx_ringbuffer.o \ + adreno_a3xx_snapshot.o \ + adreno_a5xx.o \ + adreno_a5xx_perfcounter.o \ + adreno_a5xx_preempt.o \ + adreno_a5xx_ringbuffer.o \ + adreno_a5xx_snapshot.o \ + adreno_a6xx.o \ + adreno_a6xx_gmu.o \ + adreno_a6xx_gmu_snapshot.o \ + adreno_a6xx_hfi.o \ + adreno_a6xx_hwsched.o \ + adreno_a6xx_hwsched_hfi.o \ + adreno_a6xx_perfcounter.o \ + adreno_a6xx_preempt.o \ + adreno_a6xx_rgmu.o \ + adreno_a6xx_ringbuffer.o \ + adreno_a6xx_rpmh.o \ + adreno_a6xx_snapshot.o \ + adreno_cp_parser.o \ + adreno_dispatch.o \ + adreno_drawctxt.o \ + adreno_gen7.o \ + adreno_gen7_gmu.o \ + adreno_gen7_gmu_snapshot.o \ + adreno_gen7_hfi.o \ + adreno_gen7_hwsched.o \ + adreno_gen7_hwsched_hfi.o \ + adreno_gen7_perfcounter.o \ + adreno_gen7_preempt.o \ + adreno_gen7_ringbuffer.o \ + adreno_gen7_rpmh.o \ + adreno_gen7_snapshot.o \ + adreno_gen8.o \ + adreno_gen8_gmu.o \ + adreno_gen8_gmu_snapshot.o \ + adreno_gen8_hfi.o \ + adreno_gen8_hwsched.o \ + adreno_gen8_hwsched_hfi.o \ + adreno_gen8_perfcounter.o \ + adreno_gen8_preempt.o \ + adreno_gen8_ringbuffer.o \ + adreno_gen8_rpmh.o \ + adreno_gen8_snapshot.o \ + adreno_hwsched.o \ + adreno_ioctl.o \ + adreno_perfcounter.o \ + adreno_ringbuffer.o \ + adreno_snapshot.o \ + adreno_sysfs.o \ + adreno_trace.o \ + governor_msm_adreno_tz.o \ + governor_gpubw_mon.o + +msm_kgsl-$(CONFIG_COMPAT) += adreno_compat.o +msm_kgsl-$(CONFIG_QCOM_KGSL_CORESIGHT) += adreno_coresight.o +msm_kgsl-$(CONFIG_QCOM_KGSL_CORESIGHT) += adreno_a3xx_coresight.o +msm_kgsl-$(CONFIG_QCOM_KGSL_CORESIGHT) += adreno_a5xx_coresight.o +msm_kgsl-$(CONFIG_QCOM_KGSL_CORESIGHT) += adreno_a6xx_coresight.o +msm_kgsl-$(CONFIG_QCOM_KGSL_CORESIGHT) += adreno_gen7_coresight.o +msm_kgsl-$(CONFIG_DEBUG_FS) += adreno_debugfs.o adreno_profile.o diff --git a/qcom/opensource/graphics-kernel/Kconfig b/qcom/opensource/graphics-kernel/Kconfig new file mode 100644 index 0000000000..3cb8e1f3bd --- /dev/null +++ b/qcom/opensource/graphics-kernel/Kconfig @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: GPL-2.0-only +config QCOM_KGSL + tristate "Qualcomm Technologies, Inc. 3D Graphics driver" + depends on ARCH_QCOM + depends on NVMEM_QCOM_QFPROM || QCOM_QFPROM + select QCOM_MDT_LOADER + select INTERVAL_TREE + select TRACE_GPU_MEM + help + 3D graphics driver for the Adreno family of GPUs from QTI. + Required to use hardware accelerated OpenGL, compute and Vulkan + on QTI targets. This includes power management, memory management, + and scheduling for the Adreno GPUs. + +config DEVFREQ_GOV_QCOM_ADRENO_TZ + tristate "Qualcomm Technologies, Inc. GPU frequency governor" + depends on PM_DEVFREQ + help + GPU frequency governor for the Adreno GPU. Sets the frequency + using an "on demand" algorithm in conjunction with other + components on Adreno platforms. This is not useful for non-Adreno + devices. + +config DEVFREQ_GOV_QCOM_GPUBW_MON + tristate "Qualcomm Technologies, Inc. GPU bandwidth governor" + depends on DEVFREQ_GOV_QCOM_ADRENO_TZ + help + This governor works together with the Adreno GPU governor to + select bus frequency votes using an "on-demand" algorithm. + This governor will not be useful for non-Adreno based + targets. + +config QCOM_KGSL_FENCE_TRACE + bool "Enable built-in tracing for adreno fence timeouts" + help + A boolean flag used to create a KGSL-specific tracing instance + under /tracing/instances/kgsl-fence that can be used + for debugging timeouts for fences between KGSL-contexts and + sync-point blocks. If unsure, say 'N' here. + +config QCOM_ADRENO_DEFAULT_GOVERNOR + string "devfreq governor for the adreno core" + default "msm-adreno-tz" + +config QCOM_KGSL_CORESIGHT + bool "Enable coresight support for the Adreno GPU" + depends on CORESIGHT + default y + help + When enabled, the Adreno GPU is available as a source for Coresight + data. On a6xx targets there are two sources available for the GX and + CX domains respectively. Debug kernels should say 'Y' here. + +config QCOM_KGSL_IOCOHERENCY_DEFAULT + bool "Enable I/O coherency on cached GPU memory by default" + default y if ARCH_LAHAINA + help + Say 'Y' here to enable I/O cache coherency by default on targets that + support hardware I/O coherency. If enabled all cached GPU memory + will use I/O coherency regardless of the user flags. If not enabled + the user can still selectively enable I/O coherency with a flag. + +config QCOM_KGSL_IDLE_TIMEOUT + int + default 80 + help + GPU idle timeout for Adreno GPU. This value decides after how + long the GPU will go into slumber. A higher value will mean that + the GPU is powered ON for a longer duration which will have + power costs. + +config QCOM_KGSL_CONTEXT_DEBUG + bool "Log kgsl context information for all processes" + help + When enabled, total number of KGSL contexts, number of attached and + detached contexts are dumped into kernel log for all the processes. + This gives insight about the number of contexts held by each process. + +config QCOM_KGSL_SORT_POOL + bool "Sort pool page list based on physical address" + default y + help + When enabled, the pool page list is sorted based on physical + addresses. This can be turned on for targets where better DDR + efficiency is attained on accesses for adjacent memory. + +config QCOM_KGSL_QDSS_STM + bool "Enable support for QDSS STM for Adreno GPU" + depends on CORESIGHT + help + When enabled, the Adreno GPU QDSS STM support is enabled. GPU QDSS STM + memory will be mapped to GPU and QDSS clock needed to access this memory + is voted. Debug kernels should say 'Y' here. + +config QCOM_KGSL_USE_SHMEM + bool "Enable using shmem for memory allocations" + depends on SHMEM + help + Say 'Y' to enable using shmem for memory allocations. If enabled, + there will be no support for the memory pools and higher order pages. + But using shmem will help in making kgsl pages available for + reclaiming. + +config QCOM_KGSL_PROCESS_RECLAIM + bool "Make driver pages available for reclaim" + select QCOM_KGSL_USE_SHMEM + help + Say 'Y' to make driver pages available for reclaiming. If enabled, + shmem will be used for allocation. kgsl would know the process + foreground/background activity through the sysfs entry exposed per + process. Based on this kgsl can unpin given number of pages from + background processes and make them available to the shrinker. + +config QCOM_KGSL_HIBERNATION + bool "Enable Hibernation support in KGSL" + depends on HIBERNATION + help + Say 'Y' to enable hibernation support in kgsl. If enabled, kgsl + will register necessary power manager callbacks to support + hibernation. diff --git a/qcom/opensource/graphics-kernel/Makefile b/qcom/opensource/graphics-kernel/Makefile new file mode 100644 index 0000000000..26501f9779 --- /dev/null +++ b/qcom/opensource/graphics-kernel/Makefile @@ -0,0 +1,17 @@ +ifeq ($(KGSL_MODULE_ROOT),) +CUR_MKFILE = $(abspath $(lastword $(MAKEFILE_LIST))) +KGSL_MODULE_ROOT = $(dir $(CUR_MKFILE)) +endif + +KBUILD_OPTIONS+=KGSL_PATH=$(KGSL_MODULE_ROOT) + +all: modules + +modules_install: + $(MAKE) INSTALL_MOD_STRIP=1 -C $(KERNEL_SRC) M=$(M) modules_install + +clean: + rm -f *.cmd *.d *.mod *.o *.ko *.mod.c *.mod.o Module.symvers modules.order + +%: + $(MAKE) -C $(KERNEL_SRC) M=$(M) $@ $(KBUILD_OPTIONS) diff --git a/qcom/opensource/graphics-kernel/a3xx_reg.h b/qcom/opensource/graphics-kernel/a3xx_reg.h new file mode 100644 index 0000000000..ab5079aa45 --- /dev/null +++ b/qcom/opensource/graphics-kernel/a3xx_reg.h @@ -0,0 +1,564 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2012-2017,2019-2020, The Linux Foundation. All rights reserved. + */ + +#ifndef _A300_REG_H +#define _A300_REG_H + +/* Interrupt bit positions within RBBM_INT_0 */ + +#define A3XX_INT_RBBM_GPU_IDLE 0 +#define A3XX_INT_RBBM_AHB_ERROR 1 +#define A3XX_INT_RBBM_REG_TIMEOUT 2 +#define A3XX_INT_RBBM_ME_MS_TIMEOUT 3 +#define A3XX_INT_RBBM_PFP_MS_TIMEOUT 4 +#define A3XX_INT_RBBM_ATB_BUS_OVERFLOW 5 +#define A3XX_INT_VFD_ERROR 6 +#define A3XX_INT_CP_SW_INT 7 +#define A3XX_INT_CP_T0_PACKET_IN_IB 8 +#define A3XX_INT_CP_OPCODE_ERROR 9 +#define A3XX_INT_CP_RESERVED_BIT_ERROR 10 +#define A3XX_INT_CP_HW_FAULT 11 +#define A3XX_INT_CP_DMA 12 +#define A3XX_INT_CP_IB2_INT 13 +#define A3XX_INT_CP_IB1_INT 14 +#define A3XX_INT_CP_RB_INT 15 +#define A3XX_INT_CP_REG_PROTECT_FAULT 16 +#define A3XX_INT_CP_RB_DONE_TS 17 +#define A3XX_INT_CP_VS_DONE_TS 18 +#define A3XX_INT_CP_PS_DONE_TS 19 +#define A3XX_INT_CACHE_FLUSH_TS 20 +#define A3XX_INT_CP_AHB_ERROR_HALT 21 +#define A3XX_INT_MISC_HANG_DETECT 24 +#define A3XX_INT_UCHE_OOB_ACCESS 25 + +/* Register definitions */ + +#define A3XX_RBBM_CLOCK_CTL 0x010 +#define A3XX_RBBM_SP_HYST_CNT 0x012 +#define A3XX_RBBM_SW_RESET_CMD 0x018 +#define A3XX_RBBM_AHB_CTL0 0x020 +#define A3XX_RBBM_AHB_CTL1 0x021 +#define A3XX_RBBM_AHB_CMD 0x022 +#define A3XX_RBBM_AHB_ERROR_STATUS 0x027 +#define A3XX_RBBM_GPR0_CTL 0x02E +/* This the same register as on A2XX, just in a different place */ +#define A3XX_RBBM_STATUS 0x030 +#define A3XX_RBBM_WAIT_IDLE_CLOCKS_CTL 0x33 +#define A3XX_RBBM_INTERFACE_HANG_INT_CTL 0x50 +#define A3XX_RBBM_INT_CLEAR_CMD 0x061 +#define A3XX_RBBM_INT_0_MASK 0x063 +#define A3XX_RBBM_INT_0_STATUS 0x064 +#define A3XX_RBBM_PERFCTR_CTL 0x80 +#define A3XX_RBBM_PERFCTR_LOAD_CMD0 0x81 +#define A3XX_RBBM_PERFCTR_LOAD_CMD1 0x82 +#define A3XX_RBBM_PERFCTR_LOAD_VALUE_LO 0x84 +#define A3XX_RBBM_PERFCTR_LOAD_VALUE_HI 0x85 +#define A3XX_RBBM_PERFCOUNTER0_SELECT 0x86 +#define A3XX_RBBM_PERFCOUNTER1_SELECT 0x87 +#define A3XX_RBBM_GPU_BUSY_MASKED 0x88 +#define A3XX_RBBM_PERFCTR_CP_0_LO 0x90 +#define A3XX_RBBM_PERFCTR_CP_0_HI 0x91 +#define A3XX_RBBM_PERFCTR_RBBM_0_LO 0x92 +#define A3XX_RBBM_PERFCTR_RBBM_0_HI 0x93 +#define A3XX_RBBM_PERFCTR_RBBM_1_LO 0x94 +#define A3XX_RBBM_PERFCTR_RBBM_1_HI 0x95 +#define A3XX_RBBM_PERFCTR_PC_0_LO 0x96 +#define A3XX_RBBM_PERFCTR_PC_0_HI 0x97 +#define A3XX_RBBM_PERFCTR_PC_1_LO 0x98 +#define A3XX_RBBM_PERFCTR_PC_1_HI 0x99 +#define A3XX_RBBM_PERFCTR_PC_2_LO 0x9A +#define A3XX_RBBM_PERFCTR_PC_2_HI 0x9B +#define A3XX_RBBM_PERFCTR_PC_3_LO 0x9C +#define A3XX_RBBM_PERFCTR_PC_3_HI 0x9D +#define A3XX_RBBM_PERFCTR_VFD_0_LO 0x9E +#define A3XX_RBBM_PERFCTR_VFD_0_HI 0x9F +#define A3XX_RBBM_PERFCTR_VFD_1_LO 0xA0 +#define A3XX_RBBM_PERFCTR_VFD_1_HI 0xA1 +#define A3XX_RBBM_PERFCTR_HLSQ_0_LO 0xA2 +#define A3XX_RBBM_PERFCTR_HLSQ_0_HI 0xA3 +#define A3XX_RBBM_PERFCTR_HLSQ_1_LO 0xA4 +#define A3XX_RBBM_PERFCTR_HLSQ_1_HI 0xA5 +#define A3XX_RBBM_PERFCTR_HLSQ_2_LO 0xA6 +#define A3XX_RBBM_PERFCTR_HLSQ_2_HI 0xA7 +#define A3XX_RBBM_PERFCTR_HLSQ_3_LO 0xA8 +#define A3XX_RBBM_PERFCTR_HLSQ_3_HI 0xA9 +#define A3XX_RBBM_PERFCTR_HLSQ_4_LO 0xAA +#define A3XX_RBBM_PERFCTR_HLSQ_4_HI 0xAB +#define A3XX_RBBM_PERFCTR_HLSQ_5_LO 0xAC +#define A3XX_RBBM_PERFCTR_HLSQ_5_HI 0xAD +#define A3XX_RBBM_PERFCTR_VPC_0_LO 0xAE +#define A3XX_RBBM_PERFCTR_VPC_0_HI 0xAF +#define A3XX_RBBM_PERFCTR_VPC_1_LO 0xB0 +#define A3XX_RBBM_PERFCTR_VPC_1_HI 0xB1 +#define A3XX_RBBM_PERFCTR_TSE_0_LO 0xB2 +#define A3XX_RBBM_PERFCTR_TSE_0_HI 0xB3 +#define A3XX_RBBM_PERFCTR_TSE_1_LO 0xB4 +#define A3XX_RBBM_PERFCTR_TSE_1_HI 0xB5 +#define A3XX_RBBM_PERFCTR_RAS_0_LO 0xB6 +#define A3XX_RBBM_PERFCTR_RAS_0_HI 0xB7 +#define A3XX_RBBM_PERFCTR_RAS_1_LO 0xB8 +#define A3XX_RBBM_PERFCTR_RAS_1_HI 0xB9 +#define A3XX_RBBM_PERFCTR_UCHE_0_LO 0xBA +#define A3XX_RBBM_PERFCTR_UCHE_0_HI 0xBB +#define A3XX_RBBM_PERFCTR_UCHE_1_LO 0xBC +#define A3XX_RBBM_PERFCTR_UCHE_1_HI 0xBD +#define A3XX_RBBM_PERFCTR_UCHE_2_LO 0xBE +#define A3XX_RBBM_PERFCTR_UCHE_2_HI 0xBF +#define A3XX_RBBM_PERFCTR_UCHE_3_LO 0xC0 +#define A3XX_RBBM_PERFCTR_UCHE_3_HI 0xC1 +#define A3XX_RBBM_PERFCTR_UCHE_4_LO 0xC2 +#define A3XX_RBBM_PERFCTR_UCHE_4_HI 0xC3 +#define A3XX_RBBM_PERFCTR_UCHE_5_LO 0xC4 +#define A3XX_RBBM_PERFCTR_UCHE_5_HI 0xC5 +#define A3XX_RBBM_PERFCTR_TP_0_LO 0xC6 +#define A3XX_RBBM_PERFCTR_TP_0_HI 0xC7 +#define A3XX_RBBM_PERFCTR_TP_1_LO 0xC8 +#define A3XX_RBBM_PERFCTR_TP_1_HI 0xC9 +#define A3XX_RBBM_PERFCTR_TP_2_LO 0xCA +#define A3XX_RBBM_PERFCTR_TP_2_HI 0xCB +#define A3XX_RBBM_PERFCTR_TP_3_LO 0xCC +#define A3XX_RBBM_PERFCTR_TP_3_HI 0xCD +#define A3XX_RBBM_PERFCTR_TP_4_LO 0xCE +#define A3XX_RBBM_PERFCTR_TP_4_HI 0xCF +#define A3XX_RBBM_PERFCTR_TP_5_LO 0xD0 +#define A3XX_RBBM_PERFCTR_TP_5_HI 0xD1 +#define A3XX_RBBM_PERFCTR_SP_0_LO 0xD2 +#define A3XX_RBBM_PERFCTR_SP_0_HI 0xD3 +#define A3XX_RBBM_PERFCTR_SP_1_LO 0xD4 +#define A3XX_RBBM_PERFCTR_SP_1_HI 0xD5 +#define A3XX_RBBM_PERFCTR_SP_2_LO 0xD6 +#define A3XX_RBBM_PERFCTR_SP_2_HI 0xD7 +#define A3XX_RBBM_PERFCTR_SP_3_LO 0xD8 +#define A3XX_RBBM_PERFCTR_SP_3_HI 0xD9 +#define A3XX_RBBM_PERFCTR_SP_4_LO 0xDA +#define A3XX_RBBM_PERFCTR_SP_4_HI 0xDB +#define A3XX_RBBM_PERFCTR_SP_5_LO 0xDC +#define A3XX_RBBM_PERFCTR_SP_5_HI 0xDD +#define A3XX_RBBM_PERFCTR_SP_6_LO 0xDE +#define A3XX_RBBM_PERFCTR_SP_6_HI 0xDF +#define A3XX_RBBM_PERFCTR_SP_7_LO 0xE0 +#define A3XX_RBBM_PERFCTR_SP_7_HI 0xE1 +#define A3XX_RBBM_PERFCTR_RB_0_LO 0xE2 +#define A3XX_RBBM_PERFCTR_RB_0_HI 0xE3 +#define A3XX_RBBM_PERFCTR_RB_1_LO 0xE4 +#define A3XX_RBBM_PERFCTR_RB_1_HI 0xE5 + +#define A3XX_RBBM_RBBM_CTL 0x100 +#define A3XX_RBBM_PERFCTR_PWR_0_LO 0x0EA +#define A3XX_RBBM_PERFCTR_PWR_0_HI 0x0EB +#define A3XX_RBBM_PERFCTR_PWR_1_LO 0x0EC +#define A3XX_RBBM_PERFCTR_PWR_1_HI 0x0ED +#define A3XX_RBBM_DEBUG_BUS_CTL 0x111 +#define A3XX_RBBM_DEBUG_BUS_DATA_STATUS 0x112 +#define A3XX_RBBM_DEBUG_BUS_STB_CTL0 0x11B +#define A3XX_RBBM_DEBUG_BUS_STB_CTL1 0x11C +#define A3XX_RBBM_INT_TRACE_BUS_CTL 0x11D +#define A3XX_RBBM_EXT_TRACE_BUS_CTL 0x11E +#define A3XX_RBBM_EXT_TRACE_STOP_CNT 0x11F +#define A3XX_RBBM_EXT_TRACE_START_CNT 0x120 +#define A3XX_RBBM_EXT_TRACE_PERIOD_CNT 0x121 +#define A3XX_RBBM_EXT_TRACE_CMD 0x122 +#define A3XX_CP_RB_BASE 0x01C0 +#define A3XX_CP_RB_CNTL 0x01C1 +#define A3XX_CP_RB_RPTR 0x01C4 +#define A3XX_CP_RB_WPTR 0x01C5 +/* Following two are same as on A2XX, just in a different place */ +#define A3XX_CP_PFP_UCODE_ADDR 0x1C9 +#define A3XX_CP_PFP_UCODE_DATA 0x1CA +#define A3XX_CP_ROQ_ADDR 0x1CC +#define A3XX_CP_ROQ_DATA 0x1CD +#define A3XX_CP_MERCIU_ADDR 0x1D1 +#define A3XX_CP_MERCIU_DATA 0x1D2 +#define A3XX_CP_MERCIU_DATA2 0x1D3 +#define A3XX_CP_QUEUE_THRESHOLDS 0x01D5 +#define A3XX_CP_MEQ_ADDR 0x1DA +#define A3XX_CP_MEQ_DATA 0x1DB +#define A3XX_CP_STATE_DEBUG_INDEX 0x01EC +#define A3XX_CP_STATE_DEBUG_DATA 0x01ED +#define A3XX_CP_CNTL 0x01F4 +#define A3XX_CP_WFI_PEND_CTR 0x01F5 +#define A3XX_CP_ME_CNTL 0x01F6 +#define A3XX_CP_ME_STATUS 0x01F7 +#define A3XX_CP_ME_RAM_WADDR 0x01F8 +#define A3XX_CP_ME_RAM_RADDR 0x01F9 +#define A3XX_CP_ME_RAM_DATA 0x01FA +#define A3XX_CP_DEBUG 0x01FC + +#define A3XX_RBBM_PM_OVERRIDE2 0x039D + +#define A3XX_CP_PERFCOUNTER_SELECT 0x445 +#define A3XX_CP_IB1_BASE 0x0458 +#define A3XX_CP_IB1_BUFSZ 0x0459 +#define A3XX_CP_IB2_BASE 0x045A +#define A3XX_CP_IB2_BUFSZ 0x045B + +#define A3XX_CP_HW_FAULT 0x45C +#define A3XX_CP_PROTECT_CTRL 0x45E +#define A3XX_CP_PROTECT_STATUS 0x45F +#define A3XX_CP_PROTECT_REG_0 0x460 +#define A3XX_CP_STAT 0x047F +#define A3XX_CP_SCRATCH_REG0 0x578 +#define A3XX_CP_SCRATCH_REG6 0x57E +#define A3XX_CP_SCRATCH_REG7 0x57F +#define A3XX_VSC_SIZE_ADDRESS 0xC02 +#define A3XX_VSC_PIPE_DATA_ADDRESS_0 0xC07 +#define A3XX_VSC_PIPE_DATA_LENGTH_0 0xC08 +#define A3XX_VSC_PIPE_DATA_ADDRESS_1 0xC0A +#define A3XX_VSC_PIPE_DATA_LENGTH_1 0xC0B +#define A3XX_VSC_PIPE_DATA_ADDRESS_2 0xC0D +#define A3XX_VSC_PIPE_DATA_LENGTH_2 0xC0E +#define A3XX_VSC_PIPE_DATA_ADDRESS_3 0xC10 +#define A3XX_VSC_PIPE_DATA_LENGTH_3 0xC11 +#define A3XX_VSC_PIPE_DATA_ADDRESS_4 0xC13 +#define A3XX_VSC_PIPE_DATA_LENGTH_4 0xC14 +#define A3XX_VSC_PIPE_DATA_ADDRESS_5 0xC16 +#define A3XX_VSC_PIPE_DATA_LENGTH_5 0xC17 +#define A3XX_VSC_PIPE_DATA_ADDRESS_6 0xC19 +#define A3XX_VSC_PIPE_DATA_LENGTH_6 0xC1A +#define A3XX_VSC_PIPE_DATA_ADDRESS_7 0xC1C +#define A3XX_VSC_PIPE_DATA_LENGTH_7 0xC1D +#define A3XX_PC_PERFCOUNTER0_SELECT 0xC48 +#define A3XX_PC_PERFCOUNTER1_SELECT 0xC49 +#define A3XX_PC_PERFCOUNTER2_SELECT 0xC4A +#define A3XX_PC_PERFCOUNTER3_SELECT 0xC4B +#define A3XX_GRAS_TSE_DEBUG_ECO 0xC81 +#define A3XX_GRAS_PERFCOUNTER0_SELECT 0xC88 +#define A3XX_GRAS_PERFCOUNTER1_SELECT 0xC89 +#define A3XX_GRAS_PERFCOUNTER2_SELECT 0xC8A +#define A3XX_GRAS_PERFCOUNTER3_SELECT 0xC8B +#define A3XX_GRAS_CL_USER_PLANE_X0 0xCA0 +#define A3XX_GRAS_CL_USER_PLANE_Y0 0xCA1 +#define A3XX_GRAS_CL_USER_PLANE_Z0 0xCA2 +#define A3XX_GRAS_CL_USER_PLANE_W0 0xCA3 +#define A3XX_GRAS_CL_USER_PLANE_X1 0xCA4 +#define A3XX_GRAS_CL_USER_PLANE_Y1 0xCA5 +#define A3XX_GRAS_CL_USER_PLANE_Z1 0xCA6 +#define A3XX_GRAS_CL_USER_PLANE_W1 0xCA7 +#define A3XX_GRAS_CL_USER_PLANE_X2 0xCA8 +#define A3XX_GRAS_CL_USER_PLANE_Y2 0xCA9 +#define A3XX_GRAS_CL_USER_PLANE_Z2 0xCAA +#define A3XX_GRAS_CL_USER_PLANE_W2 0xCAB +#define A3XX_GRAS_CL_USER_PLANE_X3 0xCAC +#define A3XX_GRAS_CL_USER_PLANE_Y3 0xCAD +#define A3XX_GRAS_CL_USER_PLANE_Z3 0xCAE +#define A3XX_GRAS_CL_USER_PLANE_W3 0xCAF +#define A3XX_GRAS_CL_USER_PLANE_X4 0xCB0 +#define A3XX_GRAS_CL_USER_PLANE_Y4 0xCB1 +#define A3XX_GRAS_CL_USER_PLANE_Z4 0xCB2 +#define A3XX_GRAS_CL_USER_PLANE_W4 0xCB3 +#define A3XX_GRAS_CL_USER_PLANE_X5 0xCB4 +#define A3XX_GRAS_CL_USER_PLANE_Y5 0xCB5 +#define A3XX_GRAS_CL_USER_PLANE_Z5 0xCB6 +#define A3XX_GRAS_CL_USER_PLANE_W5 0xCB7 +#define A3XX_RB_GMEM_BASE_ADDR 0xCC0 +#define A3XX_RB_DEBUG_ECO_CONTROLS_ADDR 0xCC1 +#define A3XX_RB_PERFCOUNTER0_SELECT 0xCC6 +#define A3XX_RB_PERFCOUNTER1_SELECT 0xCC7 +#define A3XX_RB_FRAME_BUFFER_DIMENSION 0xCE0 +#define A3XX_SQ_GPR_MANAGEMENT 0x0D00 +#define A3XX_SQ_INST_STORE_MANAGEMENT 0x0D02 +#define A3XX_HLSQ_PERFCOUNTER0_SELECT 0xE00 +#define A3XX_HLSQ_PERFCOUNTER1_SELECT 0xE01 +#define A3XX_HLSQ_PERFCOUNTER2_SELECT 0xE02 +#define A3XX_HLSQ_PERFCOUNTER3_SELECT 0xE03 +#define A3XX_HLSQ_PERFCOUNTER4_SELECT 0xE04 +#define A3XX_HLSQ_PERFCOUNTER5_SELECT 0xE05 +#define A3XX_TP0_CHICKEN 0x0E1E +#define A3XX_VFD_PERFCOUNTER0_SELECT 0xE44 +#define A3XX_VFD_PERFCOUNTER1_SELECT 0xE45 +#define A3XX_VPC_VPC_DEBUG_RAM_SEL 0xE61 +#define A3XX_VPC_VPC_DEBUG_RAM_READ 0xE62 +#define A3XX_VPC_PERFCOUNTER0_SELECT 0xE64 +#define A3XX_VPC_PERFCOUNTER1_SELECT 0xE65 +#define A3XX_UCHE_CACHE_MODE_CONTROL_REG 0xE82 +#define A3XX_UCHE_PERFCOUNTER0_SELECT 0xE84 +#define A3XX_UCHE_PERFCOUNTER1_SELECT 0xE85 +#define A3XX_UCHE_PERFCOUNTER2_SELECT 0xE86 +#define A3XX_UCHE_PERFCOUNTER3_SELECT 0xE87 +#define A3XX_UCHE_PERFCOUNTER4_SELECT 0xE88 +#define A3XX_UCHE_PERFCOUNTER5_SELECT 0xE89 +#define A3XX_UCHE_CACHE_INVALIDATE0_REG 0xEA0 +#define A3XX_UCHE_CACHE_INVALIDATE1_REG 0xEA1 +#define A3XX_UCHE_CACHE_WAYS_VFD 0xEA6 +#define A3XX_SP_PERFCOUNTER0_SELECT 0xEC4 +#define A3XX_SP_PERFCOUNTER1_SELECT 0xEC5 +#define A3XX_SP_PERFCOUNTER2_SELECT 0xEC6 +#define A3XX_SP_PERFCOUNTER3_SELECT 0xEC7 +#define A3XX_SP_PERFCOUNTER4_SELECT 0xEC8 +#define A3XX_SP_PERFCOUNTER5_SELECT 0xEC9 +#define A3XX_SP_PERFCOUNTER6_SELECT 0xECA +#define A3XX_SP_PERFCOUNTER7_SELECT 0xECB +#define A3XX_TP_PERFCOUNTER0_SELECT 0xF04 +#define A3XX_TP_PERFCOUNTER1_SELECT 0xF05 +#define A3XX_TP_PERFCOUNTER2_SELECT 0xF06 +#define A3XX_TP_PERFCOUNTER3_SELECT 0xF07 +#define A3XX_TP_PERFCOUNTER4_SELECT 0xF08 +#define A3XX_TP_PERFCOUNTER5_SELECT 0xF09 +#define A3XX_GRAS_CL_CLIP_CNTL 0x2040 +#define A3XX_GRAS_CL_GB_CLIP_ADJ 0x2044 +#define A3XX_GRAS_CL_VPORT_XOFFSET 0x2048 +#define A3XX_GRAS_CL_VPORT_XSCALE 0x2049 +#define A3XX_GRAS_CL_VPORT_YOFFSET 0x204A +#define A3XX_GRAS_CL_VPORT_YSCALE 0x204B +#define A3XX_GRAS_CL_VPORT_ZOFFSET 0x204C +#define A3XX_GRAS_CL_VPORT_ZSCALE 0x204D +#define A3XX_GRAS_SU_POINT_MINMAX 0x2068 +#define A3XX_GRAS_SU_POINT_SIZE 0x2069 +#define A3XX_GRAS_SU_POLY_OFFSET_SCALE 0x206C +#define A3XX_GRAS_SU_POLY_OFFSET_OFFSET 0x206D +#define A3XX_GRAS_SU_MODE_CONTROL 0x2070 +#define A3XX_GRAS_SC_CONTROL 0x2072 +#define A3XX_GRAS_SC_SCREEN_SCISSOR_TL 0x2074 +#define A3XX_GRAS_SC_SCREEN_SCISSOR_BR 0x2075 +#define A3XX_GRAS_SC_WINDOW_SCISSOR_TL 0x2079 +#define A3XX_GRAS_SC_WINDOW_SCISSOR_BR 0x207A +#define A3XX_RB_MODE_CONTROL 0x20C0 +#define A3XX_RB_RENDER_CONTROL 0x20C1 +#define A3XX_RB_MSAA_CONTROL 0x20C2 +#define A3XX_RB_ALPHA_REFERENCE 0x20C3 +#define A3XX_RB_MRT_CONTROL0 0x20C4 +#define A3XX_RB_MRT_BUF_INFO0 0x20C5 +#define A3XX_RB_MRT_BUF_BASE0 0x20C6 +#define A3XX_RB_MRT_BLEND_CONTROL0 0x20C7 +#define A3XX_RB_MRT_CONTROL1 0x20C8 +#define A3XX_RB_MRT_BUF_INFO1 0x20C9 +#define A3XX_RB_MRT_BUF_BASE1 0x20CA +#define A3XX_RB_MRT_BLEND_CONTROL1 0x20CB +#define A3XX_RB_MRT_CONTROL2 0x20CC +#define A3XX_RB_MRT_BUF_INFO2 0x20CD +#define A3XX_RB_MRT_BUF_BASE2 0x20CE +#define A3XX_RB_MRT_BLEND_CONTROL2 0x20CF +#define A3XX_RB_MRT_CONTROL3 0x20D0 +#define A3XX_RB_MRT_BUF_INFO3 0x20D1 +#define A3XX_RB_MRT_BUF_BASE3 0x20D2 +#define A3XX_RB_MRT_BLEND_CONTROL3 0x20D3 +#define A3XX_RB_BLEND_RED 0x20E4 +#define A3XX_RB_BLEND_GREEN 0x20E5 +#define A3XX_RB_BLEND_BLUE 0x20E6 +#define A3XX_RB_BLEND_ALPHA 0x20E7 +#define A3XX_RB_CLEAR_COLOR_DW0 0x20E8 +#define A3XX_RB_CLEAR_COLOR_DW1 0x20E9 +#define A3XX_RB_CLEAR_COLOR_DW2 0x20EA +#define A3XX_RB_CLEAR_COLOR_DW3 0x20EB +#define A3XX_RB_COPY_CONTROL 0x20EC +#define A3XX_RB_COPY_DEST_BASE 0x20ED +#define A3XX_RB_COPY_DEST_PITCH 0x20EE +#define A3XX_RB_COPY_DEST_INFO 0x20EF +#define A3XX_RB_DEPTH_CONTROL 0x2100 +#define A3XX_RB_DEPTH_CLEAR 0x2101 +#define A3XX_RB_DEPTH_BUF_INFO 0x2102 +#define A3XX_RB_DEPTH_BUF_PITCH 0x2103 +#define A3XX_RB_STENCIL_CONTROL 0x2104 +#define A3XX_RB_STENCIL_CLEAR 0x2105 +#define A3XX_RB_STENCIL_BUF_INFO 0x2106 +#define A3XX_RB_STENCIL_BUF_PITCH 0x2107 +#define A3XX_RB_STENCIL_REF_MASK 0x2108 +#define A3XX_RB_STENCIL_REF_MASK_BF 0x2109 +#define A3XX_RB_LRZ_VSC_CONTROL 0x210C +#define A3XX_RB_WINDOW_OFFSET 0x210E +#define A3XX_RB_SAMPLE_COUNT_CONTROL 0x2110 +#define A3XX_RB_SAMPLE_COUNT_ADDR 0x2111 +#define A3XX_RB_Z_CLAMP_MIN 0x2114 +#define A3XX_RB_Z_CLAMP_MAX 0x2115 +#define A3XX_HLSQ_CONTROL_0_REG 0x2200 +#define A3XX_HLSQ_CONTROL_1_REG 0x2201 +#define A3XX_HLSQ_CONTROL_2_REG 0x2202 +#define A3XX_HLSQ_CONTROL_3_REG 0x2203 +#define A3XX_HLSQ_VS_CONTROL_REG 0x2204 +#define A3XX_HLSQ_FS_CONTROL_REG 0x2205 +#define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG 0x2206 +#define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG 0x2207 +#define A3XX_HLSQ_CL_NDRANGE_0_REG 0x220A +#define A3XX_HLSQ_CL_NDRANGE_1_REG 0x220B +#define A3XX_HLSQ_CL_NDRANGE_2_REG 0x220C +#define A3XX_HLSQ_CL_NDRANGE_3_REG 0x220D +#define A3XX_HLSQ_CL_NDRANGE_4_REG 0x220E +#define A3XX_HLSQ_CL_NDRANGE_5_REG 0x220F +#define A3XX_HLSQ_CL_NDRANGE_6_REG 0x2210 +#define A3XX_HLSQ_CL_CONTROL_0_REG 0x2211 +#define A3XX_HLSQ_CL_CONTROL_1_REG 0x2212 +#define A3XX_HLSQ_CL_KERNEL_CONST_REG 0x2214 +#define A3XX_HLSQ_CL_KERNEL_GROUP_X_REG 0x2215 +#define A3XX_HLSQ_CL_KERNEL_GROUP_Y_REG 0x2216 +#define A3XX_HLSQ_CL_KERNEL_GROUP_Z_REG 0x2217 +#define A3XX_HLSQ_CL_WG_OFFSET_REG 0x221A +#define A3XX_VFD_FETCH_INSTR_1_0 0x2247 +#define A3XX_VFD_FETCH_INSTR_1_1 0x2249 +#define A3XX_VFD_FETCH_INSTR_1_2 0x224B +#define A3XX_VFD_FETCH_INSTR_1_3 0x224D +#define A3XX_VFD_FETCH_INSTR_1_4 0x224F +#define A3XX_VFD_FETCH_INSTR_1_5 0x2251 +#define A3XX_VFD_FETCH_INSTR_1_6 0x2253 +#define A3XX_VFD_FETCH_INSTR_1_7 0x2255 +#define A3XX_VFD_FETCH_INSTR_1_8 0x2257 +#define A3XX_VFD_FETCH_INSTR_1_9 0x2259 +#define A3XX_VFD_FETCH_INSTR_1_A 0x225B +#define A3XX_VFD_FETCH_INSTR_1_B 0x225D +#define A3XX_VFD_FETCH_INSTR_1_C 0x225F +#define A3XX_VFD_FETCH_INSTR_1_D 0x2261 +#define A3XX_VFD_FETCH_INSTR_1_E 0x2263 +#define A3XX_VFD_FETCH_INSTR_1_F 0x2265 +#define A3XX_SP_SP_CTRL_REG 0x22C0 +#define A3XX_SP_VS_CTRL_REG0 0x22C4 +#define A3XX_SP_VS_CTRL_REG1 0x22C5 +#define A3XX_SP_VS_PARAM_REG 0x22C6 +#define A3XX_SP_VS_OUT_REG_0 0x22C7 +#define A3XX_SP_VS_OUT_REG_1 0x22C8 +#define A3XX_SP_VS_OUT_REG_2 0x22C9 +#define A3XX_SP_VS_OUT_REG_3 0x22CA +#define A3XX_SP_VS_OUT_REG_4 0x22CB +#define A3XX_SP_VS_OUT_REG_5 0x22CC +#define A3XX_SP_VS_OUT_REG_6 0x22CD +#define A3XX_SP_VS_OUT_REG_7 0x22CE +#define A3XX_SP_VS_VPC_DST_REG_0 0x22D0 +#define A3XX_SP_VS_VPC_DST_REG_1 0x22D1 +#define A3XX_SP_VS_VPC_DST_REG_2 0x22D2 +#define A3XX_SP_VS_VPC_DST_REG_3 0x22D3 +#define A3XX_SP_VS_OBJ_OFFSET_REG 0x22D4 +#define A3XX_SP_VS_OBJ_START_REG 0x22D5 +#define A3XX_SP_VS_PVT_MEM_PARAM_REG 0x22D6 +#define A3XX_SP_VS_PVT_MEM_ADDR_REG 0x22D7 +#define A3XX_SP_VS_PVT_MEM_SIZE_REG 0x22D8 +#define A3XX_SP_VS_LENGTH_REG 0x22DF +#define A3XX_SP_FS_CTRL_REG0 0x22E0 +#define A3XX_SP_FS_CTRL_REG1 0x22E1 +#define A3XX_SP_FS_OBJ_OFFSET_REG 0x22E2 +#define A3XX_SP_FS_OBJ_START_REG 0x22E3 +#define A3XX_SP_FS_PVT_MEM_PARAM_REG 0x22E4 +#define A3XX_SP_FS_PVT_MEM_ADDR_REG 0x22E5 +#define A3XX_SP_FS_PVT_MEM_SIZE_REG 0x22E6 +#define A3XX_SP_FS_FLAT_SHAD_MODE_REG_0 0x22E8 +#define A3XX_SP_FS_FLAT_SHAD_MODE_REG_1 0x22E9 +#define A3XX_SP_FS_OUTPUT_REG 0x22EC +#define A3XX_SP_FS_MRT_REG_0 0x22F0 +#define A3XX_SP_FS_MRT_REG_1 0x22F1 +#define A3XX_SP_FS_MRT_REG_2 0x22F2 +#define A3XX_SP_FS_MRT_REG_3 0x22F3 +#define A3XX_SP_FS_IMAGE_OUTPUT_REG_0 0x22F4 +#define A3XX_SP_FS_IMAGE_OUTPUT_REG_1 0x22F5 +#define A3XX_SP_FS_IMAGE_OUTPUT_REG_2 0x22F6 +#define A3XX_SP_FS_IMAGE_OUTPUT_REG_3 0x22F7 +#define A3XX_SP_FS_LENGTH_REG 0x22FF +#define A3XX_PA_SC_AA_CONFIG 0x2301 +#define A3XX_VBIF_CLKON 0x3001 +#define A3XX_VBIF_ABIT_SORT 0x301C +#define A3XX_VBIF_ABIT_SORT_CONF 0x301D +#define A3XX_VBIF_GATE_OFF_WRREQ_EN 0x302A +#define A3XX_VBIF_IN_RD_LIM_CONF0 0x302C +#define A3XX_VBIF_IN_RD_LIM_CONF1 0x302D +#define A3XX_VBIF_IN_WR_LIM_CONF0 0x3030 +#define A3XX_VBIF_IN_WR_LIM_CONF1 0x3031 +#define A3XX_VBIF_OUT_RD_LIM_CONF0 0x3034 +#define A3XX_VBIF_OUT_WR_LIM_CONF0 0x3035 +#define A3XX_VBIF_DDR_OUT_MAX_BURST 0x3036 +#define A3XX_VBIF_ARB_CTL 0x303C +#define A3XX_VBIF_ROUND_ROBIN_QOS_ARB 0x3049 +#define A3XX_VBIF_OUT_AXI_AOOO_EN 0x305E +#define A3XX_VBIF_OUT_AXI_AOOO 0x305F +#define A3XX_VBIF_PERF_CNT0_LO 0x3073 +#define A3XX_VBIF_PERF_CNT0_HI 0x3074 +#define A3XX_VBIF_PERF_CNT1_LO 0x3075 +#define A3XX_VBIF_PERF_CNT1_HI 0x3076 +#define A3XX_VBIF_PERF_PWR_CNT0_LO 0x3077 +#define A3XX_VBIF_PERF_PWR_CNT0_HI 0x3078 +#define A3XX_VBIF_PERF_PWR_CNT1_LO 0x3079 +#define A3XX_VBIF_PERF_PWR_CNT1_HI 0x307a +#define A3XX_VBIF_PERF_PWR_CNT2_LO 0x307b +#define A3XX_VBIF_PERF_PWR_CNT2_HI 0x307c + +#define A3XX_VBIF_XIN_HALT_CTRL0 0x3080 +#define A3XX_VBIF_XIN_HALT_CTRL0_MASK 0x3F +#define A30X_VBIF_XIN_HALT_CTRL0_MASK 0x7 + +#define A3XX_VBIF_XIN_HALT_CTRL1 0x3081 + +/* VBIF register offsets for A306 */ +#define A3XX_VBIF2_PERF_CNT_SEL0 0x30d0 +#define A3XX_VBIF2_PERF_CNT_SEL1 0x30d1 +#define A3XX_VBIF2_PERF_CNT_SEL2 0x30d2 +#define A3XX_VBIF2_PERF_CNT_SEL3 0x30d3 +#define A3XX_VBIF2_PERF_CNT_LOW0 0x30d8 +#define A3XX_VBIF2_PERF_CNT_LOW1 0x30d9 +#define A3XX_VBIF2_PERF_CNT_LOW2 0x30da +#define A3XX_VBIF2_PERF_CNT_LOW3 0x30db +#define A3XX_VBIF2_PERF_CNT_HIGH0 0x30e0 +#define A3XX_VBIF2_PERF_CNT_HIGH1 0x30e1 +#define A3XX_VBIF2_PERF_CNT_HIGH2 0x30e2 +#define A3XX_VBIF2_PERF_CNT_HIGH3 0x30e3 + +#define A3XX_VBIF2_PERF_PWR_CNT_EN0 0x3100 +#define A3XX_VBIF2_PERF_PWR_CNT_EN1 0x3101 +#define A3XX_VBIF2_PERF_PWR_CNT_EN2 0x3102 +#define A3XX_VBIF2_PERF_PWR_CNT_LOW0 0x3110 +#define A3XX_VBIF2_PERF_PWR_CNT_LOW1 0x3111 +#define A3XX_VBIF2_PERF_PWR_CNT_LOW2 0x3112 +#define A3XX_VBIF2_PERF_PWR_CNT_HIGH0 0x3118 +#define A3XX_VBIF2_PERF_PWR_CNT_HIGH1 0x3119 +#define A3XX_VBIF2_PERF_PWR_CNT_HIGH2 0x311a + +#define A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL0 0x3800 +#define A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL1 0x3801 + +/* RBBM Debug bus block IDs */ +#define RBBM_BLOCK_ID_CP 0x1 +#define RBBM_BLOCK_ID_RBBM 0x2 +#define RBBM_BLOCK_ID_VBIF 0x3 +#define RBBM_BLOCK_ID_HLSQ 0x4 +#define RBBM_BLOCK_ID_UCHE 0x5 +#define RBBM_BLOCK_ID_PC 0x8 +#define RBBM_BLOCK_ID_VFD 0x9 +#define RBBM_BLOCK_ID_VPC 0xa +#define RBBM_BLOCK_ID_TSE 0xb +#define RBBM_BLOCK_ID_RAS 0xc +#define RBBM_BLOCK_ID_VSC 0xd +#define RBBM_BLOCK_ID_SP_0 0x10 +#define RBBM_BLOCK_ID_SP_1 0x11 +#define RBBM_BLOCK_ID_SP_2 0x12 +#define RBBM_BLOCK_ID_SP_3 0x13 +#define RBBM_BLOCK_ID_TPL1_0 0x18 +#define RBBM_BLOCK_ID_TPL1_1 0x19 +#define RBBM_BLOCK_ID_TPL1_2 0x1a +#define RBBM_BLOCK_ID_TPL1_3 0x1b +#define RBBM_BLOCK_ID_RB_0 0x20 +#define RBBM_BLOCK_ID_RB_1 0x21 +#define RBBM_BLOCK_ID_RB_2 0x22 +#define RBBM_BLOCK_ID_RB_3 0x23 +#define RBBM_BLOCK_ID_MARB_0 0x28 +#define RBBM_BLOCK_ID_MARB_1 0x29 +#define RBBM_BLOCK_ID_MARB_2 0x2a +#define RBBM_BLOCK_ID_MARB_3 0x2b + +/* RBBM_CLOCK_CTL default value */ +#define A3XX_RBBM_CLOCK_CTL_DEFAULT 0xAAAAAAAA +#define A320_RBBM_CLOCK_CTL_DEFAULT 0xBFFFFFFF +#define A330_RBBM_CLOCK_CTL_DEFAULT 0xBFFCFFFF + +#define A330_RBBM_GPR0_CTL_DEFAULT 0x00000000 +#define A330v2_RBBM_GPR0_CTL_DEFAULT 0x05515455 +#define A310_RBBM_GPR0_CTL_DEFAULT 0x000000AA + +/* COUNTABLE FOR SP PERFCOUNTER */ +#define SP_ALU_ACTIVE_CYCLES 0x1D +#define SP0_ICL1_MISSES 0x1A +#define SP_FS_CFLOW_INSTRUCTIONS 0x0C + +/* COUNTABLE FOR TSE PERFCOUNTER */ +#define TSE_INPUT_PRIM_NUM 0x0 + +/* VBIF countables */ +#define VBIF_AXI_TOTAL_BEATS 85 + +/* VBIF Recoverable HALT bit value */ +#define VBIF_RECOVERABLE_HALT_CTRL 0x1 + +/* + * CP DEBUG settings for A3XX core: + * DYNAMIC_CLK_DISABLE [27] - turn off the dynamic clock control + * MIU_128BIT_WRITE_ENABLE [25] - Allow 128 bit writes to the VBIF + */ +#define A3XX_CP_DEBUG_DEFAULT ((1 << 27) | (1 << 25)) + + +#endif diff --git a/qcom/opensource/graphics-kernel/a5xx_reg.h b/qcom/opensource/graphics-kernel/a5xx_reg.h new file mode 100644 index 0000000000..137a11c3d9 --- /dev/null +++ b/qcom/opensource/graphics-kernel/a5xx_reg.h @@ -0,0 +1,902 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2014-2016,2019, The Linux Foundation. All rights reserved. + */ + +#ifndef _A5XX_REG_H +#define _A5XX_REG_H + +/* A5XX interrupt bits */ +#define A5XX_INT_RBBM_GPU_IDLE 0 +#define A5XX_INT_RBBM_AHB_ERROR 1 +#define A5XX_INT_RBBM_TRANSFER_TIMEOUT 2 +#define A5XX_INT_RBBM_ME_MS_TIMEOUT 3 +#define A5XX_INT_RBBM_PFP_MS_TIMEOUT 4 +#define A5XX_INT_RBBM_ETS_MS_TIMEOUT 5 +#define A5XX_INT_RBBM_ATB_ASYNC_OVERFLOW 6 +#define A5XX_INT_RBBM_GPC_ERROR 7 +#define A5XX_INT_CP_SW 8 +#define A5XX_INT_CP_HW_ERROR 9 +#define A5XX_INT_CP_CCU_FLUSH_DEPTH_TS 10 +#define A5XX_INT_CP_CCU_FLUSH_COLOR_TS 11 +#define A5XX_INT_CP_CCU_RESOLVE_TS 12 +#define A5XX_INT_CP_IB2 13 +#define A5XX_INT_CP_IB1 14 +#define A5XX_INT_CP_RB 15 +#define A5XX_INT_CP_UNUSED_1 16 +#define A5XX_INT_CP_RB_DONE_TS 17 +#define A5XX_INT_CP_WT_DONE_TS 18 +#define A5XX_INT_UNKNOWN_1 19 +#define A5XX_INT_CP_CACHE_FLUSH_TS 20 +#define A5XX_INT_UNUSED_2 21 +#define A5XX_INT_RBBM_ATB_BUS_OVERFLOW 22 +#define A5XX_INT_MISC_HANG_DETECT 23 +#define A5XX_INT_UCHE_OOB_ACCESS 24 +#define A5XX_INT_UCHE_TRAP_INTR 25 +#define A5XX_INT_DEBBUS_INTR_0 26 +#define A5XX_INT_DEBBUS_INTR_1 27 +#define A5XX_INT_GPMU_VOLTAGE_DROOP 28 +#define A5XX_INT_GPMU_FIRMWARE 29 +#define A5XX_INT_ISDB_CPU_IRQ 30 +#define A5XX_INT_ISDB_UNDER_DEBUG 31 + +/* CP Interrupt bits */ +#define A5XX_CP_OPCODE_ERROR 0 +#define A5XX_CP_RESERVED_BIT_ERROR 1 +#define A5XX_CP_HW_FAULT_ERROR 2 +#define A5XX_CP_DMA_ERROR 3 +#define A5XX_CP_REGISTER_PROTECTION_ERROR 4 +#define A5XX_CP_AHB_ERROR 5 + +/* CP registers */ +#define A5XX_CP_RB_BASE 0x800 +#define A5XX_CP_RB_BASE_HI 0x801 +#define A5XX_CP_RB_CNTL 0x802 +#define A5XX_CP_RB_RPTR_ADDR_LO 0x804 +#define A5XX_CP_RB_RPTR_ADDR_HI 0x805 +#define A5XX_CP_RB_RPTR 0x806 +#define A5XX_CP_RB_WPTR 0x807 +#define A5XX_CP_PFP_STAT_ADDR 0x808 +#define A5XX_CP_PFP_STAT_DATA 0x809 +#define A5XX_CP_DRAW_STATE_ADDR 0x80B +#define A5XX_CP_DRAW_STATE_DATA 0x80C +#define A5XX_CP_CRASH_SCRIPT_BASE_LO 0x817 +#define A5XX_CP_CRASH_SCRIPT_BASE_HI 0x818 +#define A5XX_CP_CRASH_DUMP_CNTL 0x819 +#define A5XX_CP_ME_STAT_ADDR 0x81A +#define A5XX_CP_ROQ_THRESHOLDS_1 0x81F +#define A5XX_CP_ROQ_THRESHOLDS_2 0x820 +#define A5XX_CP_ROQ_DBG_ADDR 0x821 +#define A5XX_CP_ROQ_DBG_DATA 0x822 +#define A5XX_CP_MEQ_DBG_ADDR 0x823 +#define A5XX_CP_MEQ_DBG_DATA 0x824 +#define A5XX_CP_MEQ_THRESHOLDS 0x825 +#define A5XX_CP_MERCIU_SIZE 0x826 +#define A5XX_CP_MERCIU_DBG_ADDR 0x827 +#define A5XX_CP_MERCIU_DBG_DATA_1 0x828 +#define A5XX_CP_MERCIU_DBG_DATA_2 0x829 +#define A5XX_CP_PFP_UCODE_DBG_ADDR 0x82A +#define A5XX_CP_PFP_UCODE_DBG_DATA 0x82B +#define A5XX_CP_ME_UCODE_DBG_ADDR 0x82F +#define A5XX_CP_ME_UCODE_DBG_DATA 0x830 +#define A5XX_CP_CNTL 0x831 +#define A5XX_CP_ME_CNTL 0x832 +#define A5XX_CP_CHICKEN_DBG 0x833 +#define A5XX_CP_PFP_INSTR_BASE_LO 0x835 +#define A5XX_CP_PFP_INSTR_BASE_HI 0x836 +#define A5XX_CP_PM4_INSTR_BASE_LO 0x838 +#define A5XX_CP_PM4_INSTR_BASE_HI 0x839 +#define A5XX_CP_CONTEXT_SWITCH_CNTL 0x83B +#define A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO 0x83C +#define A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI 0x83D +#define A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO 0x83E +#define A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_HI 0x83F +#define A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO 0x840 +#define A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI 0x841 +#define A5XX_CP_ADDR_MODE_CNTL 0x860 +#define A5XX_CP_ME_STAT_DATA 0xB14 +#define A5XX_CP_WFI_PEND_CTR 0xB15 +#define A5XX_CP_INTERRUPT_STATUS 0xB18 +#define A5XX_CP_HW_FAULT 0xB1A +#define A5XX_CP_PROTECT_STATUS 0xB1C +#define A5XX_CP_IB1_BASE 0xB1F +#define A5XX_CP_IB1_BASE_HI 0xB20 +#define A5XX_CP_IB1_BUFSZ 0xB21 +#define A5XX_CP_IB2_BASE 0xB22 +#define A5XX_CP_IB2_BASE_HI 0xB23 +#define A5XX_CP_IB2_BUFSZ 0xB24 +#define A5XX_CP_PROTECT_REG_0 0x880 +#define A5XX_CP_PROTECT_CNTL 0x8A0 +#define A5XX_CP_AHB_FAULT 0xB1B +#define A5XX_CP_PERFCTR_CP_SEL_0 0xBB0 +#define A5XX_CP_PERFCTR_CP_SEL_1 0xBB1 +#define A5XX_CP_PERFCTR_CP_SEL_2 0xBB2 +#define A5XX_CP_PERFCTR_CP_SEL_3 0xBB3 +#define A5XX_CP_PERFCTR_CP_SEL_4 0xBB4 +#define A5XX_CP_PERFCTR_CP_SEL_5 0xBB5 +#define A5XX_CP_PERFCTR_CP_SEL_6 0xBB6 +#define A5XX_CP_PERFCTR_CP_SEL_7 0xBB7 + +#define A5XX_VSC_ADDR_MODE_CNTL 0xBC1 + +/* CP Power Counter Registers Select */ +#define A5XX_CP_POWERCTR_CP_SEL_0 0xBBA +#define A5XX_CP_POWERCTR_CP_SEL_1 0xBBB +#define A5XX_CP_POWERCTR_CP_SEL_2 0xBBC +#define A5XX_CP_POWERCTR_CP_SEL_3 0xBBD + +/* RBBM registers */ +#define A5XX_RBBM_CFG_DBGBUS_SEL_A 0x4 +#define A5XX_RBBM_CFG_DBGBUS_SEL_B 0x5 +#define A5XX_RBBM_CFG_DBGBUS_SEL_C 0x6 +#define A5XX_RBBM_CFG_DBGBUS_SEL_D 0x7 +#define A5XX_RBBM_CFG_DBGBUS_SEL_PING_INDEX_SHIFT 0x0 +#define A5XX_RBBM_CFG_DBGBUS_SEL_PING_BLK_SEL_SHIFT 0x8 + +#define A5XX_RBBM_CFG_DBGBUS_CNTLT 0x8 +#define A5XX_RBBM_CFG_DBGBUS_CNTLM 0x9 +#define A5XX_RBBM_CFG_DEBBUS_CTLTM_ENABLE_SHIFT 0x18 +#define A5XX_RBBM_CFG_DBGBUS_OPL 0xA +#define A5XX_RBBM_CFG_DBGBUS_OPE 0xB +#define A5XX_RBBM_CFG_DBGBUS_IVTL_0 0xC +#define A5XX_RBBM_CFG_DBGBUS_IVTL_1 0xD +#define A5XX_RBBM_CFG_DBGBUS_IVTL_2 0xE +#define A5XX_RBBM_CFG_DBGBUS_IVTL_3 0xF +#define A5XX_RBBM_CFG_DBGBUS_MASKL_0 0x10 +#define A5XX_RBBM_CFG_DBGBUS_MASKL_1 0x11 +#define A5XX_RBBM_CFG_DBGBUS_MASKL_2 0x12 +#define A5XX_RBBM_CFG_DBGBUS_MASKL_3 0x13 +#define A5XX_RBBM_CFG_DBGBUS_BYTEL_0 0x14 +#define A5XX_RBBM_CFG_DBGBUS_BYTEL_1 0x15 +#define A5XX_RBBM_CFG_DBGBUS_IVTE_0 0x16 +#define A5XX_RBBM_CFG_DBGBUS_IVTE_1 0x17 +#define A5XX_RBBM_CFG_DBGBUS_IVTE_2 0x18 +#define A5XX_RBBM_CFG_DBGBUS_IVTE_3 0x19 +#define A5XX_RBBM_CFG_DBGBUS_MASKE_0 0x1A +#define A5XX_RBBM_CFG_DBGBUS_MASKE_1 0x1B +#define A5XX_RBBM_CFG_DBGBUS_MASKE_2 0x1C +#define A5XX_RBBM_CFG_DBGBUS_MASKE_3 0x1D +#define A5XX_RBBM_CFG_DBGBUS_NIBBLEE 0x1E +#define A5XX_RBBM_CFG_DBGBUS_PTRC0 0x1F +#define A5XX_RBBM_CFG_DBGBUS_PTRC1 0x20 +#define A5XX_RBBM_CFG_DBGBUS_LOADREG 0x21 +#define A5XX_RBBM_CFG_DBGBUS_IDX 0x22 +#define A5XX_RBBM_CFG_DBGBUS_CLRC 0x23 +#define A5XX_RBBM_CFG_DBGBUS_LOADIVT 0x24 +#define A5XX_RBBM_INTERFACE_HANG_INT_CNTL 0x2F +#define A5XX_RBBM_INT_CLEAR_CMD 0x37 +#define A5XX_RBBM_INT_0_MASK 0x38 +#define A5XX_RBBM_AHB_DBG_CNTL 0x3F +#define A5XX_RBBM_EXT_VBIF_DBG_CNTL 0x41 +#define A5XX_RBBM_SW_RESET_CMD 0x43 +#define A5XX_RBBM_BLOCK_SW_RESET_CMD 0x45 +#define A5XX_RBBM_BLOCK_SW_RESET_CMD2 0x46 +#define A5XX_RBBM_DBG_LO_HI_GPIO 0x48 +#define A5XX_RBBM_EXT_TRACE_BUS_CNTL 0x49 +#define A5XX_RBBM_CLOCK_CNTL_TP0 0x4A +#define A5XX_RBBM_CLOCK_CNTL_TP1 0x4B +#define A5XX_RBBM_CLOCK_CNTL_TP2 0x4C +#define A5XX_RBBM_CLOCK_CNTL_TP3 0x4D +#define A5XX_RBBM_CLOCK_CNTL2_TP0 0x4E +#define A5XX_RBBM_CLOCK_CNTL2_TP1 0x4F +#define A5XX_RBBM_CLOCK_CNTL2_TP2 0x50 +#define A5XX_RBBM_CLOCK_CNTL2_TP3 0x51 +#define A5XX_RBBM_CLOCK_CNTL3_TP0 0x52 +#define A5XX_RBBM_CLOCK_CNTL3_TP1 0x53 +#define A5XX_RBBM_CLOCK_CNTL3_TP2 0x54 +#define A5XX_RBBM_CLOCK_CNTL3_TP3 0x55 +#define A5XX_RBBM_READ_AHB_THROUGH_DBG 0x59 +#define A5XX_RBBM_CLOCK_CNTL_UCHE 0x5A +#define A5XX_RBBM_CLOCK_CNTL2_UCHE 0x5B +#define A5XX_RBBM_CLOCK_CNTL3_UCHE 0x5C +#define A5XX_RBBM_CLOCK_CNTL4_UCHE 0x5D +#define A5XX_RBBM_CLOCK_HYST_UCHE 0x5E +#define A5XX_RBBM_CLOCK_DELAY_UCHE 0x5F +#define A5XX_RBBM_CLOCK_MODE_GPC 0x60 +#define A5XX_RBBM_CLOCK_DELAY_GPC 0x61 +#define A5XX_RBBM_CLOCK_HYST_GPC 0x62 +#define A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM 0x63 +#define A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM 0x64 +#define A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM 0x65 +#define A5XX_RBBM_CLOCK_DELAY_HLSQ 0x66 +#define A5XX_RBBM_CLOCK_CNTL 0x67 +#define A5XX_RBBM_CLOCK_CNTL_SP0 0x68 +#define A5XX_RBBM_CLOCK_CNTL_SP1 0x69 +#define A5XX_RBBM_CLOCK_CNTL_SP2 0x6A +#define A5XX_RBBM_CLOCK_CNTL_SP3 0x6B +#define A5XX_RBBM_CLOCK_CNTL2_SP0 0x6C +#define A5XX_RBBM_CLOCK_CNTL2_SP1 0x6D +#define A5XX_RBBM_CLOCK_CNTL2_SP2 0x6E +#define A5XX_RBBM_CLOCK_CNTL2_SP3 0x6F +#define A5XX_RBBM_CLOCK_HYST_SP0 0x70 +#define A5XX_RBBM_CLOCK_HYST_SP1 0x71 +#define A5XX_RBBM_CLOCK_HYST_SP2 0x72 +#define A5XX_RBBM_CLOCK_HYST_SP3 0x73 +#define A5XX_RBBM_CLOCK_DELAY_SP0 0x74 +#define A5XX_RBBM_CLOCK_DELAY_SP1 0x75 +#define A5XX_RBBM_CLOCK_DELAY_SP2 0x76 +#define A5XX_RBBM_CLOCK_DELAY_SP3 0x77 +#define A5XX_RBBM_CLOCK_CNTL_RB0 0x78 +#define A5XX_RBBM_CLOCK_CNTL_RB1 0x79 +#define A5XX_RBBM_CLOCK_CNTL_RB2 0x7a +#define A5XX_RBBM_CLOCK_CNTL_RB3 0x7B +#define A5XX_RBBM_CLOCK_CNTL2_RB0 0x7C +#define A5XX_RBBM_CLOCK_CNTL2_RB1 0x7D +#define A5XX_RBBM_CLOCK_CNTL2_RB2 0x7E +#define A5XX_RBBM_CLOCK_CNTL2_RB3 0x7F +#define A5XX_RBBM_CLOCK_HYST_RAC 0x80 +#define A5XX_RBBM_CLOCK_DELAY_RAC 0x81 +#define A5XX_RBBM_CLOCK_CNTL_CCU0 0x82 +#define A5XX_RBBM_CLOCK_CNTL_CCU1 0x83 +#define A5XX_RBBM_CLOCK_CNTL_CCU2 0x84 +#define A5XX_RBBM_CLOCK_CNTL_CCU3 0x85 +#define A5XX_RBBM_CLOCK_HYST_RB_CCU0 0x86 +#define A5XX_RBBM_CLOCK_HYST_RB_CCU1 0x87 +#define A5XX_RBBM_CLOCK_HYST_RB_CCU2 0x88 +#define A5XX_RBBM_CLOCK_HYST_RB_CCU3 0x89 +#define A5XX_RBBM_CLOCK_CNTL_RAC 0x8A +#define A5XX_RBBM_CLOCK_CNTL2_RAC 0x8B +#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0 0x8C +#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1 0x8D +#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2 0x8E +#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3 0x8F +#define A5XX_RBBM_CLOCK_HYST_VFD 0x90 +#define A5XX_RBBM_CLOCK_MODE_VFD 0x91 +#define A5XX_RBBM_CLOCK_DELAY_VFD 0x92 +#define A5XX_RBBM_AHB_CNTL0 0x93 +#define A5XX_RBBM_AHB_CNTL1 0x94 +#define A5XX_RBBM_AHB_CNTL2 0x95 +#define A5XX_RBBM_AHB_CMD 0x96 +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11 0x9C +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12 0x9D +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13 0x9E +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14 0x9F +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15 0xA0 +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16 0xA1 +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17 0xA2 +#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18 0xA3 +#define A5XX_RBBM_CLOCK_DELAY_TP0 0xA4 +#define A5XX_RBBM_CLOCK_DELAY_TP1 0xA5 +#define A5XX_RBBM_CLOCK_DELAY_TP2 0xA6 +#define A5XX_RBBM_CLOCK_DELAY_TP3 0xA7 +#define A5XX_RBBM_CLOCK_DELAY2_TP0 0xA8 +#define A5XX_RBBM_CLOCK_DELAY2_TP1 0xA9 +#define A5XX_RBBM_CLOCK_DELAY2_TP2 0xAA +#define A5XX_RBBM_CLOCK_DELAY2_TP3 0xAB +#define A5XX_RBBM_CLOCK_DELAY3_TP0 0xAC +#define A5XX_RBBM_CLOCK_DELAY3_TP1 0xAD +#define A5XX_RBBM_CLOCK_DELAY3_TP2 0xAE +#define A5XX_RBBM_CLOCK_DELAY3_TP3 0xAF +#define A5XX_RBBM_CLOCK_HYST_TP0 0xB0 +#define A5XX_RBBM_CLOCK_HYST_TP1 0xB1 +#define A5XX_RBBM_CLOCK_HYST_TP2 0xB2 +#define A5XX_RBBM_CLOCK_HYST_TP3 0xB3 +#define A5XX_RBBM_CLOCK_HYST2_TP0 0xB4 +#define A5XX_RBBM_CLOCK_HYST2_TP1 0xB5 +#define A5XX_RBBM_CLOCK_HYST2_TP2 0xB6 +#define A5XX_RBBM_CLOCK_HYST2_TP3 0xB7 +#define A5XX_RBBM_CLOCK_HYST3_TP0 0xB8 +#define A5XX_RBBM_CLOCK_HYST3_TP1 0xB9 +#define A5XX_RBBM_CLOCK_HYST3_TP2 0xBA +#define A5XX_RBBM_CLOCK_HYST3_TP3 0xBB +#define A5XX_RBBM_CLOCK_CNTL_GPMU 0xC8 +#define A5XX_RBBM_CLOCK_DELAY_GPMU 0xC9 +#define A5XX_RBBM_CLOCK_HYST_GPMU 0xCA +#define A5XX_RBBM_PERFCTR_CP_0_LO 0x3A0 +#define A5XX_RBBM_PERFCTR_CP_0_HI 0x3A1 +#define A5XX_RBBM_PERFCTR_CP_1_LO 0x3A2 +#define A5XX_RBBM_PERFCTR_CP_1_HI 0x3A3 +#define A5XX_RBBM_PERFCTR_CP_2_LO 0x3A4 +#define A5XX_RBBM_PERFCTR_CP_2_HI 0x3A5 +#define A5XX_RBBM_PERFCTR_CP_3_LO 0x3A6 +#define A5XX_RBBM_PERFCTR_CP_3_HI 0x3A7 +#define A5XX_RBBM_PERFCTR_CP_4_LO 0x3A8 +#define A5XX_RBBM_PERFCTR_CP_4_HI 0x3A9 +#define A5XX_RBBM_PERFCTR_CP_5_LO 0x3AA +#define A5XX_RBBM_PERFCTR_CP_5_HI 0x3AB +#define A5XX_RBBM_PERFCTR_CP_6_LO 0x3AC +#define A5XX_RBBM_PERFCTR_CP_6_HI 0x3AD +#define A5XX_RBBM_PERFCTR_CP_7_LO 0x3AE +#define A5XX_RBBM_PERFCTR_CP_7_HI 0x3AF +#define A5XX_RBBM_PERFCTR_RBBM_0_LO 0x3B0 +#define A5XX_RBBM_PERFCTR_RBBM_0_HI 0x3B1 +#define A5XX_RBBM_PERFCTR_RBBM_1_LO 0x3B2 +#define A5XX_RBBM_PERFCTR_RBBM_1_HI 0x3B3 +#define A5XX_RBBM_PERFCTR_RBBM_2_LO 0x3B4 +#define A5XX_RBBM_PERFCTR_RBBM_2_HI 0x3B5 +#define A5XX_RBBM_PERFCTR_RBBM_3_LO 0x3B6 +#define A5XX_RBBM_PERFCTR_RBBM_3_HI 0x3B7 +#define A5XX_RBBM_PERFCTR_PC_0_LO 0x3B8 +#define A5XX_RBBM_PERFCTR_PC_0_HI 0x3B9 +#define A5XX_RBBM_PERFCTR_PC_1_LO 0x3BA +#define A5XX_RBBM_PERFCTR_PC_1_HI 0x3BB +#define A5XX_RBBM_PERFCTR_PC_2_LO 0x3BC +#define A5XX_RBBM_PERFCTR_PC_2_HI 0x3BD +#define A5XX_RBBM_PERFCTR_PC_3_LO 0x3BE +#define A5XX_RBBM_PERFCTR_PC_3_HI 0x3BF +#define A5XX_RBBM_PERFCTR_PC_4_LO 0x3C0 +#define A5XX_RBBM_PERFCTR_PC_4_HI 0x3C1 +#define A5XX_RBBM_PERFCTR_PC_5_LO 0x3C2 +#define A5XX_RBBM_PERFCTR_PC_5_HI 0x3C3 +#define A5XX_RBBM_PERFCTR_PC_6_LO 0x3C4 +#define A5XX_RBBM_PERFCTR_PC_6_HI 0x3C5 +#define A5XX_RBBM_PERFCTR_PC_7_LO 0x3C6 +#define A5XX_RBBM_PERFCTR_PC_7_HI 0x3C7 +#define A5XX_RBBM_PERFCTR_VFD_0_LO 0x3C8 +#define A5XX_RBBM_PERFCTR_VFD_0_HI 0x3C9 +#define A5XX_RBBM_PERFCTR_VFD_1_LO 0x3CA +#define A5XX_RBBM_PERFCTR_VFD_1_HI 0x3CB +#define A5XX_RBBM_PERFCTR_VFD_2_LO 0x3CC +#define A5XX_RBBM_PERFCTR_VFD_2_HI 0x3CD +#define A5XX_RBBM_PERFCTR_VFD_3_LO 0x3CE +#define A5XX_RBBM_PERFCTR_VFD_3_HI 0x3CF +#define A5XX_RBBM_PERFCTR_VFD_4_LO 0x3D0 +#define A5XX_RBBM_PERFCTR_VFD_4_HI 0x3D1 +#define A5XX_RBBM_PERFCTR_VFD_5_LO 0x3D2 +#define A5XX_RBBM_PERFCTR_VFD_5_HI 0x3D3 +#define A5XX_RBBM_PERFCTR_VFD_6_LO 0x3D4 +#define A5XX_RBBM_PERFCTR_VFD_6_HI 0x3D5 +#define A5XX_RBBM_PERFCTR_VFD_7_LO 0x3D6 +#define A5XX_RBBM_PERFCTR_VFD_7_HI 0x3D7 +#define A5XX_RBBM_PERFCTR_HLSQ_0_LO 0x3D8 +#define A5XX_RBBM_PERFCTR_HLSQ_0_HI 0x3D9 +#define A5XX_RBBM_PERFCTR_HLSQ_1_LO 0x3DA +#define A5XX_RBBM_PERFCTR_HLSQ_1_HI 0x3DB +#define A5XX_RBBM_PERFCTR_HLSQ_2_LO 0x3DC +#define A5XX_RBBM_PERFCTR_HLSQ_2_HI 0x3DD +#define A5XX_RBBM_PERFCTR_HLSQ_3_LO 0x3DE +#define A5XX_RBBM_PERFCTR_HLSQ_3_HI 0x3DF +#define A5XX_RBBM_PERFCTR_HLSQ_4_LO 0x3E0 +#define A5XX_RBBM_PERFCTR_HLSQ_4_HI 0x3E1 +#define A5XX_RBBM_PERFCTR_HLSQ_5_LO 0x3E2 +#define A5XX_RBBM_PERFCTR_HLSQ_5_HI 0x3E3 +#define A5XX_RBBM_PERFCTR_HLSQ_6_LO 0x3E4 +#define A5XX_RBBM_PERFCTR_HLSQ_6_HI 0x3E5 +#define A5XX_RBBM_PERFCTR_HLSQ_7_LO 0x3E6 +#define A5XX_RBBM_PERFCTR_HLSQ_7_HI 0x3E7 +#define A5XX_RBBM_PERFCTR_VPC_0_LO 0x3E8 +#define A5XX_RBBM_PERFCTR_VPC_0_HI 0x3E9 +#define A5XX_RBBM_PERFCTR_VPC_1_LO 0x3EA +#define A5XX_RBBM_PERFCTR_VPC_1_HI 0x3EB +#define A5XX_RBBM_PERFCTR_VPC_2_LO 0x3EC +#define A5XX_RBBM_PERFCTR_VPC_2_HI 0x3ED +#define A5XX_RBBM_PERFCTR_VPC_3_LO 0x3EE +#define A5XX_RBBM_PERFCTR_VPC_3_HI 0x3EF +#define A5XX_RBBM_PERFCTR_CCU_0_LO 0x3F0 +#define A5XX_RBBM_PERFCTR_CCU_0_HI 0x3F1 +#define A5XX_RBBM_PERFCTR_CCU_1_LO 0x3F2 +#define A5XX_RBBM_PERFCTR_CCU_1_HI 0x3F3 +#define A5XX_RBBM_PERFCTR_CCU_2_LO 0x3F4 +#define A5XX_RBBM_PERFCTR_CCU_2_HI 0x3F5 +#define A5XX_RBBM_PERFCTR_CCU_3_LO 0x3F6 +#define A5XX_RBBM_PERFCTR_CCU_3_HI 0x3F7 +#define A5XX_RBBM_PERFCTR_TSE_0_LO 0x3F8 +#define A5XX_RBBM_PERFCTR_TSE_0_HI 0x3F9 +#define A5XX_RBBM_PERFCTR_TSE_1_LO 0x3FA +#define A5XX_RBBM_PERFCTR_TSE_1_HI 0x3FB +#define A5XX_RBBM_PERFCTR_TSE_2_LO 0x3FC +#define A5XX_RBBM_PERFCTR_TSE_2_HI 0x3FD +#define A5XX_RBBM_PERFCTR_TSE_3_LO 0x3FE +#define A5XX_RBBM_PERFCTR_TSE_3_HI 0x3FF +#define A5XX_RBBM_PERFCTR_RAS_0_LO 0x400 +#define A5XX_RBBM_PERFCTR_RAS_0_HI 0x401 +#define A5XX_RBBM_PERFCTR_RAS_1_LO 0x402 +#define A5XX_RBBM_PERFCTR_RAS_1_HI 0x403 +#define A5XX_RBBM_PERFCTR_RAS_2_LO 0x404 +#define A5XX_RBBM_PERFCTR_RAS_2_HI 0x405 +#define A5XX_RBBM_PERFCTR_RAS_3_LO 0x406 +#define A5XX_RBBM_PERFCTR_RAS_3_HI 0x407 +#define A5XX_RBBM_PERFCTR_UCHE_0_LO 0x408 +#define A5XX_RBBM_PERFCTR_UCHE_0_HI 0x409 +#define A5XX_RBBM_PERFCTR_UCHE_1_LO 0x40A +#define A5XX_RBBM_PERFCTR_UCHE_1_HI 0x40B +#define A5XX_RBBM_PERFCTR_UCHE_2_LO 0x40C +#define A5XX_RBBM_PERFCTR_UCHE_2_HI 0x40D +#define A5XX_RBBM_PERFCTR_UCHE_3_LO 0x40E +#define A5XX_RBBM_PERFCTR_UCHE_3_HI 0x40F +#define A5XX_RBBM_PERFCTR_UCHE_4_LO 0x410 +#define A5XX_RBBM_PERFCTR_UCHE_4_HI 0x411 +#define A5XX_RBBM_PERFCTR_UCHE_5_LO 0x412 +#define A5XX_RBBM_PERFCTR_UCHE_5_HI 0x413 +#define A5XX_RBBM_PERFCTR_UCHE_6_LO 0x414 +#define A5XX_RBBM_PERFCTR_UCHE_6_HI 0x415 +#define A5XX_RBBM_PERFCTR_UCHE_7_LO 0x416 +#define A5XX_RBBM_PERFCTR_UCHE_7_HI 0x417 +#define A5XX_RBBM_PERFCTR_TP_0_LO 0x418 +#define A5XX_RBBM_PERFCTR_TP_0_HI 0x419 +#define A5XX_RBBM_PERFCTR_TP_1_LO 0x41A +#define A5XX_RBBM_PERFCTR_TP_1_HI 0x41B +#define A5XX_RBBM_PERFCTR_TP_2_LO 0x41C +#define A5XX_RBBM_PERFCTR_TP_2_HI 0x41D +#define A5XX_RBBM_PERFCTR_TP_3_LO 0x41E +#define A5XX_RBBM_PERFCTR_TP_3_HI 0x41F +#define A5XX_RBBM_PERFCTR_TP_4_LO 0x420 +#define A5XX_RBBM_PERFCTR_TP_4_HI 0x421 +#define A5XX_RBBM_PERFCTR_TP_5_LO 0x422 +#define A5XX_RBBM_PERFCTR_TP_5_HI 0x423 +#define A5XX_RBBM_PERFCTR_TP_6_LO 0x424 +#define A5XX_RBBM_PERFCTR_TP_6_HI 0x425 +#define A5XX_RBBM_PERFCTR_TP_7_LO 0x426 +#define A5XX_RBBM_PERFCTR_TP_7_HI 0x427 +#define A5XX_RBBM_PERFCTR_SP_0_LO 0x428 +#define A5XX_RBBM_PERFCTR_SP_0_HI 0x429 +#define A5XX_RBBM_PERFCTR_SP_1_LO 0x42A +#define A5XX_RBBM_PERFCTR_SP_1_HI 0x42B +#define A5XX_RBBM_PERFCTR_SP_2_LO 0x42C +#define A5XX_RBBM_PERFCTR_SP_2_HI 0x42D +#define A5XX_RBBM_PERFCTR_SP_3_LO 0x42E +#define A5XX_RBBM_PERFCTR_SP_3_HI 0x42F +#define A5XX_RBBM_PERFCTR_SP_4_LO 0x430 +#define A5XX_RBBM_PERFCTR_SP_4_HI 0x431 +#define A5XX_RBBM_PERFCTR_SP_5_LO 0x432 +#define A5XX_RBBM_PERFCTR_SP_5_HI 0x433 +#define A5XX_RBBM_PERFCTR_SP_6_LO 0x434 +#define A5XX_RBBM_PERFCTR_SP_6_HI 0x435 +#define A5XX_RBBM_PERFCTR_SP_7_LO 0x436 +#define A5XX_RBBM_PERFCTR_SP_7_HI 0x437 +#define A5XX_RBBM_PERFCTR_SP_8_LO 0x438 +#define A5XX_RBBM_PERFCTR_SP_8_HI 0x439 +#define A5XX_RBBM_PERFCTR_SP_9_LO 0x43A +#define A5XX_RBBM_PERFCTR_SP_9_HI 0x43B +#define A5XX_RBBM_PERFCTR_SP_10_LO 0x43C +#define A5XX_RBBM_PERFCTR_SP_10_HI 0x43D +#define A5XX_RBBM_PERFCTR_SP_11_LO 0x43E +#define A5XX_RBBM_PERFCTR_SP_11_HI 0x43F +#define A5XX_RBBM_PERFCTR_RB_0_LO 0x440 +#define A5XX_RBBM_PERFCTR_RB_0_HI 0x441 +#define A5XX_RBBM_PERFCTR_RB_1_LO 0x442 +#define A5XX_RBBM_PERFCTR_RB_1_HI 0x443 +#define A5XX_RBBM_PERFCTR_RB_2_LO 0x444 +#define A5XX_RBBM_PERFCTR_RB_2_HI 0x445 +#define A5XX_RBBM_PERFCTR_RB_3_LO 0x446 +#define A5XX_RBBM_PERFCTR_RB_3_HI 0x447 +#define A5XX_RBBM_PERFCTR_RB_4_LO 0x448 +#define A5XX_RBBM_PERFCTR_RB_4_HI 0x449 +#define A5XX_RBBM_PERFCTR_RB_5_LO 0x44A +#define A5XX_RBBM_PERFCTR_RB_5_HI 0x44B +#define A5XX_RBBM_PERFCTR_RB_6_LO 0x44C +#define A5XX_RBBM_PERFCTR_RB_6_HI 0x44D +#define A5XX_RBBM_PERFCTR_RB_7_LO 0x44E +#define A5XX_RBBM_PERFCTR_RB_7_HI 0x44F +#define A5XX_RBBM_PERFCTR_VSC_0_LO 0x450 +#define A5XX_RBBM_PERFCTR_VSC_0_HI 0x451 +#define A5XX_RBBM_PERFCTR_VSC_1_LO 0x452 +#define A5XX_RBBM_PERFCTR_VSC_1_HI 0x453 +#define A5XX_RBBM_PERFCTR_LRZ_0_LO 0x454 +#define A5XX_RBBM_PERFCTR_LRZ_0_HI 0x455 +#define A5XX_RBBM_PERFCTR_LRZ_1_LO 0x456 +#define A5XX_RBBM_PERFCTR_LRZ_1_HI 0x457 +#define A5XX_RBBM_PERFCTR_LRZ_2_LO 0x458 +#define A5XX_RBBM_PERFCTR_LRZ_2_HI 0x459 +#define A5XX_RBBM_PERFCTR_LRZ_3_LO 0x45A +#define A5XX_RBBM_PERFCTR_LRZ_3_HI 0x45B +#define A5XX_RBBM_PERFCTR_CMP_0_LO 0x45C +#define A5XX_RBBM_PERFCTR_CMP_0_HI 0x45D +#define A5XX_RBBM_PERFCTR_CMP_1_LO 0x45E +#define A5XX_RBBM_PERFCTR_CMP_1_HI 0x45F +#define A5XX_RBBM_PERFCTR_CMP_2_LO 0x460 +#define A5XX_RBBM_PERFCTR_CMP_2_HI 0x461 +#define A5XX_RBBM_PERFCTR_CMP_3_LO 0x462 +#define A5XX_RBBM_PERFCTR_CMP_3_HI 0x463 +#define A5XX_RBBM_PERFCTR_RBBM_SEL_0 0x46B +#define A5XX_RBBM_PERFCTR_RBBM_SEL_1 0x46C +#define A5XX_RBBM_PERFCTR_RBBM_SEL_2 0x46D +#define A5XX_RBBM_PERFCTR_RBBM_SEL_3 0x46E +#define A5XX_RBBM_ALWAYSON_COUNTER_LO 0x4D2 +#define A5XX_RBBM_ALWAYSON_COUNTER_HI 0x4D3 +#define A5XX_RBBM_STATUS 0x4F5 +#define A5XX_RBBM_STATUS3 0x530 +#define A5XX_RBBM_INT_0_STATUS 0x4E1 +#define A5XX_RBBM_AHB_ME_SPLIT_STATUS 0x4F0 +#define A5XX_RBBM_AHB_PFP_SPLIT_STATUS 0x4F1 +#define A5XX_RBBM_AHB_ERROR_STATUS 0x4F4 +#define A5XX_RBBM_PERFCTR_CNTL 0x464 +#define A5XX_RBBM_PERFCTR_LOAD_CMD0 0x465 +#define A5XX_RBBM_PERFCTR_LOAD_CMD1 0x466 +#define A5XX_RBBM_PERFCTR_LOAD_CMD2 0x467 +#define A5XX_RBBM_PERFCTR_LOAD_CMD3 0x468 +#define A5XX_RBBM_PERFCTR_LOAD_VALUE_LO 0x469 +#define A5XX_RBBM_PERFCTR_LOAD_VALUE_HI 0x46A +#define A5XX_RBBM_PERFCTR_RBBM_SEL_0 0x46B +#define A5XX_RBBM_PERFCTR_RBBM_SEL_1 0x46C +#define A5XX_RBBM_PERFCTR_RBBM_SEL_2 0x46D +#define A5XX_RBBM_PERFCTR_RBBM_SEL_3 0x46E +#define A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED 0x46F +#define A5XX_RBBM_CFG_DBGBUS_EVENT_LOGIC 0x504 +#define A5XX_RBBM_CFG_DBGBUS_OVER 0x505 +#define A5XX_RBBM_CFG_DBGBUS_COUNT0 0x506 +#define A5XX_RBBM_CFG_DBGBUS_COUNT1 0x507 +#define A5XX_RBBM_CFG_DBGBUS_COUNT2 0x508 +#define A5XX_RBBM_CFG_DBGBUS_COUNT3 0x509 +#define A5XX_RBBM_CFG_DBGBUS_COUNT4 0x50A +#define A5XX_RBBM_CFG_DBGBUS_COUNT5 0x50B +#define A5XX_RBBM_CFG_DBGBUS_TRACE_ADDR 0x50C +#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF0 0x50D +#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF1 0x50E +#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF2 0x50F +#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF3 0x510 +#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF4 0x511 +#define A5XX_RBBM_CFG_DBGBUS_MISR0 0x512 +#define A5XX_RBBM_CFG_DBGBUS_MISR1 0x513 +#define A5XX_RBBM_ISDB_CNT 0x533 +#define A5XX_RBBM_SECVID_TRUST_CONFIG 0xF000 +#define A5XX_RBBM_SECVID_TRUST_CNTL 0xF400 +#define A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO 0xF800 +#define A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI 0xF801 +#define A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE 0xF802 +#define A5XX_RBBM_SECVID_TSB_CNTL 0xF803 +#define A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL 0xF810 + +/* VSC registers */ +#define A5XX_VSC_PERFCTR_VSC_SEL_0 0xC60 +#define A5XX_VSC_PERFCTR_VSC_SEL_1 0xC61 + +#define A5XX_GRAS_ADDR_MODE_CNTL 0xC81 + +/* TSE registers */ +#define A5XX_GRAS_PERFCTR_TSE_SEL_0 0xC90 +#define A5XX_GRAS_PERFCTR_TSE_SEL_1 0xC91 +#define A5XX_GRAS_PERFCTR_TSE_SEL_2 0xC92 +#define A5XX_GRAS_PERFCTR_TSE_SEL_3 0xC93 + +/* RAS registers */ +#define A5XX_GRAS_PERFCTR_RAS_SEL_0 0xC94 +#define A5XX_GRAS_PERFCTR_RAS_SEL_1 0xC95 +#define A5XX_GRAS_PERFCTR_RAS_SEL_2 0xC96 +#define A5XX_GRAS_PERFCTR_RAS_SEL_3 0xC97 + +/* LRZ registers */ +#define A5XX_GRAS_PERFCTR_LRZ_SEL_0 0xC98 +#define A5XX_GRAS_PERFCTR_LRZ_SEL_1 0xC99 +#define A5XX_GRAS_PERFCTR_LRZ_SEL_2 0xC9A +#define A5XX_GRAS_PERFCTR_LRZ_SEL_3 0xC9B + + +/* RB registers */ +#define A5XX_RB_DBG_ECO_CNT 0xCC4 +#define A5XX_RB_ADDR_MODE_CNTL 0xCC5 +#define A5XX_RB_MODE_CNTL 0xCC6 +#define A5XX_RB_PERFCTR_RB_SEL_0 0xCD0 +#define A5XX_RB_PERFCTR_RB_SEL_1 0xCD1 +#define A5XX_RB_PERFCTR_RB_SEL_2 0xCD2 +#define A5XX_RB_PERFCTR_RB_SEL_3 0xCD3 +#define A5XX_RB_PERFCTR_RB_SEL_4 0xCD4 +#define A5XX_RB_PERFCTR_RB_SEL_5 0xCD5 +#define A5XX_RB_PERFCTR_RB_SEL_6 0xCD6 +#define A5XX_RB_PERFCTR_RB_SEL_7 0xCD7 + +/* CCU registers */ +#define A5XX_RB_PERFCTR_CCU_SEL_0 0xCD8 +#define A5XX_RB_PERFCTR_CCU_SEL_1 0xCD9 +#define A5XX_RB_PERFCTR_CCU_SEL_2 0xCDA +#define A5XX_RB_PERFCTR_CCU_SEL_3 0xCDB + +/* RB Power Counter RB Registers Select */ +#define A5XX_RB_POWERCTR_RB_SEL_0 0xCE0 +#define A5XX_RB_POWERCTR_RB_SEL_1 0xCE1 +#define A5XX_RB_POWERCTR_RB_SEL_2 0xCE2 +#define A5XX_RB_POWERCTR_RB_SEL_3 0xCE3 + +/* RB Power Counter CCU Registers Select */ +#define A5XX_RB_POWERCTR_CCU_SEL_0 0xCE4 +#define A5XX_RB_POWERCTR_CCU_SEL_1 0xCE5 + +/* CMP registers */ +#define A5XX_RB_PERFCTR_CMP_SEL_0 0xCEC +#define A5XX_RB_PERFCTR_CMP_SEL_1 0xCED +#define A5XX_RB_PERFCTR_CMP_SEL_2 0xCEE +#define A5XX_RB_PERFCTR_CMP_SEL_3 0xCEF + +/* PC registers */ +#define A5XX_PC_DBG_ECO_CNTL 0xD00 +#define A5XX_PC_ADDR_MODE_CNTL 0xD01 +#define A5XX_PC_PERFCTR_PC_SEL_0 0xD10 +#define A5XX_PC_PERFCTR_PC_SEL_1 0xD11 +#define A5XX_PC_PERFCTR_PC_SEL_2 0xD12 +#define A5XX_PC_PERFCTR_PC_SEL_3 0xD13 +#define A5XX_PC_PERFCTR_PC_SEL_4 0xD14 +#define A5XX_PC_PERFCTR_PC_SEL_5 0xD15 +#define A5XX_PC_PERFCTR_PC_SEL_6 0xD16 +#define A5XX_PC_PERFCTR_PC_SEL_7 0xD17 + +/* HLSQ registers */ +#define A5XX_HLSQ_DBG_ECO_CNTL 0xE04 +#define A5XX_HLSQ_ADDR_MODE_CNTL 0xE05 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_0 0xE10 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_1 0xE11 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_2 0xE12 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_3 0xE13 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_4 0xE14 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_5 0xE15 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_6 0xE16 +#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_7 0xE17 +#define A5XX_HLSQ_DBG_READ_SEL 0xBC00 +#define A5XX_HLSQ_DBG_AHB_READ_APERTURE 0xA000 + +/* VFD registers */ +#define A5XX_VFD_ADDR_MODE_CNTL 0xE41 +#define A5XX_VFD_PERFCTR_VFD_SEL_0 0xE50 +#define A5XX_VFD_PERFCTR_VFD_SEL_1 0xE51 +#define A5XX_VFD_PERFCTR_VFD_SEL_2 0xE52 +#define A5XX_VFD_PERFCTR_VFD_SEL_3 0xE53 +#define A5XX_VFD_PERFCTR_VFD_SEL_4 0xE54 +#define A5XX_VFD_PERFCTR_VFD_SEL_5 0xE55 +#define A5XX_VFD_PERFCTR_VFD_SEL_6 0xE56 +#define A5XX_VFD_PERFCTR_VFD_SEL_7 0xE57 + +/* VPC registers */ +#define A5XX_VPC_DBG_ECO_CNTL 0xE60 +#define A5XX_VPC_ADDR_MODE_CNTL 0xE61 +#define A5XX_VPC_PERFCTR_VPC_SEL_0 0xE64 +#define A5XX_VPC_PERFCTR_VPC_SEL_1 0xE65 +#define A5XX_VPC_PERFCTR_VPC_SEL_2 0xE66 +#define A5XX_VPC_PERFCTR_VPC_SEL_3 0xE67 + +/* UCHE registers */ +#define A5XX_UCHE_ADDR_MODE_CNTL 0xE80 +#define A5XX_UCHE_MODE_CNTL 0xE81 +#define A5XX_UCHE_WRITE_THRU_BASE_LO 0xE87 +#define A5XX_UCHE_WRITE_THRU_BASE_HI 0xE88 +#define A5XX_UCHE_TRAP_BASE_LO 0xE89 +#define A5XX_UCHE_TRAP_BASE_HI 0xE8A +#define A5XX_UCHE_GMEM_RANGE_MIN_LO 0xE8B +#define A5XX_UCHE_GMEM_RANGE_MIN_HI 0xE8C +#define A5XX_UCHE_GMEM_RANGE_MAX_LO 0xE8D +#define A5XX_UCHE_GMEM_RANGE_MAX_HI 0xE8E +#define A5XX_UCHE_DBG_ECO_CNTL_2 0xE8F +#define A5XX_UCHE_INVALIDATE0 0xE95 +#define A5XX_UCHE_CACHE_WAYS 0xE96 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_0 0xEA0 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_1 0xEA1 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_2 0xEA2 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_3 0xEA3 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_4 0xEA4 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_5 0xEA5 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_6 0xEA6 +#define A5XX_UCHE_PERFCTR_UCHE_SEL_7 0xEA7 + +/* UCHE Power Counter UCHE Registers Select */ +#define A5XX_UCHE_POWERCTR_UCHE_SEL_0 0xEA8 +#define A5XX_UCHE_POWERCTR_UCHE_SEL_1 0xEA9 +#define A5XX_UCHE_POWERCTR_UCHE_SEL_2 0xEAA +#define A5XX_UCHE_POWERCTR_UCHE_SEL_3 0xEAB + +/* SP registers */ +#define A5XX_SP_DBG_ECO_CNTL 0xEC0 +#define A5XX_SP_ADDR_MODE_CNTL 0xEC1 +#define A5XX_SP_PERFCTR_SP_SEL_0 0xED0 +#define A5XX_SP_PERFCTR_SP_SEL_1 0xED1 +#define A5XX_SP_PERFCTR_SP_SEL_2 0xED2 +#define A5XX_SP_PERFCTR_SP_SEL_3 0xED3 +#define A5XX_SP_PERFCTR_SP_SEL_4 0xED4 +#define A5XX_SP_PERFCTR_SP_SEL_5 0xED5 +#define A5XX_SP_PERFCTR_SP_SEL_6 0xED6 +#define A5XX_SP_PERFCTR_SP_SEL_7 0xED7 +#define A5XX_SP_PERFCTR_SP_SEL_8 0xED8 +#define A5XX_SP_PERFCTR_SP_SEL_9 0xED9 +#define A5XX_SP_PERFCTR_SP_SEL_10 0xEDA +#define A5XX_SP_PERFCTR_SP_SEL_11 0xEDB + +/* SP Power Counter SP Registers Select */ +#define A5XX_SP_POWERCTR_SP_SEL_0 0xEDC +#define A5XX_SP_POWERCTR_SP_SEL_1 0xEDD +#define A5XX_SP_POWERCTR_SP_SEL_2 0xEDE +#define A5XX_SP_POWERCTR_SP_SEL_3 0xEDF + +/* TP registers */ +#define A5XX_TPL1_ADDR_MODE_CNTL 0xF01 +#define A5XX_TPL1_MODE_CNTL 0xF02 +#define A5XX_TPL1_PERFCTR_TP_SEL_0 0xF10 +#define A5XX_TPL1_PERFCTR_TP_SEL_1 0xF11 +#define A5XX_TPL1_PERFCTR_TP_SEL_2 0xF12 +#define A5XX_TPL1_PERFCTR_TP_SEL_3 0xF13 +#define A5XX_TPL1_PERFCTR_TP_SEL_4 0xF14 +#define A5XX_TPL1_PERFCTR_TP_SEL_5 0xF15 +#define A5XX_TPL1_PERFCTR_TP_SEL_6 0xF16 +#define A5XX_TPL1_PERFCTR_TP_SEL_7 0xF17 + +/* TP Power Counter TP Registers Select */ +#define A5XX_TPL1_POWERCTR_TP_SEL_0 0xF18 +#define A5XX_TPL1_POWERCTR_TP_SEL_1 0xF19 +#define A5XX_TPL1_POWERCTR_TP_SEL_2 0xF1A +#define A5XX_TPL1_POWERCTR_TP_SEL_3 0xF1B + +/* VBIF registers */ +#define A5XX_VBIF_VERSION 0x3000 +#define A5XX_VBIF_CLKON 0x3001 +#define A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_MASK 0x1 +#define A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_SHIFT 0x1 + +#define A5XX_VBIF_ROUND_ROBIN_QOS_ARB 0x3049 +#define A5XX_VBIF_GATE_OFF_WRREQ_EN 0x302A + +#define A5XX_VBIF_XIN_HALT_CTRL0 0x3080 +#define A5XX_VBIF_XIN_HALT_CTRL0_MASK 0xF +#define A510_VBIF_XIN_HALT_CTRL0_MASK 0x7 +#define A5XX_VBIF_XIN_HALT_CTRL1 0x3081 + +#define A5XX_VBIF_TEST_BUS_OUT_CTRL 0x3084 +#define A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_MASK 0x1 +#define A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_SHIFT 0x0 + +#define A5XX_VBIF_TEST_BUS1_CTRL0 0x3085 +#define A5XX_VBIF_TEST_BUS1_CTRL1 0x3086 +#define A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_MASK 0xF +#define A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_SHIFT 0x0 + +#define A5XX_VBIF_TEST_BUS2_CTRL0 0x3087 +#define A5XX_VBIF_TEST_BUS2_CTRL1 0x3088 +#define A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK 0x1FF +#define A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT 0x0 + +#define A5XX_VBIF_TEST_BUS_OUT 0x308c + +#define A5XX_VBIF_PERF_CNT_SEL0 0x30D0 +#define A5XX_VBIF_PERF_CNT_SEL1 0x30D1 +#define A5XX_VBIF_PERF_CNT_SEL2 0x30D2 +#define A5XX_VBIF_PERF_CNT_SEL3 0x30D3 +#define A5XX_VBIF_PERF_CNT_LOW0 0x30D8 +#define A5XX_VBIF_PERF_CNT_LOW1 0x30D9 +#define A5XX_VBIF_PERF_CNT_LOW2 0x30DA +#define A5XX_VBIF_PERF_CNT_LOW3 0x30DB +#define A5XX_VBIF_PERF_CNT_HIGH0 0x30E0 +#define A5XX_VBIF_PERF_CNT_HIGH1 0x30E1 +#define A5XX_VBIF_PERF_CNT_HIGH2 0x30E2 +#define A5XX_VBIF_PERF_CNT_HIGH3 0x30E3 + +#define A5XX_VBIF_PERF_PWR_CNT_EN0 0x3100 +#define A5XX_VBIF_PERF_PWR_CNT_EN1 0x3101 +#define A5XX_VBIF_PERF_PWR_CNT_EN2 0x3102 + +#define A5XX_VBIF_PERF_PWR_CNT_LOW0 0x3110 +#define A5XX_VBIF_PERF_PWR_CNT_LOW1 0x3111 +#define A5XX_VBIF_PERF_PWR_CNT_LOW2 0x3112 + +#define A5XX_VBIF_PERF_PWR_CNT_HIGH0 0x3118 +#define A5XX_VBIF_PERF_PWR_CNT_HIGH1 0x3119 +#define A5XX_VBIF_PERF_PWR_CNT_HIGH2 0x311A + +/* GPMU registers */ +#define A5XX_GPMU_INST_RAM_BASE 0x8800 +#define A5XX_GPMU_DATA_RAM_BASE 0x9800 +#define A5XX_GPMU_SP_POWER_CNTL 0xA881 +#define A5XX_GPMU_RBCCU_CLOCK_CNTL 0xA886 +#define A5XX_GPMU_RBCCU_POWER_CNTL 0xA887 +#define A5XX_GPMU_SP_PWR_CLK_STATUS 0xA88B +#define A5XX_GPMU_RBCCU_PWR_CLK_STATUS 0xA88D +#define A5XX_GPMU_PWR_COL_STAGGER_DELAY 0xA891 +#define A5XX_GPMU_PWR_COL_INTER_FRAME_CTRL 0xA892 +#define A5XX_GPMU_PWR_COL_INTER_FRAME_HYST 0xA893 +#define A5XX_GPMU_PWR_COL_BINNING_CTRL 0xA894 +#define A5XX_GPMU_CLOCK_THROTTLE_CTRL 0xA8A3 +#define A5XX_GPMU_WFI_CONFIG 0xA8C1 +#define A5XX_GPMU_RBBM_INTR_INFO 0xA8D6 +#define A5XX_GPMU_CM3_SYSRESET 0xA8D8 +#define A5XX_GPMU_GENERAL_0 0xA8E0 +#define A5XX_GPMU_GENERAL_1 0xA8E1 + +/* COUNTABLE FOR SP PERFCOUNTER */ +#define A5XX_SP_ALU_ACTIVE_CYCLES 0x1 +#define A5XX_SP0_ICL1_MISSES 0x35 +#define A5XX_SP_FS_CFLOW_INSTRUCTIONS 0x27 + +/* COUNTABLE FOR TSE PERFCOUNTER */ +#define A5XX_TSE_INPUT_PRIM_NUM 0x6 + +/* COUNTABLE FOR RBBM PERFCOUNTER */ +#define A5XX_RBBM_ALWAYS_COUNT 0x0 + +/* GPMU POWER COUNTERS */ +#define A5XX_SP_POWER_COUNTER_0_LO 0xA840 +#define A5XX_SP_POWER_COUNTER_0_HI 0xA841 +#define A5XX_SP_POWER_COUNTER_1_LO 0xA842 +#define A5XX_SP_POWER_COUNTER_1_HI 0xA843 +#define A5XX_SP_POWER_COUNTER_2_LO 0xA844 +#define A5XX_SP_POWER_COUNTER_2_HI 0xA845 +#define A5XX_SP_POWER_COUNTER_3_LO 0xA846 +#define A5XX_SP_POWER_COUNTER_3_HI 0xA847 + +#define A5XX_TP_POWER_COUNTER_0_LO 0xA848 +#define A5XX_TP_POWER_COUNTER_0_HI 0xA849 +#define A5XX_TP_POWER_COUNTER_1_LO 0xA84A +#define A5XX_TP_POWER_COUNTER_1_HI 0xA84B +#define A5XX_TP_POWER_COUNTER_2_LO 0xA84C +#define A5XX_TP_POWER_COUNTER_2_HI 0xA84D +#define A5XX_TP_POWER_COUNTER_3_LO 0xA84E +#define A5XX_TP_POWER_COUNTER_3_HI 0xA84F + +#define A5XX_RB_POWER_COUNTER_0_LO 0xA850 +#define A5XX_RB_POWER_COUNTER_0_HI 0xA851 +#define A5XX_RB_POWER_COUNTER_1_LO 0xA852 +#define A5XX_RB_POWER_COUNTER_1_HI 0xA853 +#define A5XX_RB_POWER_COUNTER_2_LO 0xA854 +#define A5XX_RB_POWER_COUNTER_2_HI 0xA855 +#define A5XX_RB_POWER_COUNTER_3_LO 0xA856 +#define A5XX_RB_POWER_COUNTER_3_HI 0xA857 + +#define A5XX_CCU_POWER_COUNTER_0_LO 0xA858 +#define A5XX_CCU_POWER_COUNTER_0_HI 0xA859 +#define A5XX_CCU_POWER_COUNTER_1_LO 0xA85A +#define A5XX_CCU_POWER_COUNTER_1_HI 0xA85B + +#define A5XX_UCHE_POWER_COUNTER_0_LO 0xA85C +#define A5XX_UCHE_POWER_COUNTER_0_HI 0xA85D +#define A5XX_UCHE_POWER_COUNTER_1_LO 0xA85E +#define A5XX_UCHE_POWER_COUNTER_1_HI 0xA85F +#define A5XX_UCHE_POWER_COUNTER_2_LO 0xA860 +#define A5XX_UCHE_POWER_COUNTER_2_HI 0xA861 +#define A5XX_UCHE_POWER_COUNTER_3_LO 0xA862 +#define A5XX_UCHE_POWER_COUNTER_3_HI 0xA863 + +#define A5XX_CP_POWER_COUNTER_0_LO 0xA864 +#define A5XX_CP_POWER_COUNTER_0_HI 0xA865 +#define A5XX_CP_POWER_COUNTER_1_LO 0xA866 +#define A5XX_CP_POWER_COUNTER_1_HI 0xA867 +#define A5XX_CP_POWER_COUNTER_2_LO 0xA868 +#define A5XX_CP_POWER_COUNTER_2_HI 0xA869 +#define A5XX_CP_POWER_COUNTER_3_LO 0xA86A +#define A5XX_CP_POWER_COUNTER_3_HI 0xA86B + +#define A5XX_GPMU_POWER_COUNTER_0_LO 0xA86C +#define A5XX_GPMU_POWER_COUNTER_0_HI 0xA86D +#define A5XX_GPMU_POWER_COUNTER_1_LO 0xA86E +#define A5XX_GPMU_POWER_COUNTER_1_HI 0xA86F +#define A5XX_GPMU_POWER_COUNTER_2_LO 0xA870 +#define A5XX_GPMU_POWER_COUNTER_2_HI 0xA871 +#define A5XX_GPMU_POWER_COUNTER_3_LO 0xA872 +#define A5XX_GPMU_POWER_COUNTER_3_HI 0xA873 +#define A5XX_GPMU_POWER_COUNTER_4_LO 0xA874 +#define A5XX_GPMU_POWER_COUNTER_4_HI 0xA875 +#define A5XX_GPMU_POWER_COUNTER_5_LO 0xA876 +#define A5XX_GPMU_POWER_COUNTER_5_HI 0xA877 + +#define A5XX_GPMU_POWER_COUNTER_ENABLE 0xA878 +#define A5XX_GPMU_ALWAYS_ON_COUNTER_LO 0xA879 +#define A5XX_GPMU_ALWAYS_ON_COUNTER_HI 0xA87A +#define A5XX_GPMU_ALWAYS_ON_COUNTER_RESET 0xA87B +#define A5XX_GPMU_POWER_COUNTER_SELECT_0 0xA87C +#define A5XX_GPMU_POWER_COUNTER_SELECT_1 0xA87D +#define A5XX_GPMU_GPMU_SP_CLOCK_CONTROL 0xA880 + +#define A5XX_GPMU_CLOCK_THROTTLE_CTRL 0xA8A3 +#define A5XX_GPMU_THROTTLE_UNMASK_FORCE_CTRL 0xA8A8 + +#define A5XX_GPMU_TEMP_SENSOR_ID 0xAC00 +#define A5XX_GPMU_TEMP_SENSOR_CONFIG 0xAC01 +#define A5XX_GPMU_DELTA_TEMP_THRESHOLD 0xAC03 +#define A5XX_GPMU_TEMP_THRESHOLD_INTR_EN_MASK 0xAC06 + +#define A5XX_GPMU_LEAKAGE_TEMP_COEFF_0_1 0xAC40 +#define A5XX_GPMU_LEAKAGE_TEMP_COEFF_2_3 0xAC41 +#define A5XX_GPMU_LEAKAGE_VTG_COEFF_0_1 0xAC42 +#define A5XX_GPMU_LEAKAGE_VTG_COEFF_2_3 0xAC43 +#define A5XX_GPMU_BASE_LEAKAGE 0xAC46 + +#define A5XX_GPMU_GPMU_VOLTAGE 0xAC60 +#define A5XX_GPMU_GPMU_VOLTAGE_INTR_STATUS 0xAC61 +#define A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK 0xAC62 +#define A5XX_GPMU_GPMU_PWR_THRESHOLD 0xAC80 +#define A5XX_GPMU_GPMU_LLM_GLM_SLEEP_CTRL 0xACC4 +#define A5XX_GPMU_GPMU_LLM_GLM_SLEEP_STATUS 0xACC5 +#define A5XX_GPMU_GPMU_ISENSE_CTRL 0xACD0 + +#define A5XX_GDPM_CONFIG1 0xB80C +#define A5XX_GDPM_INT_EN 0xB80F +#define A5XX_GDPM_INT_MASK 0xB811 +#define A5XX_GPMU_BEC_ENABLE 0xB9A0 + +/* ISENSE registers */ +#define A5XX_GPU_CS_DECIMAL_ALIGN 0xC16A +#define A5XX_GPU_CS_SENSOR_PARAM_CORE_1 0xC126 +#define A5XX_GPU_CS_SENSOR_PARAM_CORE_2 0xC127 +#define A5XX_GPU_CS_SW_OV_FUSE_EN 0xC168 +#define A5XX_GPU_CS_SENSOR_GENERAL_STATUS 0xC41A +#define A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_0 0xC41D +#define A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_2 0xC41F +#define A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_4 0xC421 +#define A5XX_GPU_CS_ENABLE_REG 0xC520 +#define A5XX_GPU_CS_AMP_CALIBRATION_CONTROL1 0xC557 +#define A5XX_GPU_CS_AMP_CALIBRATION_DONE 0xC565 +#define A5XX_GPU_CS_ENDPOINT_CALIBRATION_DONE 0xC556 +#endif /* _A5XX_REG_H */ + diff --git a/qcom/opensource/graphics-kernel/a6xx_reg.h b/qcom/opensource/graphics-kernel/a6xx_reg.h new file mode 100644 index 0000000000..0fabecbf8a --- /dev/null +++ b/qcom/opensource/graphics-kernel/a6xx_reg.h @@ -0,0 +1,1242 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2017-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _A6XX_REG_H +#define _A6XX_REG_H + +/* A6XX interrupt bits */ +#define A6XX_INT_RBBM_GPU_IDLE 0 +#define A6XX_INT_CP_AHB_ERROR 1 +#define A6XX_INT_ATB_ASYNCFIFO_OVERFLOW 6 +#define A6XX_INT_RBBM_GPC_ERROR 7 +#define A6XX_INT_CP_SW 8 +#define A6XX_INT_CP_HW_ERROR 9 +#define A6XX_INT_CP_CCU_FLUSH_DEPTH_TS 10 +#define A6XX_INT_CP_CCU_FLUSH_COLOR_TS 11 +#define A6XX_INT_CP_CCU_RESOLVE_TS 12 +#define A6XX_INT_CP_IB2 13 +#define A6XX_INT_CP_IB1 14 +#define A6XX_INT_CP_RB 15 +#define A6XX_INT_CP_RB_DONE_TS 17 +#define A6XX_INT_CP_WT_DONE_TS 18 +#define A6XX_INT_CP_CACHE_FLUSH_TS 20 +#define A6XX_INT_RBBM_ATB_BUS_OVERFLOW 22 +#define A6XX_INT_RBBM_HANG_DETECT 23 +#define A6XX_INT_UCHE_OOB_ACCESS 24 +#define A6XX_INT_UCHE_TRAP_INTR 25 +#define A6XX_INT_DEBBUS_INTR_0 26 +#define A6XX_INT_DEBBUS_INTR_1 27 +#define A6XX_INT_TSB_WRITE_ERROR 28 +#define A6XX_INT_ISDB_CPU_IRQ 30 +#define A6XX_INT_ISDB_UNDER_DEBUG 31 + +/* CP Interrupt bits */ +#define A6XX_CP_OPCODE_ERROR 0 +#define A6XX_CP_UCODE_ERROR 1 +#define A6XX_CP_HW_FAULT_ERROR 2 +#define A6XX_CP_REGISTER_PROTECTION_ERROR 4 +#define A6XX_CP_AHB_ERROR 5 +#define A6XX_CP_VSD_PARITY_ERROR 6 +#define A6XX_CP_ILLEGAL_INSTR_ERROR 7 + +/* CP registers */ +#define A6XX_CP_RB_BASE 0x800 +#define A6XX_CP_RB_BASE_HI 0x801 +#define A6XX_CP_RB_CNTL 0x802 +#define A6XX_CP_RB_RPTR_ADDR_LO 0x804 +#define A6XX_CP_RB_RPTR_ADDR_HI 0x805 +#define A6XX_CP_RB_RPTR 0x806 +#define A6XX_CP_RB_WPTR 0x807 +#define A6XX_CP_SQE_CNTL 0x808 +#define A6XX_CP_CP2GMU_STATUS 0x812 +#define A6XX_CP_HW_FAULT 0x821 +#define A6XX_CP_INTERRUPT_STATUS 0x823 +#define A6XX_CP_PROTECT_STATUS 0x824 +#define A6XX_CP_STATUS_1 0x825 +#define A6XX_CP_SQE_INSTR_BASE_LO 0x830 +#define A6XX_CP_SQE_INSTR_BASE_HI 0x831 +#define A6XX_CP_MISC_CNTL 0x840 +#define A6XX_CP_APRIV_CNTL 0X844 +#define A6XX_CP_ROQ_THRESHOLDS_1 0x8C1 +#define A6XX_CP_ROQ_THRESHOLDS_2 0x8C2 +#define A6XX_CP_MEM_POOL_SIZE 0x8C3 +#define A6XX_CP_CHICKEN_DBG 0x841 +#define A6XX_CP_ADDR_MODE_CNTL 0x842 +#define A6XX_CP_DBG_ECO_CNTL 0x843 +#define A6XX_CP_PROTECT_CNTL 0x84F +#define A6XX_CP_PROTECT_REG 0x850 +#define A6XX_CP_CONTEXT_SWITCH_CNTL 0x8A0 +#define A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO 0x8A1 +#define A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI 0x8A2 +#define A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO 0x8A3 +#define A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI 0x8A4 +#define A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO 0x8A5 +#define A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI 0x8A6 +#define A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO 0x8A7 +#define A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI 0x8A8 +#define A6XX_CP_CONTEXT_SWITCH_LEVEL_STATUS 0x8AB +#define A6XX_CP_PERFCTR_CP_SEL_0 0x8D0 +#define A6XX_CP_PERFCTR_CP_SEL_1 0x8D1 +#define A6XX_CP_PERFCTR_CP_SEL_2 0x8D2 +#define A6XX_CP_PERFCTR_CP_SEL_3 0x8D3 +#define A6XX_CP_PERFCTR_CP_SEL_4 0x8D4 +#define A6XX_CP_PERFCTR_CP_SEL_5 0x8D5 +#define A6XX_CP_PERFCTR_CP_SEL_6 0x8D6 +#define A6XX_CP_PERFCTR_CP_SEL_7 0x8D7 +#define A6XX_CP_PERFCTR_CP_SEL_8 0x8D8 +#define A6XX_CP_PERFCTR_CP_SEL_9 0x8D9 +#define A6XX_CP_PERFCTR_CP_SEL_10 0x8DA +#define A6XX_CP_PERFCTR_CP_SEL_11 0x8DB +#define A6XX_CP_PERFCTR_CP_SEL_12 0x8DC +#define A6XX_CP_PERFCTR_CP_SEL_13 0x8DD +#define A6XX_CP_CRASH_SCRIPT_BASE_LO 0x900 +#define A6XX_CP_CRASH_SCRIPT_BASE_HI 0x901 +#define A6XX_CP_CRASH_DUMP_CNTL 0x902 +#define A6XX_CP_CRASH_DUMP_STATUS 0x903 +#define A6XX_CP_SQE_STAT_ADDR 0x908 +#define A6XX_CP_SQE_STAT_DATA 0x909 +#define A6XX_CP_DRAW_STATE_ADDR 0x90A +#define A6XX_CP_DRAW_STATE_DATA 0x90B +#define A6XX_CP_ROQ_DBG_ADDR 0x90C +#define A6XX_CP_ROQ_DBG_DATA 0x90D +#define A6XX_CP_MEM_POOL_DBG_ADDR 0x90E +#define A6XX_CP_MEM_POOL_DBG_DATA 0x90F +#define A6XX_CP_SQE_UCODE_DBG_ADDR 0x910 +#define A6XX_CP_SQE_UCODE_DBG_DATA 0x911 +#define A6XX_CP_IB1_BASE 0x928 +#define A6XX_CP_IB1_BASE_HI 0x929 +#define A6XX_CP_IB1_REM_SIZE 0x92A +#define A6XX_CP_IB2_BASE 0x92B +#define A6XX_CP_IB2_BASE_HI 0x92C +#define A6XX_CP_IB2_REM_SIZE 0x92D +#define A6XX_CP_ALWAYS_ON_COUNTER_LO 0x980 +#define A6XX_CP_ALWAYS_ON_COUNTER_HI 0x981 +#define A6XX_CP_ALWAYS_ON_CONTEXT_LO 0x982 +#define A6XX_CP_ALWAYS_ON_CONTEXT_HI 0x983 +#define A6XX_CP_AHB_CNTL 0x98D +#define A6XX_CP_APERTURE_CNTL_HOST 0xA00 +#define A6XX_CP_APERTURE_CNTL_CD 0xA03 +#define A6XX_VSC_ADDR_MODE_CNTL 0xC01 + +/* LPAC registers */ +#define A6XX_CP_LPAC_DRAW_STATE_ADDR 0xB0A +#define A6XX_CP_LPAC_DRAW_STATE_DATA 0xB0B +#define A6XX_CP_LPAC_ROQ_DBG_ADDR 0xB0C +#define A6XX_CP_SQE_AC_UCODE_DBG_ADDR 0xB27 +#define A6XX_CP_SQE_AC_UCODE_DBG_DATA 0xB28 +#define A6XX_CP_SQE_AC_STAT_ADDR 0xB29 +#define A6XX_CP_SQE_AC_STAT_DATA 0xB2A +#define A6XX_CP_LPAC_ROQ_THRESHOLDS_1 0xB32 +#define A6XX_CP_LPAC_ROQ_THRESHOLDS_2 0xB33 +#define A6XX_CP_LPAC_PROG_FIFO_SIZE 0xB34 +#define A6XX_CP_LPAC_ROQ_DBG_DATA 0xB35 +#define A6XX_CP_LPAC_FIFO_DBG_DATA 0xB36 +#define A6XX_CP_LPAC_FIFO_DBG_ADDR 0xB40 + +/* RBBM registers */ +#define A6XX_RBBM_INT_0_STATUS 0x201 +#define A6XX_RBBM_STATUS 0x210 +#define A6XX_RBBM_STATUS3 0x213 +#define A6XX_RBBM_VBIF_GX_RESET_STATUS 0x215 +#define A6XX_RBBM_PERFCTR_CP_0_LO 0x400 +#define A6XX_RBBM_PERFCTR_CP_0_HI 0x401 +#define A6XX_RBBM_PERFCTR_CP_1_LO 0x402 +#define A6XX_RBBM_PERFCTR_CP_1_HI 0x403 +#define A6XX_RBBM_PERFCTR_CP_2_LO 0x404 +#define A6XX_RBBM_PERFCTR_CP_2_HI 0x405 +#define A6XX_RBBM_PERFCTR_CP_3_LO 0x406 +#define A6XX_RBBM_PERFCTR_CP_3_HI 0x407 +#define A6XX_RBBM_PERFCTR_CP_4_LO 0x408 +#define A6XX_RBBM_PERFCTR_CP_4_HI 0x409 +#define A6XX_RBBM_PERFCTR_CP_5_LO 0x40a +#define A6XX_RBBM_PERFCTR_CP_5_HI 0x40b +#define A6XX_RBBM_PERFCTR_CP_6_LO 0x40c +#define A6XX_RBBM_PERFCTR_CP_6_HI 0x40d +#define A6XX_RBBM_PERFCTR_CP_7_LO 0x40e +#define A6XX_RBBM_PERFCTR_CP_7_HI 0x40f +#define A6XX_RBBM_PERFCTR_CP_8_LO 0x410 +#define A6XX_RBBM_PERFCTR_CP_8_HI 0x411 +#define A6XX_RBBM_PERFCTR_CP_9_LO 0x412 +#define A6XX_RBBM_PERFCTR_CP_9_HI 0x413 +#define A6XX_RBBM_PERFCTR_CP_10_LO 0x414 +#define A6XX_RBBM_PERFCTR_CP_10_HI 0x415 +#define A6XX_RBBM_PERFCTR_CP_11_LO 0x416 +#define A6XX_RBBM_PERFCTR_CP_11_HI 0x417 +#define A6XX_RBBM_PERFCTR_CP_12_LO 0x418 +#define A6XX_RBBM_PERFCTR_CP_12_HI 0x419 +#define A6XX_RBBM_PERFCTR_CP_13_LO 0x41a +#define A6XX_RBBM_PERFCTR_CP_13_HI 0x41b +#define A6XX_RBBM_PERFCTR_RBBM_0_LO 0x41c +#define A6XX_RBBM_PERFCTR_RBBM_0_HI 0x41d +#define A6XX_RBBM_PERFCTR_RBBM_1_LO 0x41e +#define A6XX_RBBM_PERFCTR_RBBM_1_HI 0x41f +#define A6XX_RBBM_PERFCTR_RBBM_2_LO 0x420 +#define A6XX_RBBM_PERFCTR_RBBM_2_HI 0x421 +#define A6XX_RBBM_PERFCTR_RBBM_3_LO 0x422 +#define A6XX_RBBM_PERFCTR_RBBM_3_HI 0x423 +#define A6XX_RBBM_PERFCTR_PC_0_LO 0x424 +#define A6XX_RBBM_PERFCTR_PC_0_HI 0x425 +#define A6XX_RBBM_PERFCTR_PC_1_LO 0x426 +#define A6XX_RBBM_PERFCTR_PC_1_HI 0x427 +#define A6XX_RBBM_PERFCTR_PC_2_LO 0x428 +#define A6XX_RBBM_PERFCTR_PC_2_HI 0x429 +#define A6XX_RBBM_PERFCTR_PC_3_LO 0x42a +#define A6XX_RBBM_PERFCTR_PC_3_HI 0x42b +#define A6XX_RBBM_PERFCTR_PC_4_LO 0x42c +#define A6XX_RBBM_PERFCTR_PC_4_HI 0x42d +#define A6XX_RBBM_PERFCTR_PC_5_LO 0x42e +#define A6XX_RBBM_PERFCTR_PC_5_HI 0x42f +#define A6XX_RBBM_PERFCTR_PC_6_LO 0x430 +#define A6XX_RBBM_PERFCTR_PC_6_HI 0x431 +#define A6XX_RBBM_PERFCTR_PC_7_LO 0x432 +#define A6XX_RBBM_PERFCTR_PC_7_HI 0x433 +#define A6XX_RBBM_PERFCTR_VFD_0_LO 0x434 +#define A6XX_RBBM_PERFCTR_VFD_0_HI 0x435 +#define A6XX_RBBM_PERFCTR_VFD_1_LO 0x436 +#define A6XX_RBBM_PERFCTR_VFD_1_HI 0x437 +#define A6XX_RBBM_PERFCTR_VFD_2_LO 0x438 +#define A6XX_RBBM_PERFCTR_VFD_2_HI 0x439 +#define A6XX_RBBM_PERFCTR_VFD_3_LO 0x43a +#define A6XX_RBBM_PERFCTR_VFD_3_HI 0x43b +#define A6XX_RBBM_PERFCTR_VFD_4_LO 0x43c +#define A6XX_RBBM_PERFCTR_VFD_4_HI 0x43d +#define A6XX_RBBM_PERFCTR_VFD_5_LO 0x43e +#define A6XX_RBBM_PERFCTR_VFD_5_HI 0x43f +#define A6XX_RBBM_PERFCTR_VFD_6_LO 0x440 +#define A6XX_RBBM_PERFCTR_VFD_6_HI 0x441 +#define A6XX_RBBM_PERFCTR_VFD_7_LO 0x442 +#define A6XX_RBBM_PERFCTR_VFD_7_HI 0x443 +#define A6XX_RBBM_PERFCTR_HLSQ_0_LO 0x444 +#define A6XX_RBBM_PERFCTR_HLSQ_0_HI 0x445 +#define A6XX_RBBM_PERFCTR_HLSQ_1_LO 0x446 +#define A6XX_RBBM_PERFCTR_HLSQ_1_HI 0x447 +#define A6XX_RBBM_PERFCTR_HLSQ_2_LO 0x448 +#define A6XX_RBBM_PERFCTR_HLSQ_2_HI 0x449 +#define A6XX_RBBM_PERFCTR_HLSQ_3_LO 0x44a +#define A6XX_RBBM_PERFCTR_HLSQ_3_HI 0x44b +#define A6XX_RBBM_PERFCTR_HLSQ_4_LO 0x44c +#define A6XX_RBBM_PERFCTR_HLSQ_4_HI 0x44d +#define A6XX_RBBM_PERFCTR_HLSQ_5_LO 0x44e +#define A6XX_RBBM_PERFCTR_HLSQ_5_HI 0x44f +#define A6XX_RBBM_PERFCTR_VPC_0_LO 0x450 +#define A6XX_RBBM_PERFCTR_VPC_0_HI 0x451 +#define A6XX_RBBM_PERFCTR_VPC_1_LO 0x452 +#define A6XX_RBBM_PERFCTR_VPC_1_HI 0x453 +#define A6XX_RBBM_PERFCTR_VPC_2_LO 0x454 +#define A6XX_RBBM_PERFCTR_VPC_2_HI 0x455 +#define A6XX_RBBM_PERFCTR_VPC_3_LO 0x456 +#define A6XX_RBBM_PERFCTR_VPC_3_HI 0x457 +#define A6XX_RBBM_PERFCTR_VPC_4_LO 0x458 +#define A6XX_RBBM_PERFCTR_VPC_4_HI 0x459 +#define A6XX_RBBM_PERFCTR_VPC_5_LO 0x45a +#define A6XX_RBBM_PERFCTR_VPC_5_HI 0x45b +#define A6XX_RBBM_PERFCTR_CCU_0_LO 0x45c +#define A6XX_RBBM_PERFCTR_CCU_0_HI 0x45d +#define A6XX_RBBM_PERFCTR_CCU_1_LO 0x45e +#define A6XX_RBBM_PERFCTR_CCU_1_HI 0x45f +#define A6XX_RBBM_PERFCTR_CCU_2_LO 0x460 +#define A6XX_RBBM_PERFCTR_CCU_2_HI 0x461 +#define A6XX_RBBM_PERFCTR_CCU_3_LO 0x462 +#define A6XX_RBBM_PERFCTR_CCU_3_HI 0x463 +#define A6XX_RBBM_PERFCTR_CCU_4_LO 0x464 +#define A6XX_RBBM_PERFCTR_CCU_4_HI 0x465 +#define A6XX_RBBM_PERFCTR_TSE_0_LO 0x466 +#define A6XX_RBBM_PERFCTR_TSE_0_HI 0x467 +#define A6XX_RBBM_PERFCTR_TSE_1_LO 0x468 +#define A6XX_RBBM_PERFCTR_TSE_1_HI 0x469 +#define A6XX_RBBM_PERFCTR_TSE_2_LO 0x46a +#define A6XX_RBBM_PERFCTR_CCU_4_HI 0x465 +#define A6XX_RBBM_PERFCTR_TSE_0_LO 0x466 +#define A6XX_RBBM_PERFCTR_TSE_0_HI 0x467 +#define A6XX_RBBM_PERFCTR_TSE_1_LO 0x468 +#define A6XX_RBBM_PERFCTR_TSE_1_HI 0x469 +#define A6XX_RBBM_PERFCTR_TSE_2_LO 0x46a +#define A6XX_RBBM_PERFCTR_TSE_2_HI 0x46b +#define A6XX_RBBM_PERFCTR_TSE_3_LO 0x46c +#define A6XX_RBBM_PERFCTR_TSE_3_HI 0x46d +#define A6XX_RBBM_PERFCTR_RAS_0_LO 0x46e +#define A6XX_RBBM_PERFCTR_RAS_0_HI 0x46f +#define A6XX_RBBM_PERFCTR_RAS_1_LO 0x470 +#define A6XX_RBBM_PERFCTR_RAS_1_HI 0x471 +#define A6XX_RBBM_PERFCTR_RAS_2_LO 0x472 +#define A6XX_RBBM_PERFCTR_RAS_2_HI 0x473 +#define A6XX_RBBM_PERFCTR_RAS_3_LO 0x474 +#define A6XX_RBBM_PERFCTR_RAS_3_HI 0x475 +#define A6XX_RBBM_PERFCTR_UCHE_0_LO 0x476 +#define A6XX_RBBM_PERFCTR_UCHE_0_HI 0x477 +#define A6XX_RBBM_PERFCTR_UCHE_1_LO 0x478 +#define A6XX_RBBM_PERFCTR_UCHE_1_HI 0x479 +#define A6XX_RBBM_PERFCTR_UCHE_2_LO 0x47a +#define A6XX_RBBM_PERFCTR_UCHE_2_HI 0x47b +#define A6XX_RBBM_PERFCTR_UCHE_3_LO 0x47c +#define A6XX_RBBM_PERFCTR_UCHE_3_HI 0x47d +#define A6XX_RBBM_PERFCTR_UCHE_4_LO 0x47e +#define A6XX_RBBM_PERFCTR_UCHE_4_HI 0x47f +#define A6XX_RBBM_PERFCTR_UCHE_5_LO 0x480 +#define A6XX_RBBM_PERFCTR_UCHE_5_HI 0x481 +#define A6XX_RBBM_PERFCTR_UCHE_6_LO 0x482 +#define A6XX_RBBM_PERFCTR_UCHE_6_HI 0x483 +#define A6XX_RBBM_PERFCTR_UCHE_7_LO 0x484 +#define A6XX_RBBM_PERFCTR_UCHE_7_HI 0x485 +#define A6XX_RBBM_PERFCTR_UCHE_8_LO 0x486 +#define A6XX_RBBM_PERFCTR_UCHE_8_HI 0x487 +#define A6XX_RBBM_PERFCTR_UCHE_9_LO 0x488 +#define A6XX_RBBM_PERFCTR_UCHE_9_HI 0x489 +#define A6XX_RBBM_PERFCTR_UCHE_10_LO 0x48a +#define A6XX_RBBM_PERFCTR_UCHE_10_HI 0x48b +#define A6XX_RBBM_PERFCTR_UCHE_11_LO 0x48c +#define A6XX_RBBM_PERFCTR_UCHE_11_HI 0x48d +#define A6XX_RBBM_PERFCTR_TP_0_LO 0x48e +#define A6XX_RBBM_PERFCTR_TP_0_HI 0x48f +#define A6XX_RBBM_PERFCTR_TP_1_LO 0x490 +#define A6XX_RBBM_PERFCTR_TP_1_HI 0x491 +#define A6XX_RBBM_PERFCTR_TP_2_LO 0x492 +#define A6XX_RBBM_PERFCTR_TP_2_HI 0x493 +#define A6XX_RBBM_PERFCTR_TP_3_LO 0x494 +#define A6XX_RBBM_PERFCTR_TP_3_HI 0x495 +#define A6XX_RBBM_PERFCTR_TP_4_LO 0x496 +#define A6XX_RBBM_PERFCTR_TP_4_HI 0x497 +#define A6XX_RBBM_PERFCTR_TP_5_LO 0x498 +#define A6XX_RBBM_PERFCTR_TP_5_HI 0x499 +#define A6XX_RBBM_PERFCTR_TP_6_LO 0x49a +#define A6XX_RBBM_PERFCTR_TP_6_HI 0x49b +#define A6XX_RBBM_PERFCTR_TP_7_LO 0x49c +#define A6XX_RBBM_PERFCTR_TP_7_HI 0x49d +#define A6XX_RBBM_PERFCTR_TP_8_LO 0x49e +#define A6XX_RBBM_PERFCTR_TP_8_HI 0x49f +#define A6XX_RBBM_PERFCTR_TP_9_LO 0x4a0 +#define A6XX_RBBM_PERFCTR_TP_9_HI 0x4a1 +#define A6XX_RBBM_PERFCTR_TP_10_LO 0x4a2 +#define A6XX_RBBM_PERFCTR_TP_10_HI 0x4a3 +#define A6XX_RBBM_PERFCTR_TP_11_LO 0x4a4 +#define A6XX_RBBM_PERFCTR_TP_11_HI 0x4a5 +#define A6XX_RBBM_PERFCTR_SP_0_LO 0x4a6 +#define A6XX_RBBM_PERFCTR_SP_0_HI 0x4a7 +#define A6XX_RBBM_PERFCTR_SP_1_LO 0x4a8 +#define A6XX_RBBM_PERFCTR_SP_1_HI 0x4a9 +#define A6XX_RBBM_PERFCTR_SP_2_LO 0x4aa +#define A6XX_RBBM_PERFCTR_SP_2_HI 0x4ab +#define A6XX_RBBM_PERFCTR_SP_3_LO 0x4ac +#define A6XX_RBBM_PERFCTR_SP_3_HI 0x4ad +#define A6XX_RBBM_PERFCTR_SP_4_LO 0x4ae +#define A6XX_RBBM_PERFCTR_SP_4_HI 0x4af +#define A6XX_RBBM_PERFCTR_SP_5_LO 0x4b0 +#define A6XX_RBBM_PERFCTR_SP_5_HI 0x4b1 +#define A6XX_RBBM_PERFCTR_SP_6_LO 0x4b2 +#define A6XX_RBBM_PERFCTR_SP_6_HI 0x4b3 +#define A6XX_RBBM_PERFCTR_SP_7_LO 0x4b4 +#define A6XX_RBBM_PERFCTR_SP_7_HI 0x4b5 +#define A6XX_RBBM_PERFCTR_SP_8_LO 0x4b6 +#define A6XX_RBBM_PERFCTR_SP_8_HI 0x4b7 +#define A6XX_RBBM_PERFCTR_SP_9_LO 0x4b8 +#define A6XX_RBBM_PERFCTR_SP_9_HI 0x4b9 +#define A6XX_RBBM_PERFCTR_SP_10_LO 0x4ba +#define A6XX_RBBM_PERFCTR_SP_10_HI 0x4bb +#define A6XX_RBBM_PERFCTR_SP_11_LO 0x4bc +#define A6XX_RBBM_PERFCTR_SP_11_HI 0x4bd +#define A6XX_RBBM_PERFCTR_SP_12_LO 0x4be +#define A6XX_RBBM_PERFCTR_SP_12_HI 0x4bf +#define A6XX_RBBM_PERFCTR_SP_13_LO 0x4c0 +#define A6XX_RBBM_PERFCTR_SP_13_HI 0x4c1 +#define A6XX_RBBM_PERFCTR_SP_14_LO 0x4c2 +#define A6XX_RBBM_PERFCTR_SP_14_HI 0x4c3 +#define A6XX_RBBM_PERFCTR_SP_15_LO 0x4c4 +#define A6XX_RBBM_PERFCTR_SP_15_HI 0x4c5 +#define A6XX_RBBM_PERFCTR_SP_16_LO 0x4c6 +#define A6XX_RBBM_PERFCTR_SP_16_HI 0x4c7 +#define A6XX_RBBM_PERFCTR_SP_17_LO 0x4c8 +#define A6XX_RBBM_PERFCTR_SP_17_HI 0x4c9 +#define A6XX_RBBM_PERFCTR_SP_18_LO 0x4ca +#define A6XX_RBBM_PERFCTR_SP_18_HI 0x4cb +#define A6XX_RBBM_PERFCTR_SP_19_LO 0x4cc +#define A6XX_RBBM_PERFCTR_SP_19_HI 0x4cd +#define A6XX_RBBM_PERFCTR_SP_20_LO 0x4ce +#define A6XX_RBBM_PERFCTR_SP_20_HI 0x4cf +#define A6XX_RBBM_PERFCTR_SP_21_LO 0x4d0 +#define A6XX_RBBM_PERFCTR_SP_21_HI 0x4d1 +#define A6XX_RBBM_PERFCTR_SP_22_LO 0x4d2 +#define A6XX_RBBM_PERFCTR_SP_22_HI 0x4d3 +#define A6XX_RBBM_PERFCTR_SP_23_LO 0x4d4 +#define A6XX_RBBM_PERFCTR_SP_23_HI 0x4d5 +#define A6XX_RBBM_PERFCTR_RB_0_LO 0x4d6 +#define A6XX_RBBM_PERFCTR_RB_0_HI 0x4d7 +#define A6XX_RBBM_PERFCTR_RB_1_LO 0x4d8 +#define A6XX_RBBM_PERFCTR_RB_1_HI 0x4d9 +#define A6XX_RBBM_PERFCTR_RB_2_LO 0x4da +#define A6XX_RBBM_PERFCTR_RB_2_HI 0x4db +#define A6XX_RBBM_PERFCTR_RB_3_LO 0x4dc +#define A6XX_RBBM_PERFCTR_RB_3_HI 0x4dd +#define A6XX_RBBM_PERFCTR_RB_4_LO 0x4de +#define A6XX_RBBM_PERFCTR_RB_4_HI 0x4df +#define A6XX_RBBM_PERFCTR_RB_5_LO 0x4e0 +#define A6XX_RBBM_PERFCTR_RB_5_HI 0x4e1 +#define A6XX_RBBM_PERFCTR_RB_6_LO 0x4e2 +#define A6XX_RBBM_PERFCTR_RB_6_HI 0x4e3 +#define A6XX_RBBM_PERFCTR_RB_7_LO 0x4e4 +#define A6XX_RBBM_PERFCTR_RB_7_HI 0x4e5 +#define A6XX_RBBM_PERFCTR_VSC_0_LO 0x4e6 +#define A6XX_RBBM_PERFCTR_VSC_0_HI 0x4e7 +#define A6XX_RBBM_PERFCTR_VSC_1_LO 0x4e8 +#define A6XX_RBBM_PERFCTR_VSC_1_HI 0x4e9 +#define A6XX_RBBM_PERFCTR_LRZ_0_LO 0x4ea +#define A6XX_RBBM_PERFCTR_LRZ_0_HI 0x4eb +#define A6XX_RBBM_PERFCTR_LRZ_1_LO 0x4ec +#define A6XX_RBBM_PERFCTR_LRZ_1_HI 0x4ed +#define A6XX_RBBM_PERFCTR_LRZ_2_LO 0x4ee +#define A6XX_RBBM_PERFCTR_LRZ_2_HI 0x4ef +#define A6XX_RBBM_PERFCTR_LRZ_3_LO 0x4f0 +#define A6XX_RBBM_PERFCTR_LRZ_3_HI 0x4f1 +#define A6XX_RBBM_PERFCTR_CMP_0_LO 0x4f2 +#define A6XX_RBBM_PERFCTR_CMP_0_HI 0x4f3 +#define A6XX_RBBM_PERFCTR_CMP_1_LO 0x4f4 +#define A6XX_RBBM_PERFCTR_CMP_1_HI 0x4f5 +#define A6XX_RBBM_PERFCTR_CMP_2_LO 0x4f6 +#define A6XX_RBBM_PERFCTR_CMP_2_HI 0x4f7 +#define A6XX_RBBM_PERFCTR_CMP_3_LO 0x4f8 +#define A6XX_RBBM_PERFCTR_CMP_3_HI 0x4f9 +#define A6XX_RBBM_PERFCTR_CNTL 0x500 +#define A6XX_RBBM_PERFCTR_LOAD_CMD0 0x501 +#define A6XX_RBBM_PERFCTR_LOAD_CMD1 0x502 +#define A6XX_RBBM_PERFCTR_LOAD_CMD2 0x503 +#define A6XX_RBBM_PERFCTR_LOAD_CMD3 0x504 +#define A6XX_RBBM_PERFCTR_LOAD_VALUE_LO 0x505 +#define A6XX_RBBM_PERFCTR_LOAD_VALUE_HI 0x506 +#define A6XX_RBBM_PERFCTR_RBBM_SEL_0 0x507 +#define A6XX_RBBM_PERFCTR_RBBM_SEL_1 0x508 +#define A6XX_RBBM_PERFCTR_RBBM_SEL_2 0x509 +#define A6XX_RBBM_PERFCTR_RBBM_SEL_3 0x50A +#define A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED 0x50B +#define A6XX_RBBM_PERFCTR_SRAM_INIT_CMD 0x50e +#define A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS 0x50f + +#define A6XX_RBBM_ISDB_CNT 0x533 +#define A6XX_RBBM_NC_MODE_CNTL 0X534 +#define A6XX_RBBM_SNAPSHOT_STATUS 0x535 +#define A6XX_RBBM_LPAC_GBIF_CLIENT_QOS_CNTL 0x5ff + +#define A6XX_RBBM_SECVID_TRUST_CNTL 0xF400 +#define A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO 0xF800 +#define A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI 0xF801 +#define A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE 0xF802 +#define A6XX_RBBM_SECVID_TSB_CNTL 0xF803 +#define A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL 0xF810 + +#define A6XX_RBBM_VBIF_CLIENT_QOS_CNTL 0x00010 +#define A6XX_RBBM_GBIF_CLIENT_QOS_CNTL 0x00011 +#define A6XX_RBBM_GBIF_HALT 0x00016 +#define A6XX_RBBM_GBIF_HALT_ACK 0x00017 +#define A6XX_RBBM_GPR0_CNTL 0x00018 +#define A6XX_RBBM_INTERFACE_HANG_INT_CNTL 0x0001f +#define A6XX_RBBM_INT_CLEAR_CMD 0x00037 +#define A6XX_RBBM_INT_0_MASK 0x00038 +#define A6XX_RBBM_INT_2_MASK 0x0003A +#define A6XX_RBBM_SP_HYST_CNT 0x00042 +#define A6XX_RBBM_SW_RESET_CMD 0x00043 +#define A6XX_RBBM_RAC_THRESHOLD_CNT 0x00044 +#define A6XX_RBBM_BLOCK_SW_RESET_CMD 0x00045 +#define A6XX_RBBM_BLOCK_SW_RESET_CMD2 0x00046 +#define A6XX_RBBM_BLOCK_GX_RETENTION_CNTL 0x00050 +#define A6XX_RBBM_CLOCK_CNTL 0x000ae +#define A6XX_RBBM_CLOCK_CNTL_SP0 0x000b0 +#define A6XX_RBBM_CLOCK_CNTL_SP1 0x000b1 +#define A6XX_RBBM_CLOCK_CNTL_SP2 0x000b2 +#define A6XX_RBBM_CLOCK_CNTL_SP3 0x000b3 +#define A6XX_RBBM_CLOCK_CNTL2_SP0 0x000b4 +#define A6XX_RBBM_CLOCK_CNTL2_SP1 0x000b5 +#define A6XX_RBBM_CLOCK_CNTL2_SP2 0x000b6 +#define A6XX_RBBM_CLOCK_CNTL2_SP3 0x000b7 +#define A6XX_RBBM_CLOCK_DELAY_SP0 0x000b8 +#define A6XX_RBBM_CLOCK_DELAY_SP1 0x000b9 +#define A6XX_RBBM_CLOCK_DELAY_SP2 0x000ba +#define A6XX_RBBM_CLOCK_DELAY_SP3 0x000bb +#define A6XX_RBBM_CLOCK_HYST_SP0 0x000bc +#define A6XX_RBBM_CLOCK_HYST_SP1 0x000bd +#define A6XX_RBBM_CLOCK_HYST_SP2 0x000be +#define A6XX_RBBM_CLOCK_HYST_SP3 0x000bf +#define A6XX_RBBM_CLOCK_CNTL_TP0 0x000c0 +#define A6XX_RBBM_CLOCK_CNTL_TP1 0x000c1 +#define A6XX_RBBM_CLOCK_CNTL_TP2 0x000c2 +#define A6XX_RBBM_CLOCK_CNTL_TP3 0x000c3 +#define A6XX_RBBM_CLOCK_CNTL2_TP0 0x000c4 +#define A6XX_RBBM_CLOCK_CNTL2_TP1 0x000c5 +#define A6XX_RBBM_CLOCK_CNTL2_TP2 0x000c6 +#define A6XX_RBBM_CLOCK_CNTL2_TP3 0x000c7 +#define A6XX_RBBM_CLOCK_CNTL3_TP0 0x000c8 +#define A6XX_RBBM_CLOCK_CNTL3_TP1 0x000c9 +#define A6XX_RBBM_CLOCK_CNTL3_TP2 0x000ca +#define A6XX_RBBM_CLOCK_CNTL3_TP3 0x000cb +#define A6XX_RBBM_CLOCK_CNTL4_TP0 0x000cc +#define A6XX_RBBM_CLOCK_CNTL4_TP1 0x000cd +#define A6XX_RBBM_CLOCK_CNTL4_TP2 0x000ce +#define A6XX_RBBM_CLOCK_CNTL4_TP3 0x000cf +#define A6XX_RBBM_CLOCK_DELAY_TP0 0x000d0 +#define A6XX_RBBM_CLOCK_DELAY_TP1 0x000d1 +#define A6XX_RBBM_CLOCK_DELAY_TP2 0x000d2 +#define A6XX_RBBM_CLOCK_DELAY_TP3 0x000d3 +#define A6XX_RBBM_CLOCK_DELAY2_TP0 0x000d4 +#define A6XX_RBBM_CLOCK_DELAY2_TP1 0x000d5 +#define A6XX_RBBM_CLOCK_DELAY2_TP2 0x000d6 +#define A6XX_RBBM_CLOCK_DELAY2_TP3 0x000d7 +#define A6XX_RBBM_CLOCK_DELAY3_TP0 0x000d8 +#define A6XX_RBBM_CLOCK_DELAY3_TP1 0x000d9 +#define A6XX_RBBM_CLOCK_DELAY3_TP2 0x000da +#define A6XX_RBBM_CLOCK_DELAY3_TP3 0x000db +#define A6XX_RBBM_CLOCK_DELAY4_TP0 0x000dc +#define A6XX_RBBM_CLOCK_DELAY4_TP1 0x000dd +#define A6XX_RBBM_CLOCK_DELAY4_TP2 0x000de +#define A6XX_RBBM_CLOCK_DELAY4_TP3 0x000df +#define A6XX_RBBM_CLOCK_HYST_TP0 0x000e0 +#define A6XX_RBBM_CLOCK_HYST_TP1 0x000e1 +#define A6XX_RBBM_CLOCK_HYST_TP2 0x000e2 +#define A6XX_RBBM_CLOCK_HYST_TP3 0x000e3 +#define A6XX_RBBM_CLOCK_HYST2_TP0 0x000e4 +#define A6XX_RBBM_CLOCK_HYST2_TP1 0x000e5 +#define A6XX_RBBM_CLOCK_HYST2_TP2 0x000e6 +#define A6XX_RBBM_CLOCK_HYST2_TP3 0x000e7 +#define A6XX_RBBM_CLOCK_HYST3_TP0 0x000e8 +#define A6XX_RBBM_CLOCK_HYST3_TP1 0x000e9 +#define A6XX_RBBM_CLOCK_HYST3_TP2 0x000ea +#define A6XX_RBBM_CLOCK_HYST3_TP3 0x000eb +#define A6XX_RBBM_CLOCK_HYST4_TP0 0x000ec +#define A6XX_RBBM_CLOCK_HYST4_TP1 0x000ed +#define A6XX_RBBM_CLOCK_HYST4_TP2 0x000ee +#define A6XX_RBBM_CLOCK_HYST4_TP3 0x000ef +#define A6XX_RBBM_CLOCK_CNTL_RB0 0x000f0 +#define A6XX_RBBM_CLOCK_CNTL_RB1 0x000f1 +#define A6XX_RBBM_CLOCK_CNTL_RB2 0x000f2 +#define A6XX_RBBM_CLOCK_CNTL_RB3 0x000f3 +#define A6XX_RBBM_CLOCK_CNTL2_RB0 0x000f4 +#define A6XX_RBBM_CLOCK_CNTL2_RB1 0x000f5 +#define A6XX_RBBM_CLOCK_CNTL2_RB2 0x000f6 +#define A6XX_RBBM_CLOCK_CNTL2_RB3 0x000f7 +#define A6XX_RBBM_CLOCK_CNTL_CCU0 0x000f8 +#define A6XX_RBBM_CLOCK_CNTL_CCU1 0x000f9 +#define A6XX_RBBM_CLOCK_CNTL_CCU2 0x000fa +#define A6XX_RBBM_CLOCK_CNTL_CCU3 0x000fb +#define A6XX_RBBM_CLOCK_HYST_RB_CCU0 0x00100 +#define A6XX_RBBM_CLOCK_HYST_RB_CCU1 0x00101 +#define A6XX_RBBM_CLOCK_HYST_RB_CCU2 0x00102 +#define A6XX_RBBM_CLOCK_HYST_RB_CCU3 0x00103 +#define A6XX_RBBM_CLOCK_CNTL_RAC 0x00104 +#define A6XX_RBBM_CLOCK_CNTL2_RAC 0x00105 +#define A6XX_RBBM_CLOCK_DELAY_RAC 0x00106 +#define A6XX_RBBM_CLOCK_HYST_RAC 0x00107 +#define A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM 0x00108 +#define A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM 0x00109 +#define A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM 0x0010a +#define A6XX_RBBM_CLOCK_CNTL_UCHE 0x0010b +#define A6XX_RBBM_CLOCK_CNTL2_UCHE 0x0010c +#define A6XX_RBBM_CLOCK_CNTL3_UCHE 0x0010d +#define A6XX_RBBM_CLOCK_CNTL4_UCHE 0x0010e +#define A6XX_RBBM_CLOCK_DELAY_UCHE 0x0010f +#define A6XX_RBBM_CLOCK_HYST_UCHE 0x00110 +#define A6XX_RBBM_CLOCK_MODE_VFD 0x00111 +#define A6XX_RBBM_CLOCK_DELAY_VFD 0x00112 +#define A6XX_RBBM_CLOCK_HYST_VFD 0x00113 +#define A6XX_RBBM_CLOCK_MODE_GPC 0x00114 +#define A6XX_RBBM_CLOCK_DELAY_GPC 0x00115 +#define A6XX_RBBM_CLOCK_HYST_GPC 0x00116 +#define A6XX_RBBM_CLOCK_DELAY_HLSQ_2 0x00117 +#define A6XX_RBBM_CLOCK_CNTL_GMU_GX 0x00118 +#define A6XX_RBBM_CLOCK_DELAY_GMU_GX 0x00119 +#define A6XX_RBBM_CLOCK_CNTL_TEX_FCHE 0x00120 +#define A6XX_RBBM_CLOCK_DELAY_TEX_FCHE 0x00121 +#define A6XX_RBBM_CLOCK_HYST_TEX_FCHE 0x00122 +#define A6XX_RBBM_CLOCK_HYST_GMU_GX 0x0011a +#define A6XX_RBBM_CLOCK_MODE_HLSQ 0x0011b +#define A6XX_RBBM_CLOCK_DELAY_HLSQ 0x0011c +#define A6XX_RBBM_CLOCK_HYST_HLSQ 0x0011d +#define A6XX_RBBM_CLOCK_CNTL_FCHE 0x00123 +#define A6XX_RBBM_CLOCK_DELAY_FCHE 0x00124 +#define A6XX_RBBM_CLOCK_HYST_FCHE 0x00125 +#define A6XX_RBBM_CLOCK_CNTL_MHUB 0x00126 +#define A6XX_RBBM_CLOCK_DELAY_MHUB 0x00127 +#define A6XX_RBBM_CLOCK_HYST_MHUB 0x00128 +#define A6XX_RBBM_CLOCK_DELAY_GLC 0x00129 +#define A6XX_RBBM_CLOCK_HYST_GLC 0x0012a +#define A6XX_RBBM_CLOCK_CNTL_GLC 0x0012b + +#define A6XX_GMUAO_GMU_CGC_MODE_CNTL 0x23b09 +#define A6XX_GMUAO_GMU_CGC_DELAY_CNTL 0x23b0a +#define A6XX_GMUAO_GMU_CGC_HYST_CNTL 0x23b0b +#define A6XX_GMUCX_GMU_WFI_CONFIG 0x1f802 +#define A6XX_GMUGX_GMU_SP_RF_CONTROL_0 0x1a883 +#define A6XX_GMUGX_GMU_SP_RF_CONTROL_1 0x1a884 + +/* DBGC_CFG registers */ +#define A6XX_DBGC_CFG_DBGBUS_SEL_A 0x600 +#define A6XX_DBGC_CFG_DBGBUS_SEL_B 0x601 +#define A6XX_DBGC_CFG_DBGBUS_SEL_C 0x602 +#define A6XX_DBGC_CFG_DBGBUS_SEL_D 0x603 +#define A6XX_DBGC_CFG_DBGBUS_SEL_PING_INDEX_SHIFT 0x0 +#define A6XX_DBGC_CFG_DBGBUS_SEL_PING_BLK_SEL_SHIFT 0x8 +#define A6XX_DBGC_CFG_DBGBUS_CNTLT 0x604 +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN_SHIFT 0x0 +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU_SHIFT 0xC +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT_SHIFT 0x1C +#define A6XX_DBGC_CFG_DBGBUS_CNTLM 0x605 +#define A6XX_DBGC_CFG_DBGBUS_CTLTM_ENABLE_SHIFT 0x18 +#define A6XX_DBGC_CFG_DBGBUS_OPL 0x606 +#define A6XX_DBGC_CFG_DBGBUS_OPE 0x607 +#define A6XX_DBGC_CFG_DBGBUS_IVTL_0 0x608 +#define A6XX_DBGC_CFG_DBGBUS_IVTL_1 0x609 +#define A6XX_DBGC_CFG_DBGBUS_IVTL_2 0x60a +#define A6XX_DBGC_CFG_DBGBUS_IVTL_3 0x60b +#define A6XX_DBGC_CFG_DBGBUS_MASKL_0 0x60c +#define A6XX_DBGC_CFG_DBGBUS_MASKL_1 0x60d +#define A6XX_DBGC_CFG_DBGBUS_MASKL_2 0x60e +#define A6XX_DBGC_CFG_DBGBUS_MASKL_3 0x60f +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0 0x610 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1 0x611 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL0_SHIFT 0x0 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL1_SHIFT 0x4 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL2_SHIFT 0x8 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL3_SHIFT 0xC +#define A6XX_DBGC_CFG_DBGBUS_BYTEL4_SHIFT 0x10 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL5_SHIFT 0x14 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL6_SHIFT 0x18 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL7_SHIFT 0x1C +#define A6XX_DBGC_CFG_DBGBUS_BYTEL8_SHIFT 0x0 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL9_SHIFT 0x4 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL10_SHIFT 0x8 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL11_SHIFT 0xC +#define A6XX_DBGC_CFG_DBGBUS_BYTEL12_SHIFT 0x10 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL13_SHIFT 0x14 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL14_SHIFT 0x18 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL15_SHIFT 0x1C +#define A6XX_DBGC_CFG_DBGBUS_IVTE_0 0x612 +#define A6XX_DBGC_CFG_DBGBUS_IVTE_1 0x613 +#define A6XX_DBGC_CFG_DBGBUS_IVTE_2 0x614 +#define A6XX_DBGC_CFG_DBGBUS_IVTE_3 0x615 +#define A6XX_DBGC_CFG_DBGBUS_MASKE_0 0x616 +#define A6XX_DBGC_CFG_DBGBUS_MASKE_1 0x617 +#define A6XX_DBGC_CFG_DBGBUS_MASKE_2 0x618 +#define A6XX_DBGC_CFG_DBGBUS_MASKE_3 0x619 +#define A6XX_DBGC_CFG_DBGBUS_NIBBLEE 0x61a +#define A6XX_DBGC_CFG_DBGBUS_PTRC0 0x61b +#define A6XX_DBGC_CFG_DBGBUS_PTRC1 0x61c +#define A6XX_DBGC_CFG_DBGBUS_LOADREG 0x61d +#define A6XX_DBGC_CFG_DBGBUS_IDX 0x61e +#define A6XX_DBGC_CFG_DBGBUS_CLRC 0x61f +#define A6XX_DBGC_CFG_DBGBUS_LOADIVT 0x620 +#define A6XX_DBGC_VBIF_DBG_CNTL 0x621 +#define A6XX_DBGC_DBG_LO_HI_GPIO 0x622 +#define A6XX_DBGC_EXT_TRACE_BUS_CNTL 0x623 +#define A6XX_DBGC_READ_AHB_THROUGH_DBG 0x624 +#define A6XX_DBGC_CFG_DBGBUS_TRACE_BUF1 0x62f +#define A6XX_DBGC_CFG_DBGBUS_TRACE_BUF2 0x630 +#define A6XX_DBGC_EVT_CFG 0x640 +#define A6XX_DBGC_EVT_INTF_SEL_0 0x641 +#define A6XX_DBGC_EVT_INTF_SEL_1 0x642 +#define A6XX_DBGC_PERF_ATB_CFG 0x643 +#define A6XX_DBGC_PERF_ATB_COUNTER_SEL_0 0x644 +#define A6XX_DBGC_PERF_ATB_COUNTER_SEL_1 0x645 +#define A6XX_DBGC_PERF_ATB_COUNTER_SEL_2 0x646 +#define A6XX_DBGC_PERF_ATB_COUNTER_SEL_3 0x647 +#define A6XX_DBGC_PERF_ATB_TRIG_INTF_SEL_0 0x648 +#define A6XX_DBGC_PERF_ATB_TRIG_INTF_SEL_1 0x649 +#define A6XX_DBGC_PERF_ATB_DRAIN_CMD 0x64a +#define A6XX_DBGC_ECO_CNTL 0x650 +#define A6XX_DBGC_AHB_DBG_CNTL 0x651 + +/* VSC registers */ +#define A6XX_VSC_PERFCTR_VSC_SEL_0 0xCD8 +#define A6XX_VSC_PERFCTR_VSC_SEL_1 0xCD9 + +/* GRAS registers */ +#define A6XX_GRAS_ADDR_MODE_CNTL 0x8601 +#define A6XX_GRAS_PERFCTR_TSE_SEL_0 0x8610 +#define A6XX_GRAS_PERFCTR_TSE_SEL_1 0x8611 +#define A6XX_GRAS_PERFCTR_TSE_SEL_2 0x8612 +#define A6XX_GRAS_PERFCTR_TSE_SEL_3 0x8613 +#define A6XX_GRAS_PERFCTR_RAS_SEL_0 0x8614 +#define A6XX_GRAS_PERFCTR_RAS_SEL_1 0x8615 +#define A6XX_GRAS_PERFCTR_RAS_SEL_2 0x8616 +#define A6XX_GRAS_PERFCTR_RAS_SEL_3 0x8617 +#define A6XX_GRAS_PERFCTR_LRZ_SEL_0 0x8618 +#define A6XX_GRAS_PERFCTR_LRZ_SEL_1 0x8619 +#define A6XX_GRAS_PERFCTR_LRZ_SEL_2 0x861A +#define A6XX_GRAS_PERFCTR_LRZ_SEL_3 0x861B + +/* RB registers */ +#define A6XX_RB_ADDR_MODE_CNTL 0x8E05 +#define A6XX_RB_NC_MODE_CNTL 0x8E08 +#define A6XX_RB_PERFCTR_RB_SEL_0 0x8E10 +#define A6XX_RB_PERFCTR_RB_SEL_1 0x8E11 +#define A6XX_RB_PERFCTR_RB_SEL_2 0x8E12 +#define A6XX_RB_PERFCTR_RB_SEL_3 0x8E13 +#define A6XX_RB_PERFCTR_RB_SEL_4 0x8E14 +#define A6XX_RB_PERFCTR_RB_SEL_5 0x8E15 +#define A6XX_RB_PERFCTR_RB_SEL_6 0x8E16 +#define A6XX_RB_PERFCTR_RB_SEL_7 0x8E17 +#define A6XX_RB_PERFCTR_CCU_SEL_0 0x8E18 +#define A6XX_RB_PERFCTR_CCU_SEL_1 0x8E19 +#define A6XX_RB_PERFCTR_CCU_SEL_2 0x8E1A +#define A6XX_RB_PERFCTR_CCU_SEL_3 0x8E1B +#define A6XX_RB_PERFCTR_CCU_SEL_4 0x8E1C +#define A6XX_RB_PERFCTR_CMP_SEL_0 0x8E2C +#define A6XX_RB_PERFCTR_CMP_SEL_1 0x8E2D +#define A6XX_RB_PERFCTR_CMP_SEL_2 0x8E2E +#define A6XX_RB_PERFCTR_CMP_SEL_3 0x8E2F +#define A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_HOST 0x8E3B +#define A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_CD 0x8E3D +#define A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE 0x8E50 + +/* PC registers */ +#define A6XX_PC_DBG_ECO_CNTL 0x9E00 +#define A6XX_PC_ADDR_MODE_CNTL 0x9E01 +#define A6XX_PC_PERFCTR_PC_SEL_0 0x9E34 +#define A6XX_PC_PERFCTR_PC_SEL_1 0x9E35 +#define A6XX_PC_PERFCTR_PC_SEL_2 0x9E36 +#define A6XX_PC_PERFCTR_PC_SEL_3 0x9E37 +#define A6XX_PC_PERFCTR_PC_SEL_4 0x9E38 +#define A6XX_PC_PERFCTR_PC_SEL_5 0x9E39 +#define A6XX_PC_PERFCTR_PC_SEL_6 0x9E3A +#define A6XX_PC_PERFCTR_PC_SEL_7 0x9E3B + +/* HLSQ registers */ +#define A6XX_HLSQ_ADDR_MODE_CNTL 0xBE05 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_0 0xBE10 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_1 0xBE11 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_2 0xBE12 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_3 0xBE13 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_4 0xBE14 +#define A6XX_HLSQ_PERFCTR_HLSQ_SEL_5 0xBE15 +#define A6XX_HLSQ_DBG_AHB_READ_APERTURE 0xC800 +#define A6XX_HLSQ_DBG_READ_SEL 0xD000 + +/* VFD registers */ +#define A6XX_VFD_ADDR_MODE_CNTL 0xA601 +#define A6XX_VFD_PERFCTR_VFD_SEL_0 0xA610 +#define A6XX_VFD_PERFCTR_VFD_SEL_1 0xA611 +#define A6XX_VFD_PERFCTR_VFD_SEL_2 0xA612 +#define A6XX_VFD_PERFCTR_VFD_SEL_3 0xA613 +#define A6XX_VFD_PERFCTR_VFD_SEL_4 0xA614 +#define A6XX_VFD_PERFCTR_VFD_SEL_5 0xA615 +#define A6XX_VFD_PERFCTR_VFD_SEL_6 0xA616 +#define A6XX_VFD_PERFCTR_VFD_SEL_7 0xA617 + +/* VPC registers */ +#define A6XX_VPC_ADDR_MODE_CNTL 0x9601 +#define A6XX_VPC_PERFCTR_VPC_SEL_0 0x9604 +#define A6XX_VPC_PERFCTR_VPC_SEL_1 0x9605 +#define A6XX_VPC_PERFCTR_VPC_SEL_2 0x9606 +#define A6XX_VPC_PERFCTR_VPC_SEL_3 0x9607 +#define A6XX_VPC_PERFCTR_VPC_SEL_4 0x9608 +#define A6XX_VPC_PERFCTR_VPC_SEL_5 0x9609 + +/* UCHE registers */ +#define A6XX_UCHE_ADDR_MODE_CNTL 0xE00 +#define A6XX_UCHE_MODE_CNTL 0xE01 +#define A6XX_UCHE_WRITE_RANGE_MAX_LO 0xE05 +#define A6XX_UCHE_WRITE_RANGE_MAX_HI 0xE06 +#define A6XX_UCHE_WRITE_THRU_BASE_LO 0xE07 +#define A6XX_UCHE_WRITE_THRU_BASE_HI 0xE08 +#define A6XX_UCHE_TRAP_BASE_LO 0xE09 +#define A6XX_UCHE_TRAP_BASE_HI 0xE0A +#define A6XX_UCHE_GMEM_RANGE_MIN_LO 0xE0B +#define A6XX_UCHE_GMEM_RANGE_MIN_HI 0xE0C +#define A6XX_UCHE_GMEM_RANGE_MAX_LO 0xE0D +#define A6XX_UCHE_GMEM_RANGE_MAX_HI 0xE0E +#define A6XX_UCHE_CACHE_WAYS 0xE17 +#define A6XX_UCHE_FILTER_CNTL 0xE18 +#define A6XX_UCHE_CLIENT_PF 0xE19 +#define A6XX_UCHE_CLIENT_PF_CLIENT_ID_MASK 0x7 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_0 0xE1C +#define A6XX_UCHE_PERFCTR_UCHE_SEL_1 0xE1D +#define A6XX_UCHE_PERFCTR_UCHE_SEL_2 0xE1E +#define A6XX_UCHE_PERFCTR_UCHE_SEL_3 0xE1F +#define A6XX_UCHE_PERFCTR_UCHE_SEL_4 0xE20 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_5 0xE21 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_6 0xE22 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_7 0xE23 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_8 0xE24 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_9 0xE25 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_10 0xE26 +#define A6XX_UCHE_PERFCTR_UCHE_SEL_11 0xE27 +#define A6XX_UCHE_GBIF_GX_CONFIG 0xE3A +#define A6XX_UCHE_CMDQ_CONFIG 0xE3C + +/* SP registers */ +#define A6XX_SP_ADDR_MODE_CNTL 0xAE01 +#define A6XX_SP_NC_MODE_CNTL 0xAE02 +#define A6XX_SP_PERFCTR_SP_SEL_0 0xAE10 +#define A6XX_SP_PERFCTR_SP_SEL_1 0xAE11 +#define A6XX_SP_PERFCTR_SP_SEL_2 0xAE12 +#define A6XX_SP_PERFCTR_SP_SEL_3 0xAE13 +#define A6XX_SP_PERFCTR_SP_SEL_4 0xAE14 +#define A6XX_SP_PERFCTR_SP_SEL_5 0xAE15 +#define A6XX_SP_PERFCTR_SP_SEL_6 0xAE16 +#define A6XX_SP_PERFCTR_SP_SEL_7 0xAE17 +#define A6XX_SP_PERFCTR_SP_SEL_8 0xAE18 +#define A6XX_SP_PERFCTR_SP_SEL_9 0xAE19 +#define A6XX_SP_PERFCTR_SP_SEL_10 0xAE1A +#define A6XX_SP_PERFCTR_SP_SEL_11 0xAE1B +#define A6XX_SP_PERFCTR_SP_SEL_12 0xAE1C +#define A6XX_SP_PERFCTR_SP_SEL_13 0xAE1D +#define A6XX_SP_PERFCTR_SP_SEL_14 0xAE1E +#define A6XX_SP_PERFCTR_SP_SEL_15 0xAE1F +#define A6XX_SP_PERFCTR_SP_SEL_16 0xAE20 +#define A6XX_SP_PERFCTR_SP_SEL_17 0xAE21 +#define A6XX_SP_PERFCTR_SP_SEL_18 0xAE22 +#define A6XX_SP_PERFCTR_SP_SEL_19 0xAE23 +#define A6XX_SP_PERFCTR_SP_SEL_20 0xAE24 +#define A6XX_SP_PERFCTR_SP_SEL_21 0xAE25 +#define A6XX_SP_PERFCTR_SP_SEL_22 0xAE26 +#define A6XX_SP_PERFCTR_SP_SEL_23 0xAE27 + +/* TP registers */ +#define A6XX_TPL1_ADDR_MODE_CNTL 0xB601 +#define A6XX_TPL1_NC_MODE_CNTL 0xB604 +#define A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0 0xB608 +#define A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1 0xB609 +#define A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2 0xB60A +#define A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3 0xB60B +#define A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4 0xB60C +#define A6XX_TPL1_PERFCTR_TP_SEL_0 0xB610 +#define A6XX_TPL1_PERFCTR_TP_SEL_1 0xB611 +#define A6XX_TPL1_PERFCTR_TP_SEL_2 0xB612 +#define A6XX_TPL1_PERFCTR_TP_SEL_3 0xB613 +#define A6XX_TPL1_PERFCTR_TP_SEL_4 0xB614 +#define A6XX_TPL1_PERFCTR_TP_SEL_5 0xB615 +#define A6XX_TPL1_PERFCTR_TP_SEL_6 0xB616 +#define A6XX_TPL1_PERFCTR_TP_SEL_7 0xB617 +#define A6XX_TPL1_PERFCTR_TP_SEL_8 0xB618 +#define A6XX_TPL1_PERFCTR_TP_SEL_9 0xB619 +#define A6XX_TPL1_PERFCTR_TP_SEL_10 0xB61A +#define A6XX_TPL1_PERFCTR_TP_SEL_11 0xB61B + +/* VBIF registers */ +#define A6XX_VBIF_VERSION 0x3000 +#define A6XX_VBIF_CLKON 0x3001 +#define A6XX_VBIF_CLKON_FORCE_ON_TESTBUS_MASK 0x1 +#define A6XX_VBIF_CLKON_FORCE_ON_TESTBUS_SHIFT 0x1 +#define A6XX_VBIF_GATE_OFF_WRREQ_EN 0x302A +#define A6XX_VBIF_XIN_HALT_CTRL0 0x3080 +#define A6XX_VBIF_XIN_HALT_CTRL0_MASK 0xF +#define A6XX_VBIF_XIN_HALT_CTRL1 0x3081 +#define A6XX_VBIF_TEST_BUS_OUT_CTRL 0x3084 +#define A6XX_VBIF_TEST_BUS_OUT_CTRL_EN_MASK 0x1 +#define A6XX_VBIF_TEST_BUS_OUT_CTRL_EN_SHIFT 0x0 +#define A6XX_VBIF_TEST_BUS1_CTRL0 0x3085 +#define A6XX_VBIF_TEST_BUS1_CTRL1 0x3086 +#define A6XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_MASK 0xF +#define A6XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_SHIFT 0x0 +#define A6XX_VBIF_TEST_BUS2_CTRL0 0x3087 +#define A6XX_VBIF_TEST_BUS2_CTRL1 0x3088 +#define A6XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK 0x1FF +#define A6XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT 0x0 +#define A6XX_VBIF_TEST_BUS_OUT 0x308C +#define A6XX_VBIF_PERF_CNT_SEL0 0x30d0 +#define A6XX_VBIF_PERF_CNT_SEL1 0x30d1 +#define A6XX_VBIF_PERF_CNT_SEL2 0x30d2 +#define A6XX_VBIF_PERF_CNT_SEL3 0x30d3 +#define A6XX_VBIF_PERF_CNT_LOW0 0x30d8 +#define A6XX_VBIF_PERF_CNT_LOW1 0x30d9 +#define A6XX_VBIF_PERF_CNT_LOW2 0x30da +#define A6XX_VBIF_PERF_CNT_LOW3 0x30db +#define A6XX_VBIF_PERF_CNT_HIGH0 0x30e0 +#define A6XX_VBIF_PERF_CNT_HIGH1 0x30e1 +#define A6XX_VBIF_PERF_CNT_HIGH2 0x30e2 +#define A6XX_VBIF_PERF_CNT_HIGH3 0x30e3 +#define A6XX_VBIF_PERF_PWR_CNT_EN0 0x3100 +#define A6XX_VBIF_PERF_PWR_CNT_EN1 0x3101 +#define A6XX_VBIF_PERF_PWR_CNT_EN2 0x3102 +#define A6XX_VBIF_PERF_PWR_CNT_LOW0 0x3110 +#define A6XX_VBIF_PERF_PWR_CNT_LOW1 0x3111 +#define A6XX_VBIF_PERF_PWR_CNT_LOW2 0x3112 +#define A6XX_VBIF_PERF_PWR_CNT_HIGH0 0x3118 +#define A6XX_VBIF_PERF_PWR_CNT_HIGH1 0x3119 +#define A6XX_VBIF_PERF_PWR_CNT_HIGH2 0x311a + +/* GBIF countables */ +#define GBIF_AXI0_READ_DATA_TOTAL_BEATS 34 +#define GBIF_AXI1_READ_DATA_TOTAL_BEATS 35 +#define GBIF_AXI0_WRITE_DATA_TOTAL_BEATS 46 +#define GBIF_AXI1_WRITE_DATA_TOTAL_BEATS 47 + +/* GBIF registers */ +#define A6XX_GBIF_CX_CONFIG 0x3c00 +#define A6XX_GBIF_SCACHE_CNTL0 0x3c01 +#define A6XX_GBIF_SCACHE_CNTL1 0x3c02 +#define A6XX_GBIF_QSB_SIDE0 0x3c03 +#define A6XX_GBIF_QSB_SIDE1 0x3c04 +#define A6XX_GBIF_QSB_SIDE2 0x3c05 +#define A6XX_GBIF_QSB_SIDE3 0x3c06 +#define A6XX_GBIF_HALT 0x3c45 +#define A6XX_GBIF_HALT_ACK 0x3c46 + +#define A6XX_GBIF_CLIENT_HALT_MASK BIT(0) +#define A6XX_GBIF_ARB_HALT_MASK BIT(1) +#define A6XX_GBIF_GX_HALT_MASK BIT(0) + +#define A6XX_GBIF_PERF_PWR_CNT_EN 0x3cc0 +#define A6XX_GBIF_PERF_CNT_SEL 0x3cc2 +#define A6XX_GBIF_PERF_PWR_CNT_SEL 0x3cc3 +#define A6XX_GBIF_PERF_CNT_LOW0 0x3cc4 +#define A6XX_GBIF_PERF_CNT_LOW1 0x3cc5 +#define A6XX_GBIF_PERF_CNT_LOW2 0x3cc6 +#define A6XX_GBIF_PERF_CNT_LOW3 0x3cc7 +#define A6XX_GBIF_PERF_CNT_HIGH0 0x3cc8 +#define A6XX_GBIF_PERF_CNT_HIGH1 0x3cc9 +#define A6XX_GBIF_PERF_CNT_HIGH2 0x3cca +#define A6XX_GBIF_PERF_CNT_HIGH3 0x3ccb +#define A6XX_GBIF_PWR_CNT_LOW0 0x3ccc +#define A6XX_GBIF_PWR_CNT_LOW1 0x3ccd +#define A6XX_GBIF_PWR_CNT_LOW2 0x3cce +#define A6XX_GBIF_PWR_CNT_HIGH0 0x3ccf +#define A6XX_GBIF_PWR_CNT_HIGH1 0x3cd0 +#define A6XX_GBIF_PWR_CNT_HIGH2 0x3cd1 + + +/* CX_DBGC_CFG registers */ +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_A 0x18400 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_B 0x18401 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_C 0x18402 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_D 0x18403 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_PING_INDEX_SHIFT 0x0 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_PING_BLK_SEL_SHIFT 0x8 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT 0x18404 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN_SHIFT 0x0 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU_SHIFT 0xC +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT_SHIFT 0x1C +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLM 0x18405 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE_SHIFT 0x18 +#define A6XX_CX_DBGC_CFG_DBGBUS_OPL 0x18406 +#define A6XX_CX_DBGC_CFG_DBGBUS_OPE 0x18407 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTL_0 0x18408 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTL_1 0x18409 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTL_2 0x1840A +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTL_3 0x1840B +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKL_0 0x1840C +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKL_1 0x1840D +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKL_2 0x1840E +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKL_3 0x1840F +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0 0x18410 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1 0x18411 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL0_SHIFT 0x0 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL1_SHIFT 0x4 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL2_SHIFT 0x8 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL3_SHIFT 0xC +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL4_SHIFT 0x10 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL5_SHIFT 0x14 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL6_SHIFT 0x18 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL7_SHIFT 0x1C +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL8_SHIFT 0x0 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL9_SHIFT 0x4 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL10_SHIFT 0x8 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL11_SHIFT 0xC +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL12_SHIFT 0x10 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL13_SHIFT 0x14 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL14_SHIFT 0x18 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL15_SHIFT 0x1C +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTE_0 0x18412 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTE_1 0x18413 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTE_2 0x18414 +#define A6XX_CX_DBGC_CFG_DBGBUS_IVTE_3 0x18415 +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKE_0 0x18416 +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKE_1 0x18417 +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKE_2 0x18418 +#define A6XX_CX_DBGC_CFG_DBGBUS_MASKE_3 0x18419 +#define A6XX_CX_DBGC_CFG_DBGBUS_NIBBLEE 0x1841A +#define A6XX_CX_DBGC_CFG_DBGBUS_PTRC0 0x1841B +#define A6XX_CX_DBGC_CFG_DBGBUS_PTRC1 0x1841C +#define A6XX_CX_DBGC_CFG_DBGBUS_LOADREG 0x1841D +#define A6XX_CX_DBGC_CFG_DBGBUS_IDX 0x1841E +#define A6XX_CX_DBGC_CFG_DBGBUS_CLRC 0x1841F +#define A6XX_CX_DBGC_CFG_DBGBUS_LOADIVT 0x18420 +#define A6XX_CX_DBGC_VBIF_DBG_CNTL 0x18421 +#define A6XX_CX_DBGC_DBG_LO_HI_GPIO 0x18422 +#define A6XX_CX_DBGC_EXT_TRACE_BUS_CNTL 0x18423 +#define A6XX_CX_DBGC_READ_AHB_THROUGH_DBG 0x18424 +#define A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF1 0x1842F +#define A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF2 0x18430 +#define A6XX_CX_DBGC_EVT_CFG 0x18440 +#define A6XX_CX_DBGC_EVT_INTF_SEL_0 0x18441 +#define A6XX_CX_DBGC_EVT_INTF_SEL_1 0x18442 +#define A6XX_CX_DBGC_PERF_ATB_CFG 0x18443 +#define A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_0 0x18444 +#define A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_1 0x18445 +#define A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_2 0x18446 +#define A6XX_CX_DBGC_PERF_ATB_COUNTER_SEL_3 0x18447 +#define A6XX_CX_DBGC_PERF_ATB_TRIG_INTF_SEL_0 0x18448 +#define A6XX_CX_DBGC_PERF_ATB_TRIG_INTF_SEL_1 0x18449 +#define A6XX_CX_DBGC_PERF_ATB_DRAIN_CMD 0x1844A +#define A6XX_CX_DBGC_ECO_CNTL 0x18450 +#define A6XX_CX_DBGC_AHB_DBG_CNTL 0x18451 + +/* GMU control registers */ +#define A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL 0x1A880 +#define A6XX_GMU_GX_SPTPRAC_POWER_CONTROL 0x1A881 +#define A6XX_GMU_CM3_ITCM_START 0x1B400 +#define A6XX_GMU_CM3_DTCM_START 0x1C400 +#define A6XX_GMU_NMI_CONTROL_STATUS 0x1CBF0 +#define A6XX_GMU_BOOT_SLUMBER_OPTION 0x1CBF8 +#define A6XX_GMU_GX_VOTE_IDX 0x1CBF9 +#define A6XX_GMU_MX_VOTE_IDX 0x1CBFA +#define A6XX_GMU_DCVS_ACK_OPTION 0x1CBFC +#define A6XX_GMU_DCVS_PERF_SETTING 0x1CBFD +#define A6XX_GMU_DCVS_BW_SETTING 0x1CBFE +#define A6XX_GMU_DCVS_RETURN 0x1CBFF +#define A6XX_GMU_ICACHE_CONFIG 0x1F400 +#define A6XX_GMU_DCACHE_CONFIG 0x1F401 +#define A6XX_GMU_SYS_BUS_CONFIG 0x1F40F +#define A6XX_GMU_CM3_SYSRESET 0x1F800 +#define A6XX_GMU_CM3_BOOT_CONFIG 0x1F801 +#define A6XX_GMU_CX_GMU_WFI_CONFIG 0x1F802 +#define A6XX_GMU_CX_GMU_WDOG_CTRL 0x1F813 +#define A6XX_GMU_CM3_FW_BUSY 0x1F81A +#define A6XX_GMU_CM3_FW_INIT_RESULT 0x1F81C +#define A6XX_GMU_CM3_CFG 0x1F82D +#define A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE 0x1F840 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0 0x1F841 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1 0x1F842 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L 0x1F844 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H 0x1F845 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_L 0x1F846 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_H 0x1F847 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_L 0x1F848 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_H 0x1F849 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_L 0x1F84A +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_H 0x1F84B +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_L 0x1F84C +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_H 0x1F84D +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_L 0x1F84E +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_H 0x1F84F +#define A6XX_GMU_CX_GMU_POWER_COUNTER_GMUCLK_0_L 0x1F850 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_GMUCLK_0_H 0x1F851 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_GMUCLK_1_L 0x1F852 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_GMUCLK_1_H 0x1F853 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_2 0x1F860 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_6_L 0x1F870 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_6_H 0x1F871 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_7_L 0x1F872 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_7_H 0x1F843 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_8_L 0x1F874 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_8_H 0x1F875 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_9_L 0x1F876 +#define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_9_H 0x1F877 +#define A6XX_GMU_CX_GMU_PERF_COUNTER_ENABLE 0x1F8A0 +#define A6XX_GMU_CX_GMU_PERF_COUNTER_SELECT_0 0x1F8A1 +#define A6XX_GMU_CX_GMU_PERF_COUNTER_SELECT_1 0x1F8A2 +#define A6XX_GMU_CX_GMU_PERF_COUNTER_0_L 0x1F8A4 +#define A6XX_GMU_CX_GMU_PERF_COUNTER_0_H 0x1F8A5 +#define A6XX_GMU_CX_GMU_PERF_COUNTER_1_L 0x1F8A6 +#define A6XX_GMU_CX_GMU_PERF_COUNTER_1_H 0x1F8A7 +#define A6XX_GMU_CX_GMU_PERF_COUNTER_2_L 0x1F8A8 +#define A6XX_GMU_CX_GMU_PERF_COUNTER_2_H 0x1F8A9 +#define A6XX_GMU_CX_GMU_PERF_COUNTER_3_L 0x1F8AA +#define A6XX_GMU_CX_GMU_PERF_COUNTER_3_H 0x1F8AB +#define A6XX_GMU_CX_GMU_PERF_COUNTER_4_L 0x1F8AC +#define A6XX_GMU_CX_GMU_PERF_COUNTER_4_H 0x1F8AD +#define A6XX_GMU_CX_GMU_PERF_COUNTER_5_L 0x1F8AE +#define A6XX_GMU_CX_GMU_PERF_COUNTER_5_H 0x1F8AF +#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL 0x1F8C0 +#define A6XX_GMU_PWR_COL_INTER_FRAME_HYST 0x1F8C1 +#define A6XX_GMU_PWR_COL_SPTPRAC_HYST 0x1F8C2 +#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS 0x1F8D0 +#define A6XX_GMU_GPU_NAP_CTRL 0x1F8E4 +#define A6XX_GMU_RPMH_CTRL 0x1F8E8 +#define A6XX_GMU_RPMH_HYST_CTRL 0x1F8E9 +#define A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE 0x1F8EC +#define A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG 0x1F900 +#define A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP 0x1F901 +#define A6XX_GMU_BOOT_KMD_LM_HANDSHAKE 0x1F9F0 +#define A6XX_GMU_LLM_GLM_SLEEP_CTRL 0x1F957 +#define A6XX_GMU_LLM_GLM_SLEEP_STATUS 0x1F958 + +/* HFI registers*/ +#define A6XX_GMU_ALWAYS_ON_COUNTER_L 0x1F888 +#define A6XX_GMU_ALWAYS_ON_COUNTER_H 0x1F889 +#define A6XX_GMU_GMU_PWR_COL_KEEPALIVE 0x1F8C3 +#define A6XX_GMU_PWR_COL_PREEMPT_KEEPALIVE 0x1F8C4 +#define A6XX_GMU_HFI_CTRL_STATUS 0x1F980 +#define A6XX_GMU_HFI_VERSION_INFO 0x1F981 +#define A6XX_GMU_HFI_SFR_ADDR 0x1F982 +#define A6XX_GMU_HFI_MMAP_ADDR 0x1F983 +#define A6XX_GMU_HFI_QTBL_INFO 0x1F984 +#define A6XX_GMU_HFI_QTBL_ADDR 0x1F985 +#define A6XX_GMU_HFI_CTRL_INIT 0x1F986 +#define A6XX_GMU_GMU2HOST_INTR_SET 0x1F990 +#define A6XX_GMU_GMU2HOST_INTR_CLR 0x1F991 +#define A6XX_GMU_GMU2HOST_INTR_INFO 0x1F992 +#define A6XX_GMU_GMU2HOST_INTR_MASK 0x1F993 +#define A6XX_GMU_HOST2GMU_INTR_SET 0x1F994 +#define A6XX_GMU_HOST2GMU_INTR_CLR 0x1F995 +#define A6XX_GMU_HOST2GMU_INTR_RAW_INFO 0x1F996 +#define A6XX_GMU_HOST2GMU_INTR_EN_0 0x1F997 +#define A6XX_GMU_HOST2GMU_INTR_EN_1 0x1F998 +#define A6XX_GMU_HOST2GMU_INTR_EN_2 0x1F999 +#define A6XX_GMU_HOST2GMU_INTR_EN_3 0x1F99A +#define A6XX_GMU_HOST2GMU_INTR_INFO_0 0x1F99B +#define A6XX_GMU_HOST2GMU_INTR_INFO_1 0x1F99C +#define A6XX_GMU_HOST2GMU_INTR_INFO_2 0x1F99D +#define A6XX_GMU_HOST2GMU_INTR_INFO_3 0x1F99E +#define A6XX_GMU_GENERAL_0 0x1F9C5 +#define A6XX_GMU_GENERAL_1 0x1F9C6 +#define A6XX_GMU_GENERAL_6 0x1F9CB +#define A6XX_GMU_GENERAL_7 0x1F9CC +#define A6XX_GMU_GENERAL_11 0x1F9D0 + +/* ISENSE registers */ +#define A6XX_GMU_ISENSE_CTRL 0x1F95D +#define A6XX_GPU_GMU_CX_GMU_ISENSE_CTRL 0x1f95d +#define A6XX_GPU_CS_ENABLE_REG 0x23120 + +/* LM registers */ +#define A6XX_GPU_GMU_CX_GMU_PWR_THRESHOLD 0x1F94D + +/* FAL10 veto register */ +#define A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF 0x1F8F0 +#define A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF 0x1F8F1 + +#define A6XX_GMU_AO_INTERRUPT_EN 0x23B03 +#define A6XX_GMU_AO_HOST_INTERRUPT_CLR 0x23B04 +#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS 0x23B05 +#define A6XX_GMU_AO_HOST_INTERRUPT_MASK 0x23B06 +#define A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL 0x23B09 +#define A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL 0x23B0A +#define A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL 0x23B0B +#define A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS 0x23B0C +#define A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2 0x23B0D +#define A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK 0x23B0E +#define A6XX_GMU_AO_AHB_FENCE_CTRL 0x23B10 +#define A6XX_GMU_AHB_FENCE_STATUS 0x23B13 +#define A6XX_GMU_AHB_FENCE_STATUS_CLR 0x23B14 +#define A6XX_GMU_RBBM_INT_UNMASKED_STATUS 0x23B15 +#define A6XX_GMU_AO_SPARE_CNTL 0x23B16 + +/* RGMU GLM registers */ +#define A6XX_GMU_AO_RGMU_GLM_SLEEP_CTRL 0x23B80 +#define A6XX_GMU_AO_RGMU_GLM_SLEEP_STATUS 0x23B81 +#define A6XX_GMU_AO_RGMU_GLM_HW_CRC_DISABLE 0x23B82 + +/* GMU RSC control registers */ +#define A6XX_GMU_RSCC_CONTROL_REQ 0x23B07 +#define A6XX_GMU_RSCC_CONTROL_ACK 0x23B08 + +/* FENCE control registers */ +#define A6XX_GMU_AHB_FENCE_RANGE_0 0x23B11 +#define A6XX_GMU_AHB_FENCE_RANGE_1 0x23B12 + +/* GMU countables */ +#define A6XX_GMU_CM3_BUSY_CYCLES 0 + +/* GPUCC registers */ +#define A6XX_GPU_CC_GX_GDSCR 0x24403 +#define A6XX_GPU_CC_GX_DOMAIN_MISC 0x24542 +#define A6XX_GPU_CC_GX_DOMAIN_MISC3 0x24563 +#define A6XX_GPU_CC_CX_CFG_GDSCR 0x2441C + +/* GPUCC offsets are different for A662 */ +#define A662_GPU_CC_GX_GDSCR 0x26417 +#define A662_GPU_CC_GX_DOMAIN_MISC3 0x26541 +#define A662_GPU_CC_CX_CFG_GDSCR 0x26443 + +/* GPU CPR registers */ +#define A6XX_GPU_CPR_FSM_CTL 0x26801 + +/* GPU RSC sequencer registers */ +#define A6XX_GPU_RSCC_RSC_STATUS0_DRV0 0x00004 +#define A6XX_RSCC_PDC_SEQ_START_ADDR 0x00008 +#define A6XX_RSCC_PDC_MATCH_VALUE_LO 0x00009 +#define A6XX_RSCC_PDC_MATCH_VALUE_HI 0x0000A +#define A6XX_RSCC_PDC_SLAVE_ID_DRV0 0x0000B +#define A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR 0x0000D +#define A6XX_RSCC_HIDDEN_TCS_CMD0_DATA 0x0000E +#define A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_L_DRV0 0x00082 +#define A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_H_DRV0 0x00083 +#define A6XX_RSCC_TIMESTAMP_UNIT1_EN_DRV0 0x00089 +#define A6XX_RSCC_TIMESTAMP_UNIT1_OUTPUT_DRV0 0x0008C +#define A6XX_RSCC_OVERRIDE_START_ADDR 0x00100 +#define A6XX_RSCC_SEQ_BUSY_DRV0 0x00101 +#define A6XX_RSCC_SEQ_MEM_0_DRV0 0x00180 +#define A6XX_RSCC_TCS0_DRV0_STATUS 0x00346 +#define A6XX_RSCC_TCS1_DRV0_STATUS 0x003EE +#define A6XX_RSCC_TCS2_DRV0_STATUS 0x00496 +#define A6XX_RSCC_TCS3_DRV0_STATUS 0x0053E +#define A6XX_RSCC_TCS4_DRV0_STATUS 0x005E6 +#define A6XX_RSCC_TCS5_DRV0_STATUS 0x0068E +#define A6XX_RSCC_TCS6_DRV0_STATUS 0x00736 +#define A6XX_RSCC_TCS7_DRV0_STATUS 0x007DE +#define A6XX_RSCC_TCS8_DRV0_STATUS 0x00886 +#define A6XX_RSCC_TCS9_DRV0_STATUS 0x0092E + +/* GPU PDC sequencer registers in AOSS.RPMh domain */ +#define PDC_GPU_ENABLE_PDC 0x1140 +#define PDC_GPU_SEQ_START_ADDR 0x1148 +#define PDC_GPU_TCS0_CONTROL 0x1540 +#define PDC_GPU_TCS0_CMD_ENABLE_BANK 0x1541 +#define PDC_GPU_TCS0_CMD_WAIT_FOR_CMPL_BANK 0x1542 +#define PDC_GPU_TCS0_CMD0_MSGID 0x1543 +#define PDC_GPU_TCS0_CMD0_ADDR 0x1544 +#define PDC_GPU_TCS0_CMD0_DATA 0x1545 +#define PDC_GPU_TCS1_CONTROL 0x1572 +#define PDC_GPU_TCS1_CMD_ENABLE_BANK 0x1573 +#define PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK 0x1574 +#define PDC_GPU_TCS1_CMD0_MSGID 0x1575 +#define PDC_GPU_TCS1_CMD0_ADDR 0x1576 +#define PDC_GPU_TCS1_CMD0_DATA 0x1577 +#define PDC_GPU_TCS2_CONTROL 0x15A4 +#define PDC_GPU_TCS2_CMD_ENABLE_BANK 0x15A5 +#define PDC_GPU_TCS2_CMD_WAIT_FOR_CMPL_BANK 0x15A6 +#define PDC_GPU_TCS2_CMD0_MSGID 0x15A7 +#define PDC_GPU_TCS2_CMD0_ADDR 0x15A8 +#define PDC_GPU_TCS2_CMD0_DATA 0x15A9 +#define PDC_GPU_TCS3_CONTROL 0x15D6 +#define PDC_GPU_TCS3_CMD_ENABLE_BANK 0x15D7 +#define PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK 0x15D8 +#define PDC_GPU_TCS3_CMD0_MSGID 0x15D9 +#define PDC_GPU_TCS3_CMD0_ADDR 0x15DA +#define PDC_GPU_TCS3_CMD0_DATA 0x15DB + +/* + * Legacy DTSI used an offset from the start of the PDC resource + * for PDC SEQ programming. We are now using PDC subsections so + * start the PDC SEQ offset at zero. + */ +#define PDC_GPU_SEQ_MEM_0 0x0 + +/* + * Legacy RSCC register range was a part of the GMU register space + * now we are using a separate section for RSCC regsiters. Add the + * offset for backward compatibility. + */ +#define RSCC_OFFSET_LEGACY 0x23400 + +/* RGMU(PCC) registers in A6X_GMU_CX_0_NON_CONTEXT_DEC domain */ +#define A6XX_RGMU_CX_INTR_GEN_EN 0x1F80F +#define A6XX_RGMU_CX_RGMU_TIMER0 0x1F834 +#define A6XX_RGMU_CX_RGMU_TIMER1 0x1F835 +#define A6XX_RGMU_CX_PCC_CTRL 0x1F838 +#define A6XX_RGMU_CX_PCC_INIT_RESULT 0x1F839 +#define A6XX_RGMU_CX_PCC_BKPT_CFG 0x1F83A +#define A6XX_RGMU_CX_PCC_BKPT_ADDR 0x1F83B +#define A6XX_RGMU_CX_PCC_STATUS 0x1F83C +#define A6XX_RGMU_CX_PCC_DEBUG 0x1F83D + +/* GPU CX_MISC registers */ +#define A6XX_GPU_CX_MISC_SYSTEM_CACHE_CNTL_0 0x1 +#define A6XX_GPU_CX_MISC_SYSTEM_CACHE_CNTL_1 0x2 +#define A6XX_LLC_NUM_GPU_SCIDS 5 +#define A6XX_GPU_LLC_SCID_NUM_BITS 5 +#define A6XX_GPU_LLC_SCID_MASK \ + ((1 << (A6XX_LLC_NUM_GPU_SCIDS * A6XX_GPU_LLC_SCID_NUM_BITS)) - 1) +#define A6XX_GPUHTW_LLC_SCID_SHIFT 25 +#define A6XX_GPUHTW_LLC_SCID_MASK \ + (((1 << A6XX_GPU_LLC_SCID_NUM_BITS) - 1) << A6XX_GPUHTW_LLC_SCID_SHIFT) + +/* FUSA registers */ +#define A6XX_GPU_FUSA_REG_BASE 0x3FC00 +#define A6XX_GPU_FUSA_REG_ECC_CTRL 0x3FC00 +#define A6XX_GPU_FUSA_REG_CSR_PRIY 0x3FC52 +#define A6XX_GPU_FUSA_DISABLE_NUM_BITS 4 +#define A6XX_GPU_FUSA_DISABLE_BITS 0x5 +#define A6XX_GPU_FUSA_DISABLE_MASK \ + ((1 << A6XX_GPU_FUSA_DISABLE_NUM_BITS) - 1) + +#endif /* _A6XX_REG_H */ + diff --git a/qcom/opensource/graphics-kernel/adreno-gpulist.h b/qcom/opensource/graphics-kernel/adreno-gpulist.h new file mode 100644 index 0000000000..6834f5da37 --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno-gpulist.h @@ -0,0 +1,2839 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2002,2007-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define ANY_ID (~0) + +#define DEFINE_ADRENO_REV(_rev, _core, _major, _minor, _patchid) \ + .gpurev = _rev, .core = _core, .major = _major, .minor = _minor, \ + .patchid = _patchid + +#define DEFINE_DEPRECATED_CORE(_name, _rev, _core, _major, _minor, _patchid) \ +static const struct adreno_gpu_core adreno_gpu_core_##_name = { \ + DEFINE_ADRENO_REV(_rev, _core, _major, _minor, _patchid), \ + .features = ADRENO_DEPRECATED, \ +} + +#define MHZ_TO_KBPS(mhz, w) ((u64)(mhz * 1000000ULL * w) / (1024)) + +static const struct kgsl_regmap_list a306_vbif_regs[] = { + { A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003 }, + { A3XX_VBIF_OUT_RD_LIM_CONF0, 0x0000000A }, + { A3XX_VBIF_OUT_WR_LIM_CONF0, 0x0000000A }, +}; + +static const struct adreno_a3xx_core adreno_gpu_core_a306 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A306, 3, 0, 6, 0), + .features = ADRENO_SOFT_FAULT_DETECT, + .gpudev = &adreno_a3xx_gpudev, + .perfcounters = &adreno_a3xx_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_128K, + .bus_width = 0, + .snapshot_size = 600 * SZ_1K, + }, + .pm4fw_name = "a300_pm4.fw", + .pfpfw_name = "a300_pfp.fw", + .vbif = a306_vbif_regs, + .vbif_count = ARRAY_SIZE(a306_vbif_regs), +}; + +static const struct kgsl_regmap_list a306a_vbif_regs[] = { + { A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003 }, + { A3XX_VBIF_OUT_RD_LIM_CONF0, 0x00000010 }, + { A3XX_VBIF_OUT_WR_LIM_CONF0, 0x00000010 }, +}; + +static const struct adreno_a3xx_core adreno_gpu_core_a306a = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A306A, 3, 0, 6, 0x20), + .features = ADRENO_SOFT_FAULT_DETECT, + .gpudev = &adreno_a3xx_gpudev, + .perfcounters = &adreno_a3xx_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_128K, + .bus_width = 16, + .snapshot_size = 600 * SZ_1K, + }, + .pm4fw_name = "a300_pm4.fw", + .pfpfw_name = "a300_pfp.fw", + .vbif = a306a_vbif_regs, + .vbif_count = ARRAY_SIZE(a306a_vbif_regs), +}; + +static const struct kgsl_regmap_list a304_vbif_regs[] = { + { A3XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x0003 }, +}; + +static const struct adreno_a3xx_core adreno_gpu_core_a304 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A304, 3, 0, 4, 0), + .features = ADRENO_SOFT_FAULT_DETECT, + .gpudev = &adreno_a3xx_gpudev, + .perfcounters = &adreno_a3xx_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = (SZ_64K + SZ_32K), + .bus_width = 0, + .snapshot_size = 600 * SZ_1K, + }, + .pm4fw_name = "a300_pm4.fw", + .pfpfw_name = "a300_pfp.fw", + .vbif = a304_vbif_regs, + .vbif_count = ARRAY_SIZE(a304_vbif_regs), +}; + +DEFINE_DEPRECATED_CORE(a405, ADRENO_REV_A405, 4, 0, 5, ANY_ID); +DEFINE_DEPRECATED_CORE(a418, ADRENO_REV_A418, 4, 1, 8, ANY_ID); +DEFINE_DEPRECATED_CORE(a420, ADRENO_REV_A420, 4, 2, 0, ANY_ID); +DEFINE_DEPRECATED_CORE(a430, ADRENO_REV_A430, 4, 3, 0, ANY_ID); +DEFINE_DEPRECATED_CORE(a530v1, ADRENO_REV_A530, 5, 3, 0, 0); + +static const struct kgsl_regmap_list a530_hwcg_regs[] = { + {A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220}, + {A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, + {A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777}, + {A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111}, + {A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, + {A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, +}; + +/* VBIF control registers for a530, a510, a508, a505 and a506 */ +static const struct kgsl_regmap_list a530_vbif_regs[] = { + {A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003}, +}; + +static const struct adreno_a5xx_core adreno_gpu_core_a530v2 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A530, 5, 3, 0, 1), + .features = ADRENO_SPTP_PC | ADRENO_LM | + ADRENO_PREEMPTION | + ADRENO_CONTENT_PROTECTION, + .gpudev = &adreno_a5xx_gpudev, + .perfcounters = &adreno_a5xx_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_1M, + .bus_width = 32, + .snapshot_size = SZ_1M, + }, + .gpmu_tsens = 0x00060007, + .max_power = 5448, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .gpmufw_name = "a530_gpmu.fw2", + .regfw_name = "a530v2_seq.fw2", + .zap_name = "a530_zap.mdt", + .hwcg = a530_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a530_hwcg_regs), + .vbif = a530_vbif_regs, + .vbif_count = ARRAY_SIZE(a530_vbif_regs), + .highest_bank_bit = 15, +}; + +static const struct adreno_a5xx_core adreno_gpu_core_a530v3 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A530, 5, 3, 0, ANY_ID), + .features = ADRENO_SPTP_PC | ADRENO_LM | + ADRENO_PREEMPTION | + ADRENO_CONTENT_PROTECTION, + .gpudev = &adreno_a5xx_gpudev, + .perfcounters = &adreno_a5xx_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_1M, + .bus_width = 32, + .snapshot_size = SZ_1M, + }, + .gpmu_tsens = 0x00060007, + .max_power = 5448, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .gpmufw_name = "a530v3_gpmu.fw2", + .regfw_name = "a530v3_seq.fw2", + .zap_name = "a530_zap.mdt", + .hwcg = a530_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a530_hwcg_regs), + .vbif = a530_vbif_regs, + .vbif_count = ARRAY_SIZE(a530_vbif_regs), + .highest_bank_bit = 15, +}; + +/* For a505, a506 and a508 */ +static const struct kgsl_regmap_list a50x_hwcg_regs[] = { + {A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_HYST_UCHE, 0x00FFFFF4}, + {A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222} +}; + +static const struct adreno_a5xx_core adreno_gpu_core_a505 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A505, 5, 0, 5, ANY_ID), + .features = ADRENO_PREEMPTION, + .gpudev = &adreno_a5xx_gpudev, + .perfcounters = &adreno_a5xx_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = (SZ_128K + SZ_8K), + .bus_width = 16, + .snapshot_size = SZ_1M, + }, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .hwcg = a50x_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a50x_hwcg_regs), + .vbif = a530_vbif_regs, + .vbif_count = ARRAY_SIZE(a530_vbif_regs), +}; + +static const struct adreno_a5xx_core adreno_gpu_core_a506 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A506, 5, 0, 6, ANY_ID), + .features = ADRENO_PREEMPTION | + ADRENO_CONTENT_PROTECTION | ADRENO_CPZ_RETENTION, + .gpudev = &adreno_a5xx_gpudev, + .perfcounters = &adreno_a5xx_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = (SZ_128K + SZ_8K), + .bus_width = 16, + .snapshot_size = SZ_1M, + }, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .zap_name = "a506_zap.mdt", + .hwcg = a50x_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a50x_hwcg_regs), + .vbif = a530_vbif_regs, + .vbif_count = ARRAY_SIZE(a530_vbif_regs), + .highest_bank_bit = 14, +}; + +static const struct kgsl_regmap_list a510_hwcg_regs[] = { + {A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, + {A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, + {A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, + {A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, + {A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, +}; + +static const struct adreno_a5xx_core adreno_gpu_core_a510 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A510, 5, 1, 0, ANY_ID), + .gpudev = &adreno_a5xx_gpudev, + .perfcounters = &adreno_a5xx_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_256K, + .bus_width = 16, + .snapshot_size = SZ_1M, + }, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .hwcg = a510_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a510_hwcg_regs), + .vbif = a530_vbif_regs, + .vbif_count = ARRAY_SIZE(a530_vbif_regs), +}; + +DEFINE_DEPRECATED_CORE(a540v1, ADRENO_REV_A540, 5, 4, 0, 0); + +static const struct kgsl_regmap_list a540_hwcg_regs[] = { + {A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220}, + {A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, + {A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777}, + {A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111}, + {A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, + {A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_GPMU, 0x00000222}, + {A5XX_RBBM_CLOCK_DELAY_GPMU, 0x00000770}, + {A5XX_RBBM_CLOCK_HYST_GPMU, 0x00000004}, +}; + +static const struct kgsl_regmap_list a540_vbif_regs[] = { + {A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003}, + {A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009}, +}; + +static const struct adreno_a5xx_core adreno_gpu_core_a540v2 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A540, 5, 4, 0, ANY_ID), + .features = ADRENO_PREEMPTION | + ADRENO_CONTENT_PROTECTION | + ADRENO_SPTP_PC, + .gpudev = &adreno_a5xx_gpudev, + .perfcounters = &adreno_a5xx_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_1M, + .bus_width = 32, + .snapshot_size = SZ_1M, + }, + .gpmu_tsens = 0x000c000d, + .max_power = 5448, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .gpmufw_name = "a540_gpmu.fw2", + .zap_name = "a540_zap.mdt", + .hwcg = a540_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a540_hwcg_regs), + .vbif = a540_vbif_regs, + .vbif_count = ARRAY_SIZE(a540_vbif_regs), + .highest_bank_bit = 15, +}; + +static const struct kgsl_regmap_list a512_hwcg_regs[] = { + {A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, + {A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220}, + {A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, + {A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222}, + {A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777}, + {A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777}, + {A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111}, + {A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111}, + {A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444}, + {A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222}, + {A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220}, + {A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222}, + {A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404}, + {A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002}, + {A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011}, + {A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, +}; + +static const struct adreno_a5xx_core adreno_gpu_core_a512 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A512, 5, 1, 2, ANY_ID), + .features = ADRENO_PREEMPTION | + ADRENO_CONTENT_PROTECTION | ADRENO_CPZ_RETENTION, + .gpudev = &adreno_a5xx_gpudev, + .perfcounters = &adreno_a5xx_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = (SZ_256K + SZ_16K), + .bus_width = 32, + .snapshot_size = SZ_1M, + }, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .zap_name = "a512_zap.mdt", + .hwcg = a512_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a512_hwcg_regs), + .highest_bank_bit = 14, +}; + +static const struct adreno_a5xx_core adreno_gpu_core_a508 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A508, 5, 0, 8, ANY_ID), + .features = ADRENO_PREEMPTION | + ADRENO_CONTENT_PROTECTION | ADRENO_CPZ_RETENTION, + .gpudev = &adreno_a5xx_gpudev, + .perfcounters = &adreno_a5xx_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = (SZ_128K + SZ_8K), + .bus_width = 32, + .snapshot_size = SZ_1M, + }, + .pm4fw_name = "a530_pm4.fw", + .pfpfw_name = "a530_pfp.fw", + .zap_name = "a508_zap.mdt", + .hwcg = a50x_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a50x_hwcg_regs), + .vbif = a530_vbif_regs, + .vbif_count = ARRAY_SIZE(a530_vbif_regs), + .highest_bank_bit = 14, +}; + +DEFINE_DEPRECATED_CORE(a630v1, ADRENO_REV_A630, 6, 3, 0, 0); + +static const struct kgsl_regmap_list a630_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_SP1, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_SP2, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_SP3, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220}, + {A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220}, + {A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220}, + {A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, + {A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222}, + {A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222}, + {A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777}, + {A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777}, + {A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111}, + {A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111}, + {A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, +}; + +static const struct kgsl_regmap_list a630_vbif_regs[] = { + {A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009}, + {A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3}, +}; + + +/* For a615, a616, a618, A619, a630, a640 and a680 */ +static const struct adreno_protected_regs a630_protected_regs[] = { + { A6XX_CP_PROTECT_REG + 0, 0x00000, 0x004ff, 0 }, + { A6XX_CP_PROTECT_REG + 1, 0x00501, 0x00506, 0 }, + { A6XX_CP_PROTECT_REG + 2, 0x0050b, 0x007ff, 0 }, + { A6XX_CP_PROTECT_REG + 3, 0x0050e, 0x0050e, 1 }, + { A6XX_CP_PROTECT_REG + 4, 0x00510, 0x00510, 1 }, + { A6XX_CP_PROTECT_REG + 5, 0x00534, 0x00534, 1 }, + { A6XX_CP_PROTECT_REG + 6, 0x00800, 0x00882, 1 }, + { A6XX_CP_PROTECT_REG + 7, 0x008a0, 0x008a8, 1 }, + { A6XX_CP_PROTECT_REG + 8, 0x008ab, 0x008cf, 1 }, + { A6XX_CP_PROTECT_REG + 9, 0x008d0, 0x0098c, 0 }, + { A6XX_CP_PROTECT_REG + 10, 0x00900, 0x0094d, 1 }, + { A6XX_CP_PROTECT_REG + 11, 0x0098d, 0x00bff, 1 }, + { A6XX_CP_PROTECT_REG + 12, 0x00e00, 0x00e01, 1 }, + { A6XX_CP_PROTECT_REG + 13, 0x00e03, 0x00e0f, 1 }, + { A6XX_CP_PROTECT_REG + 14, 0x03c00, 0x03cc3, 1 }, + { A6XX_CP_PROTECT_REG + 15, 0x03cc4, 0x05cc3, 0 }, + { A6XX_CP_PROTECT_REG + 16, 0x08630, 0x087ff, 1 }, + { A6XX_CP_PROTECT_REG + 17, 0x08e00, 0x08e00, 1 }, + { A6XX_CP_PROTECT_REG + 18, 0x08e08, 0x08e08, 1 }, + { A6XX_CP_PROTECT_REG + 19, 0x08e50, 0x08e6f, 1 }, + { A6XX_CP_PROTECT_REG + 20, 0x09624, 0x097ff, 1 }, + { A6XX_CP_PROTECT_REG + 21, 0x09e70, 0x09e71, 1 }, + { A6XX_CP_PROTECT_REG + 22, 0x09e78, 0x09fff, 1 }, + { A6XX_CP_PROTECT_REG + 23, 0x0a630, 0x0a7ff, 1 }, + { A6XX_CP_PROTECT_REG + 24, 0x0ae02, 0x0ae02, 1 }, + { A6XX_CP_PROTECT_REG + 25, 0x0ae50, 0x0b17f, 1 }, + { A6XX_CP_PROTECT_REG + 26, 0x0b604, 0x0b604, 1 }, + { A6XX_CP_PROTECT_REG + 27, 0x0be02, 0x0be03, 1 }, + { A6XX_CP_PROTECT_REG + 28, 0x0be20, 0x0d5ff, 1 }, + { A6XX_CP_PROTECT_REG + 29, 0x0f000, 0x0fbff, 1 }, + { A6XX_CP_PROTECT_REG + 30, 0x0fc00, 0x11bff, 0 }, + { A6XX_CP_PROTECT_REG + 31, 0x11c00, 0x11c00, 1 }, + { 0 }, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a630v2 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A630, 6, 3, 0, ANY_ID), + .features = ADRENO_IFPC | ADRENO_CONTENT_PROTECTION | + ADRENO_IOCOHERENT | ADRENO_PREEMPTION, + .gpudev = &adreno_a630_gpudev.base, + .perfcounters = &adreno_a630_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_1M, + .bus_width = 32, + .snapshot_size = SZ_1M, + }, + .prim_fifo_threshold = 0x0018000, + .gmu_major = 1, + .gmu_minor = 3, + .sqefw_name = "a630_sqe.fw", + .gmufw_name = "a630_gmu.bin", + .zap_name = "a630_zap.mdt", + .hwcg = a630_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a630_hwcg_regs), + .vbif = a630_vbif_regs, + .vbif_count = ARRAY_SIZE(a630_vbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = a630_protected_regs, + .highest_bank_bit = 15, +}; + +/* For a615, a616, a618 and a619 */ +static const struct kgsl_regmap_list a615_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220}, + {A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040F00}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555} +}; + +/* For a615, a616, a618 and a619 */ +static const struct kgsl_regmap_list a615_gbif_regs[] = { + {A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3}, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a615 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A615, 6, 1, 5, ANY_ID), + .features = ADRENO_PREEMPTION | + ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | + ADRENO_IOCOHERENT, + .gpudev = &adreno_a630_gpudev.base, + .perfcounters = &adreno_a6xx_legacy_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_512K, + .bus_width = 32, + .snapshot_size = 600 * SZ_1K, + }, + .prim_fifo_threshold = 0x0018000, + .gmu_major = 1, + .gmu_minor = 3, + .sqefw_name = "a630_sqe.fw", + .gmufw_name = "a630_gmu.bin", + .zap_name = "a615_zap.mdt", + .hwcg = a615_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a615_hwcg_regs), + .vbif = a615_gbif_regs, + .vbif_count = ARRAY_SIZE(a615_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = a630_protected_regs, + .highest_bank_bit = 14, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a618 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A618, 6, 1, 8, ANY_ID), + .features = ADRENO_PREEMPTION | + ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | + ADRENO_IOCOHERENT, + .gpudev = &adreno_a630_gpudev.base, + .perfcounters = &adreno_a6xx_legacy_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_512K, + .bus_width = 32, + .snapshot_size = SZ_1M, + }, + .prim_fifo_threshold = 0x0018000, + .gmu_major = 1, + .gmu_minor = 7, + .sqefw_name = "a630_sqe.fw", + .gmufw_name = "a630_gmu.bin", + .zap_name = "a615_zap.mdt", + .hwcg = a615_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a615_hwcg_regs), + .vbif = a615_gbif_regs, + .vbif_count = ARRAY_SIZE(a615_gbif_regs), + .hang_detect_cycles = 0x3fffff, + .protected_regs = a630_protected_regs, + .highest_bank_bit = 14, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a619 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A619, 6, 1, 9, ANY_ID), + .features = ADRENO_PREEMPTION | + ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | + ADRENO_IOCOHERENT, + .gpudev = &adreno_a630_gpudev.base, + .perfcounters = &adreno_a6xx_legacy_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_512K, + .bus_width = 32, + .snapshot_size = SZ_2M, + }, + .prim_fifo_threshold = 0x0018000, + .gmu_major = 1, + .gmu_minor = 9, + .sqefw_name = "a630_sqe.fw", + .gmufw_name = "a619_gmu.bin", + .zap_name = "a615_zap.mdt", + .hwcg = a615_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a615_hwcg_regs), + .vbif = a615_gbif_regs, + .vbif_count = ARRAY_SIZE(a615_gbif_regs), + .hang_detect_cycles = 0x3fffff, + .protected_regs = a630_protected_regs, + .highest_bank_bit = 14, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a619_variant = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A619, 6, 1, 9, ANY_ID), + .compatible = "qcom,adreno-gpu-a619-holi", + .features = ADRENO_PREEMPTION | ADRENO_CONTENT_PROTECTION, + .gpudev = &adreno_a619_holi_gpudev, + .perfcounters = &adreno_a6xx_legacy_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_512K, + .bus_width = 32, + .snapshot_size = SZ_2M, + }, + .prim_fifo_threshold = 0x0018000, + .sqefw_name = "a630_sqe.fw", + .zap_name = "gen6_3_25_0_zap.mdt", + .hwcg = a615_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a615_hwcg_regs), + .vbif = a615_gbif_regs, + .vbif_count = ARRAY_SIZE(a615_gbif_regs), + .hang_detect_cycles = 0x3fffff, + .protected_regs = a630_protected_regs, + .gx_cpr_toggle = true, + .highest_bank_bit = 14, +}; + +static const struct kgsl_regmap_list a620_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000777}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_ISDB_CNT, 0x00000182}, + {A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, + {A6XX_RBBM_SP_HYST_CNT, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, +}; + +/* a620, a621 and a650 */ +static const struct kgsl_regmap_list a650_gbif_regs[] = { + {A6XX_GBIF_QSB_SIDE0, 0x00071620}, + {A6XX_GBIF_QSB_SIDE1, 0x00071620}, + {A6XX_GBIF_QSB_SIDE2, 0x00071620}, + {A6XX_GBIF_QSB_SIDE3, 0x00071620}, + {A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x3}, +}; + +/* These are for a620, a621 and a650 */ +static const struct adreno_protected_regs a620_protected_regs[] = { + { A6XX_CP_PROTECT_REG + 0, 0x00000, 0x004ff, 0 }, + { A6XX_CP_PROTECT_REG + 1, 0x00501, 0x00506, 0 }, + { A6XX_CP_PROTECT_REG + 2, 0x0050b, 0x007ff, 0 }, + { A6XX_CP_PROTECT_REG + 3, 0x0050e, 0x0050e, 1 }, + { A6XX_CP_PROTECT_REG + 4, 0x00510, 0x00510, 1 }, + { A6XX_CP_PROTECT_REG + 5, 0x00534, 0x00534, 1 }, + { A6XX_CP_PROTECT_REG + 6, 0x00800, 0x00882, 1 }, + { A6XX_CP_PROTECT_REG + 7, 0x008a0, 0x008a8, 1 }, + { A6XX_CP_PROTECT_REG + 8, 0x008ab, 0x008cf, 1 }, + { A6XX_CP_PROTECT_REG + 9, 0x008d0, 0x0098c, 0 }, + { A6XX_CP_PROTECT_REG + 10, 0x00900, 0x0094d, 1 }, + { A6XX_CP_PROTECT_REG + 11, 0x0098d, 0x00bff, 1 }, + { A6XX_CP_PROTECT_REG + 12, 0x00e00, 0x00e01, 1 }, + { A6XX_CP_PROTECT_REG + 13, 0x00e03, 0x00e0f, 1 }, + { A6XX_CP_PROTECT_REG + 14, 0x03c00, 0x03cc3, 1 }, + { A6XX_CP_PROTECT_REG + 15, 0x03cc4, 0x05cc3, 0 }, + { A6XX_CP_PROTECT_REG + 16, 0x08630, 0x087ff, 1 }, + { A6XX_CP_PROTECT_REG + 17, 0x08e00, 0x08e00, 1 }, + { A6XX_CP_PROTECT_REG + 18, 0x08e08, 0x08e08, 1 }, + { A6XX_CP_PROTECT_REG + 19, 0x08e50, 0x08e6f, 1 }, + { A6XX_CP_PROTECT_REG + 20, 0x08e80, 0x090ff, 1 }, + { A6XX_CP_PROTECT_REG + 21, 0x09624, 0x097ff, 1 }, + { A6XX_CP_PROTECT_REG + 22, 0x09e60, 0x09e71, 1 }, + { A6XX_CP_PROTECT_REG + 23, 0x09e78, 0x09fff, 1 }, + { A6XX_CP_PROTECT_REG + 24, 0x0a630, 0x0a7ff, 1 }, + { A6XX_CP_PROTECT_REG + 25, 0x0ae02, 0x0ae02, 1 }, + { A6XX_CP_PROTECT_REG + 26, 0x0ae50, 0x0b17f, 1 }, + { A6XX_CP_PROTECT_REG + 27, 0x0b604, 0x0b604, 1 }, + { A6XX_CP_PROTECT_REG + 28, 0x0b608, 0x0b60f, 1 }, + { A6XX_CP_PROTECT_REG + 29, 0x0be02, 0x0be03, 1 }, + { A6XX_CP_PROTECT_REG + 30, 0x0be20, 0x0d5ff, 1 }, + { A6XX_CP_PROTECT_REG + 31, 0x0f000, 0x0fbff, 1 }, + { A6XX_CP_PROTECT_REG + 32, 0x0fc00, 0x11bff, 0 }, + { A6XX_CP_PROTECT_REG + 33, 0x18400, 0x1a3ff, 1 }, + { A6XX_CP_PROTECT_REG + 34, 0x1a800, 0x1c7ff, 1 }, + { A6XX_CP_PROTECT_REG + 35, 0x1c800, 0x1e7ff, 1 }, + { A6XX_CP_PROTECT_REG + 36, 0x1f400, 0x1f843, 1 }, + { A6XX_CP_PROTECT_REG + 37, 0x1f844, 0x1f8bf, 0 }, + { A6XX_CP_PROTECT_REG + 38, 0x1f887, 0x1f8a2, 1 }, + { A6XX_CP_PROTECT_REG + 47, 0x1f8c0, 0x1f8c0, 1 }, + { 0 }, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a620 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A620, 6, 2, 0, ANY_ID), + .features = ADRENO_CONTENT_PROTECTION | ADRENO_IOCOHERENT | + ADRENO_IFPC | ADRENO_PREEMPTION | ADRENO_ACD | + ADRENO_APRIV, + .gpudev = &adreno_a630_gpudev.base, + .perfcounters = &adreno_a6xx_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_512K, + .bus_width = 32, + .snapshot_size = 2 * SZ_1M, + }, + .prim_fifo_threshold = 0x0010000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a650_sqe.fw", + .gmufw_name = "a650_gmu.bin", + .zap_name = "a620_zap.mdt", + .hwcg = a620_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a620_hwcg_regs), + .vbif = a650_gbif_regs, + .vbif_count = ARRAY_SIZE(a650_gbif_regs), + .veto_fal10 = true, + .hang_detect_cycles = 0x3ffff, + .protected_regs = a620_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 14, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a621 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A621, 6, 2, 1, ANY_ID), + .compatible = "qcom,adreno-gpu-a621", + .features = ADRENO_CONTENT_PROTECTION | ADRENO_IOCOHERENT | + ADRENO_APRIV | ADRENO_LSR | ADRENO_PREEMPTION | + ADRENO_IFPC, + .gpudev = &adreno_a6xx_hwsched_gpudev.base, + .perfcounters = &adreno_a6xx_hwsched_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_512K, + .bus_width = 32, + .snapshot_size = 2 * SZ_1M, + }, + .prim_fifo_threshold = 0x0010000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a650_sqe.fw", + .gmufw_name = "a621_gmu.bin", + .zap_name = "a620_zap.mdt", + .hwcg = a620_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a620_hwcg_regs), + .vbif = a650_gbif_regs, + .vbif_count = ARRAY_SIZE(a650_gbif_regs), + .veto_fal10 = true, + .pdc_in_aop = true, + .hang_detect_cycles = 0x3ffff, + .protected_regs = a620_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 13, +}; + +static const struct kgsl_regmap_list a640_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x05222022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_ISDB_CNT, 0x00000182}, + {A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, + {A6XX_RBBM_SP_HYST_CNT, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, +}; + +static const struct kgsl_regmap_list a680_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x05222022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_CLOCK_CNTL, 0x8AA8AA82}, + {A6XX_RBBM_ISDB_CNT, 0x00000182}, + {A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, + {A6XX_RBBM_SP_HYST_CNT, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, + {A6XX_GMUGX_GMU_SP_RF_CONTROL_0, 0x00000001}, + {A6XX_GMUGX_GMU_SP_RF_CONTROL_1, 0x00000001}, +}; + +/* These apply to a640, a680, a612, a610 and a702 */ +static const struct kgsl_regmap_list a640_vbif_regs[] = { + {A6XX_GBIF_QSB_SIDE0, 0x00071620}, + {A6XX_GBIF_QSB_SIDE1, 0x00071620}, + {A6XX_GBIF_QSB_SIDE2, 0x00071620}, + {A6XX_GBIF_QSB_SIDE3, 0x00071620}, + {A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x3}, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a640 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A640, 6, 4, 0, ANY_ID), + .features = ADRENO_CONTENT_PROTECTION | ADRENO_IOCOHERENT | + ADRENO_IFPC | ADRENO_PREEMPTION | ADRENO_L3_VOTE, + .gpudev = &adreno_a6xx_gmu_gpudev.base, + .perfcounters = &adreno_a6xx_legacy_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_1M, //Verified 1MB + .bus_width = 32, + .snapshot_size = 2 * SZ_1M, + }, + .prim_fifo_threshold = 0x00200000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a630_sqe.fw", + .gmufw_name = "a640_gmu.bin", + .zap_name = "a640_zap.mdt", + .hwcg = a640_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a640_hwcg_regs), + .vbif = a640_vbif_regs, + .vbif_count = ARRAY_SIZE(a640_vbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = a630_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 15, +}; + +static const struct kgsl_regmap_list a650_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000777}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_ISDB_CNT, 0x00000182}, + {A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, + {A6XX_RBBM_SP_HYST_CNT, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a650 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A650, 6, 5, 0, 0), + .features = ADRENO_IOCOHERENT | ADRENO_CONTENT_PROTECTION | + ADRENO_IFPC | ADRENO_APRIV | ADRENO_L3_VOTE, + .gpudev = &adreno_a6xx_gmu_gpudev.base, + .perfcounters = &adreno_a6xx_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_1M + SZ_128K, /* verified 1152kB */ + .bus_width = 32, + .snapshot_size = 2 * SZ_1M, + }, + .prim_fifo_threshold = 0x00300000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a650_sqe.fw", + .gmufw_name = "a650_gmu.bin", + .zap_name = "a650_zap.mdt", + .hwcg = a650_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a650_hwcg_regs), + .vbif = a650_gbif_regs, + .vbif_count = ARRAY_SIZE(a650_gbif_regs), + .veto_fal10 = true, + .pdc_in_aop = true, + .hang_detect_cycles = 0xcfffff, + .protected_regs = a620_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 16, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a650v2 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A650, 6, 5, 0, ANY_ID), + .features = ADRENO_IOCOHERENT | ADRENO_CONTENT_PROTECTION | + ADRENO_IFPC | ADRENO_PREEMPTION | ADRENO_ACD | + ADRENO_LM | ADRENO_APRIV | ADRENO_L3_VOTE, + .gpudev = &adreno_a6xx_gmu_gpudev.base, + .perfcounters = &adreno_a6xx_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_1M + SZ_128K, /* verified 1152kB */ + .bus_width = 32, + .snapshot_size = 2 * SZ_1M, + }, + .prim_fifo_threshold = 0x00300000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a650_sqe.fw", + .gmufw_name = "a650_gmu.bin", + .zap_name = "a650_zap.mdt", + .hwcg = a650_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a650_hwcg_regs), + .vbif = a650_gbif_regs, + .vbif_count = ARRAY_SIZE(a650_gbif_regs), + .veto_fal10 = true, + .pdc_in_aop = true, + .hang_detect_cycles = 0x3ffff, + .protected_regs = a620_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 16, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a680 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A680, 6, 8, 0, ANY_ID), + .features = ADRENO_IOCOHERENT | ADRENO_CONTENT_PROTECTION | + ADRENO_IFPC | ADRENO_PREEMPTION, + .gpudev = &adreno_a6xx_gmu_gpudev.base, + .perfcounters = &adreno_a6xx_legacy_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_2M, + .bus_width = 32, + .snapshot_size = SZ_2M, + }, + .prim_fifo_threshold = 0x00400000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a630_sqe.fw", + .gmufw_name = "a640_gmu.bin", + .zap_name = "a640_zap.mdt", + .hwcg = a680_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a680_hwcg_regs), + .vbif = a640_vbif_regs, + .vbif_count = ARRAY_SIZE(a640_vbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = a630_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 16, +}; + +static const struct kgsl_regmap_list a612_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000081}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01202222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x05522022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_ISDB_CNT, 0x00000182}, + {A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, + {A6XX_RBBM_SP_HYST_CNT, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a612 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A612, 6, 1, 2, ANY_ID), + .features = ADRENO_CONTENT_PROTECTION | + ADRENO_IOCOHERENT | ADRENO_PREEMPTION | ADRENO_IFPC, + .gpudev = &adreno_a6xx_rgmu_gpudev, + .perfcounters = &adreno_a6xx_legacy_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = (SZ_128K + SZ_4K), + .bus_width = 32, + .snapshot_size = SZ_1M, + }, + .prim_fifo_threshold = 0x00080000, + .sqefw_name = "a630_sqe.fw", + .gmufw_name = "a612_rgmu.bin", + .zap_name = "a612_zap.mdt", + .hwcg = a612_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a612_hwcg_regs), + .vbif = a640_vbif_regs, + .vbif_count = ARRAY_SIZE(a640_vbif_regs), + .hang_detect_cycles = 0x3fffff, + .protected_regs = a630_protected_regs, + .highest_bank_bit = 14, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a616 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A616, 6, 1, 6, ANY_ID), + .features = ADRENO_PREEMPTION | + ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | + ADRENO_IOCOHERENT, + .gpudev = &adreno_a630_gpudev.base, + .perfcounters = &adreno_a6xx_legacy_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = SZ_512K, + .bus_width = 32, + .snapshot_size = SZ_1M, + }, + .prim_fifo_threshold = 0x0018000, + .gmu_major = 1, + .gmu_minor = 3, + .sqefw_name = "a630_sqe.fw", + .gmufw_name = "a630_gmu.bin", + .zap_name = "a615_zap.mdt", + .hwcg = a615_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a615_hwcg_regs), + .vbif = a615_gbif_regs, + .vbif_count = ARRAY_SIZE(a615_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = a630_protected_regs, + .highest_bank_bit = 14, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a610 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A610, 6, 1, 0, ANY_ID), + .compatible = "qcom,adreno-gpu-a610", + .features = ADRENO_CONTENT_PROTECTION | + ADRENO_PREEMPTION, + .gpudev = &adreno_a6xx_gpudev, + .perfcounters = &adreno_a6xx_legacy_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = (SZ_128K + SZ_4K), + .bus_width = 32, + .snapshot_size = SZ_1M, + }, + .prim_fifo_threshold = 0x00080000, + .sqefw_name = "a630_sqe.fw", + .zap_name = "a610_zap.mdt", + .hwcg = a612_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a612_hwcg_regs), + .vbif = a640_vbif_regs, + .vbif_count = ARRAY_SIZE(a640_vbif_regs), + .hang_detect_cycles = 0x3ffff, + .protected_regs = a630_protected_regs, + .highest_bank_bit = 14, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a611 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A611, 6, 1, 1, ANY_ID), + .compatible = "qcom,adreno-gpu-a611", + .features = ADRENO_CONTENT_PROTECTION | + ADRENO_PREEMPTION, + .gpudev = &adreno_a6xx_gpudev, + .perfcounters = &adreno_a6xx_legacy_perfcounters, + .uche_gmem_alignment = SZ_1M, + .gmem_size = (SZ_128K + SZ_4K), + .bus_width = 32, + .snapshot_size = SZ_1M, + }, + .prim_fifo_threshold = 0x00080000, + .sqefw_name = "a630_sqe.fw", + .zap_name = "a610_zap.mbn", + .hwcg = a612_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a612_hwcg_regs), + .vbif = a640_vbif_regs, + .vbif_count = ARRAY_SIZE(a640_vbif_regs), + .hang_detect_cycles = 0x3ffff, + .protected_regs = a630_protected_regs, + .highest_bank_bit = 14, +}; + +static const struct kgsl_regmap_list a660_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_ISDB_CNT, 0x00000182}, + {A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, + {A6XX_RBBM_SP_HYST_CNT, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, +}; + +/* A660 protected register list */ +static const struct adreno_protected_regs a660_protected_regs[] = { + { A6XX_CP_PROTECT_REG + 0, 0x00000, 0x004ff, 0 }, + { A6XX_CP_PROTECT_REG + 1, 0x00501, 0x00506, 0 }, + { A6XX_CP_PROTECT_REG + 2, 0x0050b, 0x007ff, 0 }, + { A6XX_CP_PROTECT_REG + 3, 0x0050e, 0x0050e, 1 }, + { A6XX_CP_PROTECT_REG + 4, 0x00510, 0x00510, 1 }, + { A6XX_CP_PROTECT_REG + 5, 0x00534, 0x00534, 1 }, + { A6XX_CP_PROTECT_REG + 6, 0x00800, 0x00882, 1 }, + { A6XX_CP_PROTECT_REG + 7, 0x008a0, 0x008a8, 1 }, + { A6XX_CP_PROTECT_REG + 8, 0x008ab, 0x008cf, 1 }, + { A6XX_CP_PROTECT_REG + 9, 0x008d0, 0x0098c, 0 }, + { A6XX_CP_PROTECT_REG + 10, 0x00900, 0x0094d, 1 }, + { A6XX_CP_PROTECT_REG + 11, 0x0098d, 0x00bff, 1 }, + { A6XX_CP_PROTECT_REG + 12, 0x00e00, 0x00e01, 1 }, + { A6XX_CP_PROTECT_REG + 13, 0x00e03, 0x00e0f, 1 }, + { A6XX_CP_PROTECT_REG + 14, 0x03c00, 0x03cc3, 1 }, + { A6XX_CP_PROTECT_REG + 15, 0x03cc4, 0x05cc3, 0 }, + { A6XX_CP_PROTECT_REG + 16, 0x08630, 0x087ff, 1 }, + { A6XX_CP_PROTECT_REG + 17, 0x08e00, 0x08e00, 1 }, + { A6XX_CP_PROTECT_REG + 18, 0x08e08, 0x08e08, 1 }, + { A6XX_CP_PROTECT_REG + 19, 0x08e50, 0x08e6f, 1 }, + { A6XX_CP_PROTECT_REG + 20, 0x08e80, 0x090ff, 1 }, + { A6XX_CP_PROTECT_REG + 21, 0x09624, 0x097ff, 1 }, + { A6XX_CP_PROTECT_REG + 22, 0x09e60, 0x09e71, 1 }, + { A6XX_CP_PROTECT_REG + 23, 0x09e78, 0x09fff, 1 }, + { A6XX_CP_PROTECT_REG + 24, 0x0a630, 0x0a7ff, 1 }, + { A6XX_CP_PROTECT_REG + 25, 0x0ae02, 0x0ae02, 1 }, + { A6XX_CP_PROTECT_REG + 26, 0x0ae50, 0x0af7f, 1 }, + { A6XX_CP_PROTECT_REG + 27, 0x0b604, 0x0b604, 1 }, + { A6XX_CP_PROTECT_REG + 28, 0x0b608, 0x0b60e, 1 }, + { A6XX_CP_PROTECT_REG + 29, 0x0be02, 0x0be03, 1 }, + { A6XX_CP_PROTECT_REG + 30, 0x0be20, 0x0bf7f, 1 }, + { A6XX_CP_PROTECT_REG + 31, 0x0d000, 0x0d5ff, 1 }, + { A6XX_CP_PROTECT_REG + 32, 0x0f000, 0x0fbff, 1 }, + { A6XX_CP_PROTECT_REG + 33, 0x0fc00, 0x11bff, 0 }, + { A6XX_CP_PROTECT_REG + 34, 0x18400, 0x1a3ff, 1 }, + { A6XX_CP_PROTECT_REG + 35, 0x1a400, 0x1c3ff, 1 }, + { A6XX_CP_PROTECT_REG + 36, 0x1c400, 0x1e3ff, 1 }, + { A6XX_CP_PROTECT_REG + 37, 0x1f400, 0x1f843, 1 }, + { A6XX_CP_PROTECT_REG + 38, 0x1f844, 0x1f8bf, 0 }, + { A6XX_CP_PROTECT_REG + 39, 0x1f860, 0x1f860, 1 }, + { A6XX_CP_PROTECT_REG + 40, 0x1f887, 0x1f8a2, 1 }, + { A6XX_CP_PROTECT_REG + 47, 0x1f8c0, 0x1f8c0, 1 }, + { 0 }, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a660 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A660, 6, 6, 0, 0), + .features = ADRENO_APRIV | + ADRENO_IOCOHERENT | ADRENO_CONTENT_PROTECTION | + ADRENO_IFPC | ADRENO_PREEMPTION | ADRENO_L3_VOTE, + .gpudev = &adreno_a6xx_gmu_gpudev.base, + .perfcounters = &adreno_a6xx_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_1M + SZ_512K, + .bus_width = 32, + .snapshot_size = SZ_2M, + }, + .prim_fifo_threshold = 0x00300000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a660_sqe.fw", + .gmufw_name = "a660_gmu.bin", + .zap_name = "a660_zap.mdt", + .hwcg = a660_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a660_hwcg_regs), + .vbif = a650_gbif_regs, + .vbif_count = ARRAY_SIZE(a650_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .veto_fal10 = true, + .protected_regs = a660_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 16, + .pdc_in_aop = true, + .ctxt_record_size = 2496 * 1024, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a660v2 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A660, 6, 6, 0, ANY_ID), + .features = ADRENO_APRIV | + ADRENO_IOCOHERENT | ADRENO_CONTENT_PROTECTION | + ADRENO_IFPC | ADRENO_PREEMPTION | ADRENO_ACD | + ADRENO_L3_VOTE, + .gpudev = &adreno_a6xx_gmu_gpudev.base, + .perfcounters = &adreno_a6xx_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_1M + SZ_512K, + .bus_width = 32, + .snapshot_size = SZ_2M, + }, + .prim_fifo_threshold = 0x00300000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a660_sqe.fw", + .gmufw_name = "a660_gmu.bin", + .zap_name = "a660_zap.mdt", + .hwcg = a660_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a660_hwcg_regs), + .vbif = a650_gbif_regs, + .vbif_count = ARRAY_SIZE(a650_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .veto_fal10 = true, + .protected_regs = a660_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 16, + .pdc_in_aop = true, + .ctxt_record_size = 2496 * 1024, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a660_shima = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A660, 6, 6, 0, ANY_ID), + .compatible = "qcom,adreno-gpu-a660-shima", + .features = ADRENO_APRIV | + ADRENO_IOCOHERENT | ADRENO_CONTENT_PROTECTION | + ADRENO_IFPC | ADRENO_PREEMPTION | ADRENO_ACD, + .gpudev = &adreno_a6xx_gmu_gpudev.base, + .perfcounters = &adreno_a6xx_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_1M + SZ_512K, + .bus_width = 32, + .snapshot_size = SZ_2M, + }, + .prim_fifo_threshold = 0x00300000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a660_sqe.fw", + .gmufw_name = "a660_gmu.bin", + .zap_name = "a660_zap.mdt", + .hwcg = a660_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a660_hwcg_regs), + .vbif = a650_gbif_regs, + .vbif_count = ARRAY_SIZE(a650_gbif_regs), + .hang_detect_cycles = 0x3ffff, + .veto_fal10 = true, + .protected_regs = a660_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 15, + .pdc_in_aop = true, + .ctxt_record_size = 2496 * 1024, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a635 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A635, 6, 3, 5, ANY_ID), + .features = ADRENO_APRIV | ADRENO_IOCOHERENT | + ADRENO_CONTENT_PROTECTION, + .gpudev = &adreno_a6xx_gmu_gpudev.base, + .perfcounters = &adreno_a6xx_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_512K, + .bus_width = 32, + .snapshot_size = SZ_2M, + }, + .prim_fifo_threshold = 0x00200000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a660_sqe.fw", + .gmufw_name = "a660_gmu.bin", + .zap_name = "a660_zap.mdt", + .hwcg = a660_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a660_hwcg_regs), + .vbif = a650_gbif_regs, + .vbif_count = ARRAY_SIZE(a650_gbif_regs), + .hang_detect_cycles = 0x3ffff, + .veto_fal10 = true, + .protected_regs = a660_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 15, + .pdc_in_aop = true, + .ctxt_record_size = 2496 * 1024, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a662 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A662, ANY_ID, ANY_ID, ANY_ID, ANY_ID), + .compatible = "qcom,adreno-gpu-a662", + .features = ADRENO_APRIV | ADRENO_IOCOHERENT | + ADRENO_CONTENT_PROTECTION | ADRENO_PREEMPTION | + ADRENO_IFPC | ADRENO_BCL | ADRENO_ACD, + .gpudev = &adreno_a6xx_gmu_gpudev.base, + .perfcounters = &adreno_a6xx_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_1M + SZ_512K, + .bus_width = 32, + .snapshot_size = SZ_2M, + }, + .prim_fifo_threshold = 0x00300000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a660_sqe.fw", + .gmufw_name = "a662_gmu.bin", + .zap_name = "a662_zap.mdt", + .hwcg = a660_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a660_hwcg_regs), + .vbif = a650_gbif_regs, + .vbif_count = ARRAY_SIZE(a650_gbif_regs), + .hang_detect_cycles = 0x3ffff, + .veto_fal10 = true, + .protected_regs = a660_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 15, + .pdc_in_aop = true, + .ctxt_record_size = 2496 * 1024, +}; + +extern const struct gen7_snapshot_block_list gen7_0_0_snapshot_block_list; + +static const struct kgsl_regmap_list gen7_0_0_gbif_regs[] = { + { GEN7_GBIF_QSB_SIDE0, 0x00071620 }, + { GEN7_GBIF_QSB_SIDE1, 0x00071620 }, + { GEN7_GBIF_QSB_SIDE2, 0x00071620 }, + { GEN7_GBIF_QSB_SIDE3, 0x00071620 }, + { GEN7_RBBM_GBIF_CLIENT_QOS_CNTL, 0x2120212 }, +}; + +static const struct kgsl_regmap_list a702_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000081}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01202222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x05522022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x02222222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_ISDB_CNT, 0x00000182}, + {A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, + {A6XX_RBBM_SP_HYST_CNT, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, + {A6XX_RBBM_CLOCK_CNTL_FCHE, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_FCHE, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_FCHE, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_GLC, 0x00222222}, + {A6XX_RBBM_CLOCK_DELAY_GLC, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GLC, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_MHUB, 0x00000002}, + {A6XX_RBBM_CLOCK_DELAY_MHUB, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_MHUB, 0x00000000}, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a702 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A702, 7, 0, 2, ANY_ID), + .features = ADRENO_CONTENT_PROTECTION | + ADRENO_APRIV | ADRENO_PREEMPTION, + .gpudev = &adreno_a6xx_gpudev, + .perfcounters = &adreno_a6xx_legacy_perfcounters, + .gmem_size = SZ_128K, + .bus_width = 16, + .snapshot_size = SZ_1M, + }, + .prim_fifo_threshold = 0x0000c000, + .sqefw_name = "a702_sqe.fw", + .zap_name = "a702_zap.mdt", + .hwcg = a702_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a702_hwcg_regs), + .vbif = a640_vbif_regs, + .vbif_count = ARRAY_SIZE(a640_vbif_regs), + .hang_detect_cycles = 0x3ffff, + .protected_regs = a620_protected_regs, + .highest_bank_bit = 14, +}; + +static const struct kgsl_regmap_list gen7_0_0_hwcg_regs[] = { + { GEN7_RBBM_CLOCK_CNTL_SP0, 0x02222222 }, + { GEN7_RBBM_CLOCK_CNTL2_SP0, 0x02022222 }, + { GEN7_RBBM_CLOCK_HYST_SP0, 0x0000f3cf }, + { GEN7_RBBM_CLOCK_DELAY_SP0, 0x00000080 }, + { GEN7_RBBM_CLOCK_CNTL_TP0, 0x22222220 }, + { GEN7_RBBM_CLOCK_CNTL2_TP0, 0x22222222 }, + { GEN7_RBBM_CLOCK_CNTL3_TP0, 0x22222222 }, + { GEN7_RBBM_CLOCK_CNTL4_TP0, 0x00222222 }, + { GEN7_RBBM_CLOCK_HYST_TP0, 0x77777777 }, + { GEN7_RBBM_CLOCK_HYST2_TP0, 0x77777777 }, + { GEN7_RBBM_CLOCK_HYST3_TP0, 0x77777777 }, + { GEN7_RBBM_CLOCK_HYST4_TP0, 0x00077777 }, + { GEN7_RBBM_CLOCK_DELAY_TP0, 0x11111111 }, + { GEN7_RBBM_CLOCK_DELAY2_TP0, 0x11111111 }, + { GEN7_RBBM_CLOCK_DELAY3_TP0, 0x11111111 }, + { GEN7_RBBM_CLOCK_DELAY4_TP0, 0x00011111 }, + { GEN7_RBBM_CLOCK_CNTL_UCHE, 0x22222222 }, + { GEN7_RBBM_CLOCK_HYST_UCHE, 0x00000004 }, + { GEN7_RBBM_CLOCK_DELAY_UCHE, 0x00000002 }, + { GEN7_RBBM_CLOCK_CNTL_RB0, 0x22222222 }, + { GEN7_RBBM_CLOCK_CNTL2_RB0, 0x01002222 }, + { GEN7_RBBM_CLOCK_CNTL_CCU0, 0x00002220 }, + { GEN7_RBBM_CLOCK_HYST_RB_CCU0, 0x44000f00 }, + { GEN7_RBBM_CLOCK_CNTL_RAC, 0x25222022 }, + { GEN7_RBBM_CLOCK_CNTL2_RAC, 0x00555555 }, + { GEN7_RBBM_CLOCK_DELAY_RAC, 0x00000011 }, + { GEN7_RBBM_CLOCK_HYST_RAC, 0x00440044 }, + { GEN7_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 }, + { GEN7_RBBM_CLOCK_MODE2_GRAS, 0x00000222 }, + { GEN7_RBBM_CLOCK_MODE_BV_GRAS, 0x00222222 }, + { GEN7_RBBM_CLOCK_MODE_GPC, 0x02222223 }, + { GEN7_RBBM_CLOCK_MODE_VFD, 0x00002222 }, + { GEN7_RBBM_CLOCK_MODE_BV_GPC, 0x00222222 }, + { GEN7_RBBM_CLOCK_MODE_BV_VFD, 0x00002222 }, + { GEN7_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 }, + { GEN7_RBBM_CLOCK_HYST_GPC, 0x04104004 }, + { GEN7_RBBM_CLOCK_HYST_VFD, 0x00000000 }, + { GEN7_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000 }, + { GEN7_RBBM_CLOCK_DELAY_GPC, 0x00000200 }, + { GEN7_RBBM_CLOCK_DELAY_VFD, 0x00002222 }, + { GEN7_RBBM_CLOCK_MODE_HLSQ, 0x00002222 }, + { GEN7_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 }, + { GEN7_RBBM_CLOCK_HYST_HLSQ, 0x00000000 }, + { GEN7_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002 }, + { GEN7_RBBM_CLOCK_MODE_BV_LRZ, 0x55555552 }, + { GEN7_RBBM_CLOCK_MODE_CP, 0x00000223 }, + { GEN7_RBBM_CLOCK_CNTL, 0x8aa8aa82 }, + { GEN7_RBBM_ISDB_CNT, 0x00000182 }, + { GEN7_RBBM_RAC_THRESHOLD_CNT, 0x00000000 }, + { GEN7_RBBM_SP_HYST_CNT, 0x00000000 }, + { GEN7_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 }, + { GEN7_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 }, + { GEN7_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 }, +}; + +static const struct kgsl_regmap_list gen7_0_0_ao_hwcg_regs[] = { + { GEN7_GPU_GMU_AO_GMU_CGC_MODE_CNTL, 0x00020000 }, + { GEN7_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, 0x00010111 }, + { GEN7_GPU_GMU_AO_GMU_CGC_HYST_CNTL, 0x00005555 }, +}; + +/* GEN7_0_0 protected register list */ +static const struct gen7_protected_regs gen7_0_0_protected_regs[] = { + { GEN7_CP_PROTECT_REG + 0, 0x00000, 0x004ff, 0 }, + { GEN7_CP_PROTECT_REG + 1, 0x0050b, 0x00563, 0 }, + { GEN7_CP_PROTECT_REG + 2, 0x0050e, 0x0050e, 1 }, + { GEN7_CP_PROTECT_REG + 3, 0x00510, 0x00510, 1 }, + { GEN7_CP_PROTECT_REG + 4, 0x00534, 0x00534, 1 }, + { GEN7_CP_PROTECT_REG + 5, 0x005fb, 0x00698, 0 }, + { GEN7_CP_PROTECT_REG + 6, 0x00699, 0x00882, 1 }, + { GEN7_CP_PROTECT_REG + 7, 0x008a0, 0x008a8, 1 }, + { GEN7_CP_PROTECT_REG + 8, 0x008ab, 0x008cf, 1 }, + { GEN7_CP_PROTECT_REG + 9, 0x008d0, 0x00a40, 0 }, + { GEN7_CP_PROTECT_REG + 10, 0x00900, 0x0094d, 1 }, + { GEN7_CP_PROTECT_REG + 11, 0x0098d, 0x00a3f, 1 }, + { GEN7_CP_PROTECT_REG + 12, 0x00a41, 0x00bff, 1 }, + { GEN7_CP_PROTECT_REG + 13, 0x00df0, 0x00df1, 1 }, + { GEN7_CP_PROTECT_REG + 14, 0x00e01, 0x00e01, 1 }, + { GEN7_CP_PROTECT_REG + 15, 0x00e07, 0x00e0f, 1 }, + { GEN7_CP_PROTECT_REG + 16, 0x03c00, 0x03cc3, 1 }, + { GEN7_CP_PROTECT_REG + 17, 0x03cc4, 0x05cc3, 0 }, + { GEN7_CP_PROTECT_REG + 18, 0x08630, 0x087ff, 1 }, + { GEN7_CP_PROTECT_REG + 19, 0x08e00, 0x08e00, 1 }, + { GEN7_CP_PROTECT_REG + 20, 0x08e08, 0x08e08, 1 }, + { GEN7_CP_PROTECT_REG + 21, 0x08e50, 0x08e6f, 1 }, + { GEN7_CP_PROTECT_REG + 22, 0x08e80, 0x09100, 1 }, + { GEN7_CP_PROTECT_REG + 23, 0x09624, 0x097ff, 1 }, + { GEN7_CP_PROTECT_REG + 24, 0x09e40, 0x09e40, 1 }, + { GEN7_CP_PROTECT_REG + 25, 0x09e64, 0x09e71, 1 }, + { GEN7_CP_PROTECT_REG + 26, 0x09e78, 0x09fff, 1 }, + { GEN7_CP_PROTECT_REG + 27, 0x0a630, 0x0a7ff, 1 }, + { GEN7_CP_PROTECT_REG + 28, 0x0ae02, 0x0ae02, 1 }, + { GEN7_CP_PROTECT_REG + 29, 0x0ae50, 0x0ae5f, 1 }, + { GEN7_CP_PROTECT_REG + 30, 0x0ae66, 0x0ae69, 1 }, + { GEN7_CP_PROTECT_REG + 31, 0x0ae6f, 0x0ae72, 1 }, + { GEN7_CP_PROTECT_REG + 32, 0x0b604, 0x0b607, 1 }, + { GEN7_CP_PROTECT_REG + 33, 0x0ec00, 0x0fbff, 1 }, + { GEN7_CP_PROTECT_REG + 34, 0x0fc00, 0x11bff, 0 }, + { GEN7_CP_PROTECT_REG + 35, 0x18400, 0x1844a, 1 }, + { GEN7_CP_PROTECT_REG + 36, 0x1844b, 0x1857f, 0 }, + { GEN7_CP_PROTECT_REG + 37, 0x1844c, 0x18453, 1 }, + { GEN7_CP_PROTECT_REG + 38, 0x18580, 0x1a57f, 1 }, + { GEN7_CP_PROTECT_REG + 39, 0x1a580, 0x1c57f, 1 }, + { GEN7_CP_PROTECT_REG + 40, 0x1c580, 0x1e57f, 1 }, + { GEN7_CP_PROTECT_REG + 41, 0x1f400, 0x1f843, 1 }, + { GEN7_CP_PROTECT_REG + 42, 0x1f844, 0x1f8bf, 0 }, + { GEN7_CP_PROTECT_REG + 43, 0x1f860, 0x1f860, 1 }, + { GEN7_CP_PROTECT_REG + 44, 0x1f87f, 0x1f8a2, 1 }, + { GEN7_CP_PROTECT_REG + 47, 0x1f8c0, 0x1f8c0, 1 }, + { 0 }, +}; + +static const struct adreno_gen7_core adreno_gpu_core_gen7_0_0 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_GEN7_0_0, + UINT_MAX, UINT_MAX, UINT_MAX, 0), + .compatible = "qcom,adreno-gpu-gen7-0-0", + .features = ADRENO_APRIV | ADRENO_IOCOHERENT | + ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | + ADRENO_ACD | ADRENO_L3_VOTE | ADRENO_BCL | + ADRENO_PREEMPTION, + .gpudev = &adreno_gen7_gmu_gpudev.base, + .perfcounters = &adreno_gen7_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_2M, + .bus_width = 32, + .snapshot_size = SZ_4M, + }, + .gmu_fw_version = GMU_VERSION(4, 0, 0), + .sqefw_name = "a730_sqe.fw", + .gmufw_name = "gmu_gen70000.bin", + .gmufw_bak_name = "c500_gmu.bin", + .zap_name = "a730_zap.mdt", + .hwcg = gen7_0_0_hwcg_regs, + .hwcg_count = ARRAY_SIZE(gen7_0_0_hwcg_regs), + .ao_hwcg = gen7_0_0_ao_hwcg_regs, + .ao_hwcg_count = ARRAY_SIZE(gen7_0_0_ao_hwcg_regs), + .gbif = gen7_0_0_gbif_regs, + .gbif_count = ARRAY_SIZE(gen7_0_0_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = gen7_0_0_protected_regs, + .highest_bank_bit = 16, + .gen7_snapshot_block_list = &gen7_0_0_snapshot_block_list, + .preempt_level = 1, + .ctxt_record_size = (2860 * SZ_1K), + .fast_bus_hint = true, +}; + +static const struct adreno_gen7_core adreno_gpu_core_gen7_0_1 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_GEN7_0_1, + UINT_MAX, UINT_MAX, UINT_MAX, ANY_ID), + .compatible = "qcom,adreno-gpu-gen7-0-1", + .features = ADRENO_APRIV | ADRENO_IOCOHERENT | + ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | + ADRENO_ACD | ADRENO_L3_VOTE | ADRENO_BCL | + ADRENO_PREEMPTION, + .gpudev = &adreno_gen7_gmu_gpudev.base, + .perfcounters = &adreno_gen7_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_2M, + .bus_width = 32, + .snapshot_size = SZ_4M, + }, + .gmu_fw_version = GMU_VERSION(4, 0, 0), + .sqefw_name = "a730_sqe.fw", + .gmufw_name = "gmu_gen70000.bin", + .gmufw_bak_name = "c500_gmu.bin", + .zap_name = "a730_zap.mdt", + .hwcg = gen7_0_0_hwcg_regs, + .hwcg_count = ARRAY_SIZE(gen7_0_0_hwcg_regs), + .ao_hwcg = gen7_0_0_ao_hwcg_regs, + .ao_hwcg_count = ARRAY_SIZE(gen7_0_0_ao_hwcg_regs), + .gbif = gen7_0_0_gbif_regs, + .gbif_count = ARRAY_SIZE(gen7_0_0_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = gen7_0_0_protected_regs, + .highest_bank_bit = 16, + .gen7_snapshot_block_list = &gen7_0_0_snapshot_block_list, + .preempt_level = 1, + .ctxt_record_size = (2860 * SZ_1K), + .fast_bus_hint = true, +}; + +extern const struct gen7_snapshot_block_list gen7_2_0_snapshot_block_list; + +static const struct kgsl_regmap_list gen7_2_0_gbif_regs[] = { + { GEN7_GBIF_QSB_SIDE0, 0x00071620 }, + { GEN7_GBIF_QSB_SIDE1, 0x00071620 }, + { GEN7_GBIF_QSB_SIDE2, 0x00071620 }, + { GEN7_GBIF_QSB_SIDE3, 0x00071620 }, + { GEN7_RBBM_GBIF_CLIENT_QOS_CNTL, 0x2120212 }, + { GEN7_GMU_CX_MRC_GBIF_QOS_CTRL, 0x33 }, +}; + +static const struct kgsl_regmap_list gen7_2_0_hwcg_regs[] = { + { GEN7_RBBM_CLOCK_CNTL_SP0, 0x02222222 }, + { GEN7_RBBM_CLOCK_CNTL2_SP0, 0x22022222 }, + { GEN7_RBBM_CLOCK_HYST_SP0, 0x003cf3cf }, + { GEN7_RBBM_CLOCK_DELAY_SP0, 0x00000080 }, + { GEN7_RBBM_CLOCK_CNTL_TP0, 0x22222220 }, + { GEN7_RBBM_CLOCK_CNTL2_TP0, 0x22222222 }, + { GEN7_RBBM_CLOCK_CNTL3_TP0, 0x22222222 }, + { GEN7_RBBM_CLOCK_CNTL4_TP0, 0x00222222 }, + { GEN7_RBBM_CLOCK_HYST_TP0, 0x77777777 }, + { GEN7_RBBM_CLOCK_HYST2_TP0, 0x77777777 }, + { GEN7_RBBM_CLOCK_HYST3_TP0, 0x77777777 }, + { GEN7_RBBM_CLOCK_HYST4_TP0, 0x00077777 }, + { GEN7_RBBM_CLOCK_DELAY_TP0, 0x11111111 }, + { GEN7_RBBM_CLOCK_DELAY2_TP0, 0x11111111 }, + { GEN7_RBBM_CLOCK_DELAY3_TP0, 0x11111111 }, + { GEN7_RBBM_CLOCK_DELAY4_TP0, 0x00011111 }, + { GEN7_RBBM_CLOCK_CNTL_UCHE, 0x22222222 }, + { GEN7_RBBM_CLOCK_CNTL2_UCHE, 0x00222222 }, + { GEN7_RBBM_CLOCK_HYST_UCHE, 0x00000444 }, + { GEN7_RBBM_CLOCK_DELAY_UCHE, 0x00000222 }, + { GEN7_RBBM_CLOCK_CNTL_RB0, 0x22222222 }, + { GEN7_RBBM_CLOCK_CNTL2_RB0, 0x01002222 }, + { GEN7_RBBM_CLOCK_CNTL_CCU0, 0x00002220 }, + { GEN7_RBBM_CLOCK_HYST_RB_CCU0, 0x44000f00 }, + { GEN7_RBBM_CLOCK_CNTL_RAC, 0x25222022 }, + { GEN7_RBBM_CLOCK_CNTL2_RAC, 0x00555555 }, + { GEN7_RBBM_CLOCK_DELAY_RAC, 0x00000011 }, + { GEN7_RBBM_CLOCK_HYST_RAC, 0x00440044 }, + { GEN7_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 }, + { GEN7_RBBM_CLOCK_MODE2_GRAS, 0x00000222 }, + { GEN7_RBBM_CLOCK_MODE_BV_GRAS, 0x00222222 }, + { GEN7_RBBM_CLOCK_MODE_GPC, 0x02222223 }, + { GEN7_RBBM_CLOCK_MODE_VFD, 0x00222222 }, + { GEN7_RBBM_CLOCK_MODE_BV_GPC, 0x00222222 }, + { GEN7_RBBM_CLOCK_MODE_BV_VFD, 0x00002222 }, + { GEN7_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 }, + { GEN7_RBBM_CLOCK_HYST_GPC, 0x04104004 }, + { GEN7_RBBM_CLOCK_HYST_VFD, 0x00000000 }, + { GEN7_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00000000 }, + { GEN7_RBBM_CLOCK_DELAY_GPC, 0x00000200 }, + { GEN7_RBBM_CLOCK_DELAY_VFD, 0x00000000 }, + { GEN7_RBBM_CLOCK_MODE_HLSQ, 0x00002222 }, + { GEN7_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 }, + { GEN7_RBBM_CLOCK_HYST_HLSQ, 0x00000000 }, + { GEN7_RBBM_CLOCK_MODE_BV_LRZ, 0x55555552 }, + { GEN7_RBBM_CLOCK_HYST2_VFD, 0x00000000 }, + { GEN7_RBBM_CLOCK_MODE_CP, 0x00000222 }, + { GEN7_RBBM_CLOCK_CNTL, 0x8aa8aa82 }, + { GEN7_RBBM_ISDB_CNT, 0x00000182 }, + { GEN7_RBBM_RAC_THRESHOLD_CNT, 0x00000000 }, + { GEN7_RBBM_SP_HYST_CNT, 0x00000000 }, + { GEN7_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 }, + { GEN7_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 }, + { GEN7_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 }, +}; + +static const struct kgsl_regmap_list gen7_2_0_ao_hwcg_regs[] = { + { GEN7_GPU_GMU_AO_GMU_CGC_MODE_CNTL, 0x00020202 }, + { GEN7_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, 0x00010111 }, + { GEN7_GPU_GMU_AO_GMU_CGC_HYST_CNTL, 0x00005555 }, +}; + +static const struct adreno_gen7_core adreno_gpu_core_gen7_2_0 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_GEN7_2_0, + UINT_MAX, UINT_MAX, UINT_MAX, ANY_ID), + .compatible = "qcom,adreno-gpu-gen7-2-0", + .features = ADRENO_APRIV | ADRENO_IOCOHERENT | ADRENO_IFPC | + ADRENO_CONTENT_PROTECTION | ADRENO_ACD | + ADRENO_LPAC | ADRENO_BCL | ADRENO_L3_VOTE | + ADRENO_PREEMPTION | ADRENO_DMS, + .gpudev = &adreno_gen7_hwsched_gpudev.base, + .perfcounters = &adreno_gen7_hwsched_perfcounters, + .uche_gmem_alignment = SZ_16M, + .gmem_size = 3 * SZ_1M, + .bus_width = 32, + .snapshot_size = SZ_8M, + }, + .gmu_fw_version = GMU_VERSION(4, 1, 0), + .sqefw_name = "a740_sqe.fw", + .gmufw_name = "gmu_gen70200.bin", + .zap_name = "a740_zap.mbn", + .hwcg = gen7_2_0_hwcg_regs, + .hwcg_count = ARRAY_SIZE(gen7_2_0_hwcg_regs), + .ao_hwcg = gen7_2_0_ao_hwcg_regs, + .ao_hwcg_count = ARRAY_SIZE(gen7_2_0_ao_hwcg_regs), + .gbif = gen7_2_0_gbif_regs, + .gbif_count = ARRAY_SIZE(gen7_2_0_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = gen7_0_0_protected_regs, + .highest_bank_bit = 16, + .gmu_hub_clk_freq = 200000000, + .gen7_snapshot_block_list = &gen7_2_0_snapshot_block_list, + .bcl_data = 1, + .preempt_level = 1, + .ctxt_record_size = (4192 * SZ_1K), + .fast_bus_hint = true, +}; + +static const struct adreno_gen7_core adreno_gpu_core_gen7_2_1 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_GEN7_2_1, + UINT_MAX, UINT_MAX, UINT_MAX, ANY_ID), + .compatible = "qcom,adreno-gpu-gen7-2-1", + .features = ADRENO_APRIV | ADRENO_IOCOHERENT | ADRENO_IFPC | + ADRENO_CONTENT_PROTECTION | ADRENO_LPAC | + ADRENO_BCL | ADRENO_L3_VOTE | ADRENO_ACD | + ADRENO_PREEMPTION | ADRENO_DMS, + .gpudev = &adreno_gen7_hwsched_gpudev.base, + .perfcounters = &adreno_gen7_hwsched_perfcounters, + .uche_gmem_alignment = SZ_16M, + .gmem_size = 3 * SZ_1M, + .bus_width = 32, + .snapshot_size = SZ_8M, + }, + .gmu_fw_version = GMU_VERSION(4, 1, 0), + .sqefw_name = "a740_sqe.fw", + .gmufw_name = "gmu_gen70200.bin", + .zap_name = "a740_zap.mbn", + .hwcg = gen7_2_0_hwcg_regs, + .hwcg_count = ARRAY_SIZE(gen7_2_0_hwcg_regs), + .ao_hwcg = gen7_2_0_ao_hwcg_regs, + .ao_hwcg_count = ARRAY_SIZE(gen7_2_0_ao_hwcg_regs), + .gbif = gen7_2_0_gbif_regs, + .gbif_count = ARRAY_SIZE(gen7_2_0_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = gen7_0_0_protected_regs, + .highest_bank_bit = 16, + .gmu_hub_clk_freq = 200000000, + .gen7_snapshot_block_list = &gen7_2_0_snapshot_block_list, + .bcl_data = 1, + .preempt_level = 1, + .ctxt_record_size = (4192 * SZ_1K), + .fast_bus_hint = true, +}; + +static const struct adreno_gen7_core adreno_gpu_core_gen7_4_0 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_GEN7_4_0, + UINT_MAX, UINT_MAX, UINT_MAX, ANY_ID), + .compatible = "qcom,adreno-gpu-gen7-4-0", + .features = ADRENO_APRIV | ADRENO_IOCOHERENT | + ADRENO_CONTENT_PROTECTION | ADRENO_L3_VOTE | + ADRENO_PREEMPTION | ADRENO_IFPC | ADRENO_ACD | + ADRENO_BCL, + .gpudev = &adreno_gen7_gmu_gpudev.base, + .perfcounters = &adreno_gen7_perfcounters, + .uche_gmem_alignment = 0, + .gmem_size = SZ_2M, + .bus_width = 32, + .snapshot_size = SZ_4M, + }, + .gmu_fw_version = GMU_VERSION(4, 0, 7), + .sqefw_name = "a730_sqe.fw", + .gmufw_name = "gmu_gen70000.bin", + .gmufw_bak_name = "c500_gmu.bin", + .zap_name = "a730_zap.mdt", + .hwcg = gen7_0_0_hwcg_regs, + .hwcg_count = ARRAY_SIZE(gen7_0_0_hwcg_regs), + .ao_hwcg = gen7_0_0_ao_hwcg_regs, + .ao_hwcg_count = ARRAY_SIZE(gen7_0_0_ao_hwcg_regs), + .gbif = gen7_0_0_gbif_regs, + .gbif_count = ARRAY_SIZE(gen7_0_0_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = gen7_0_0_protected_regs, + .highest_bank_bit = 16, + .gen7_snapshot_block_list = &gen7_0_0_snapshot_block_list, + .preempt_level = 1, + .ctxt_record_size = (2860 * SZ_1K), + .fast_bus_hint = true, +}; + +extern const struct gen7_snapshot_block_list gen7_9_0_snapshot_block_list; + +/* GEN7_9_0 protected register list */ +static const struct gen7_protected_regs gen7_9_0_protected_regs[] = { + { GEN7_CP_PROTECT_REG + 0, 0x00000, 0x004ff, 0 }, + { GEN7_CP_PROTECT_REG + 1, 0x0050b, 0x00563, 0 }, + { GEN7_CP_PROTECT_REG + 2, 0x00584, 0x006c1, 0 }, + { GEN7_CP_PROTECT_REG + 3, 0x00706, 0x00706, 0 }, + { GEN7_CP_PROTECT_REG + 4, 0x00720, 0x0073f, 0 }, + { GEN7_CP_PROTECT_REG + 5, 0x00760, 0x007ff, 0 }, + { GEN7_CP_PROTECT_REG + 6, 0x00800, 0x00882, 1 }, + { GEN7_CP_PROTECT_REG + 7, 0x008a0, 0x008a8, 1 }, + { GEN7_CP_PROTECT_REG + 8, 0x008ab, 0x00a40, 0 }, + { GEN7_CP_PROTECT_REG + 9, 0x00900, 0x0094d, 1 }, + { GEN7_CP_PROTECT_REG + 10, 0x0098d, 0x00a3f, 1 }, + { GEN7_CP_PROTECT_REG + 11, 0x00a41, 0x00bff, 1 }, + { GEN7_CP_PROTECT_REG + 12, 0x00df0, 0x00df1, 1 }, + { GEN7_CP_PROTECT_REG + 13, 0x00e01, 0x00e01, 1 }, + { GEN7_CP_PROTECT_REG + 14, 0x00e07, 0x00e0f, 1 }, + { GEN7_CP_PROTECT_REG + 15, 0x02840, 0x03cc3, 1 }, + { GEN7_CP_PROTECT_REG + 16, 0x03cc4, 0x05cc3, 0 }, + { GEN7_CP_PROTECT_REG + 17, 0x08630, 0x087ff, 1 }, + { GEN7_CP_PROTECT_REG + 18, 0x08e00, 0x08e00, 1 }, + { GEN7_CP_PROTECT_REG + 19, 0x08e08, 0x08e08, 1 }, + { GEN7_CP_PROTECT_REG + 20, 0x08e50, 0x08e6f, 1 }, + { GEN7_CP_PROTECT_REG + 21, 0x08e79, 0x09100, 1 }, + { GEN7_CP_PROTECT_REG + 22, 0x09624, 0x097ff, 1 }, + { GEN7_CP_PROTECT_REG + 23, 0x09b0b, 0x09dff, 0 }, + { GEN7_CP_PROTECT_REG + 24, 0x09e1a, 0x09e1b, 1 }, + { GEN7_CP_PROTECT_REG + 25, 0x09e40, 0x09e40, 1 }, + { GEN7_CP_PROTECT_REG + 26, 0x09e64, 0x09e64, 1 }, + { GEN7_CP_PROTECT_REG + 27, 0x09e70, 0x09e71, 1 }, + { GEN7_CP_PROTECT_REG + 28, 0x09e78, 0x09fff, 1 }, + { GEN7_CP_PROTECT_REG + 29, 0x0a630, 0x0a7ff, 1 }, + { GEN7_CP_PROTECT_REG + 30, 0x0ae02, 0x0ae02, 1 }, + { GEN7_CP_PROTECT_REG + 31, 0x0ae50, 0x0ae5f, 1 }, + { GEN7_CP_PROTECT_REG + 32, 0x0ae66, 0x0ae69, 1 }, + { GEN7_CP_PROTECT_REG + 33, 0x0ae6f, 0x0ae72, 1 }, + { GEN7_CP_PROTECT_REG + 34, 0x0b602, 0x0b607, 1 }, + { GEN7_CP_PROTECT_REG + 35, 0x0ec00, 0x0fbff, 1 }, + { GEN7_CP_PROTECT_REG + 36, 0x0fc00, 0x11bff, 0 }, + { GEN7_CP_PROTECT_REG + 37, 0x18400, 0x1857f, 0 }, + { GEN7_CP_PROTECT_REG + 38, 0x18580, 0x1a57f, 1 }, + { GEN7_CP_PROTECT_REG + 39, 0x1a580, 0x1c57f, 1 }, + { GEN7_CP_PROTECT_REG + 40, 0x1c580, 0x1e57f, 1 }, + { GEN7_CP_PROTECT_REG + 41, 0x1f400, 0x1f843, 1 }, + { GEN7_CP_PROTECT_REG + 42, 0x1f844, 0x1f8b7, 0 }, + { GEN7_CP_PROTECT_REG + 43, 0x1f87f, 0x1f8a2, 1 }, + { GEN7_CP_PROTECT_REG + 44, 0x1f8b8, 0x218b7, 1 }, + { GEN7_CP_PROTECT_REG + 45, 0x27800, 0x2787f, 1 }, + { GEN7_CP_PROTECT_REG + 46, 0x27880, 0x27c01, 0 }, + { GEN7_CP_PROTECT_REG + 47, 0x27c02, 0x27c02, 1 }, + { 0 }, +}; + +static const struct adreno_gen7_core adreno_gpu_core_gen7_9_0 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_GEN7_9_0, + UINT_MAX, UINT_MAX, UINT_MAX, ANY_ID), + .compatible = "qcom,adreno-gpu-gen7-9-0", + .features = ADRENO_APRIV | ADRENO_IOCOHERENT | ADRENO_AQE | + ADRENO_CONTENT_PROTECTION | ADRENO_LPAC | ADRENO_IFPC | + ADRENO_L3_VOTE | ADRENO_BCL | ADRENO_DMS | + ADRENO_HW_FENCE | ADRENO_PREEMPTION | ADRENO_ACD | + ADRENO_GMU_WARMBOOT, + .gpudev = &adreno_gen7_9_0_hwsched_gpudev.base, + .perfcounters = &adreno_gen7_9_0_hwsched_perfcounters, + .uche_gmem_alignment = SZ_16M, + .gmem_size = 3 * SZ_1M, + .bus_width = 32, + .snapshot_size = SZ_8M, + .num_ddr_channels = 4, + }, + .aqefw_name = "gen70900_aqe.fw", + .sqefw_name = "gen70900_sqe.fw", + .gmufw_name = "gmu_gen70900.bin", + .zap_name = "gen70900_zap.mbn", + .ao_hwcg = gen7_2_0_ao_hwcg_regs, + .ao_hwcg_count = ARRAY_SIZE(gen7_2_0_ao_hwcg_regs), + .gbif = gen7_2_0_gbif_regs, + .gbif_count = ARRAY_SIZE(gen7_2_0_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = gen7_9_0_protected_regs, + .highest_bank_bit = 16, + .gmu_hub_clk_freq = 200000000, + .gen7_snapshot_block_list = &gen7_9_0_snapshot_block_list, + .bcl_data = 1, + .acv_perfmode_vote = BIT(2), + .acv_perfmode_ddr_freq = MHZ_TO_KBPS(2736, 4), + .ctxt_record_size = (4208 * SZ_1K), + .preempt_level = 1, + .fast_bus_hint = true, +}; + +static const struct adreno_gen7_core adreno_gpu_core_gen7_9_1 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_GEN7_9_1, + UINT_MAX, UINT_MAX, UINT_MAX, ANY_ID), + .compatible = "qcom,adreno-gpu-gen7-9-1", + .features = ADRENO_APRIV | ADRENO_IOCOHERENT | ADRENO_AQE | + ADRENO_CONTENT_PROTECTION | ADRENO_LPAC | ADRENO_IFPC | + ADRENO_L3_VOTE | ADRENO_BCL | ADRENO_DMS | + ADRENO_HW_FENCE | ADRENO_PREEMPTION | ADRENO_ACD | + ADRENO_GMU_WARMBOOT, + .gpudev = &adreno_gen7_9_0_hwsched_gpudev.base, + .perfcounters = &adreno_gen7_9_0_hwsched_perfcounters, + .uche_gmem_alignment = SZ_16M, + .gmem_size = 3 * SZ_1M, + .bus_width = 32, + .snapshot_size = SZ_8M, + .num_ddr_channels = 4, + }, + .aqefw_name = "gen70900_aqe.fw", + .sqefw_name = "gen70900_sqe.fw", + .gmufw_name = "gmu_gen70900.bin", + .zap_name = "gen70900_zap.mbn", + .ao_hwcg = gen7_2_0_ao_hwcg_regs, + .ao_hwcg_count = ARRAY_SIZE(gen7_2_0_ao_hwcg_regs), + .gbif = gen7_0_0_gbif_regs, + .gbif_count = ARRAY_SIZE(gen7_0_0_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = gen7_9_0_protected_regs, + .highest_bank_bit = 16, + .gmu_hub_clk_freq = 200000000, + .gen7_snapshot_block_list = &gen7_9_0_snapshot_block_list, + .bcl_data = 1, + .acv_perfmode_vote = BIT(2), + .acv_perfmode_ddr_freq = MHZ_TO_KBPS(2736, 4), + .ctxt_record_size = (4208 * SZ_1K), + .preempt_level = 1, + .fast_bus_hint = true, +}; + +extern const struct gen7_snapshot_block_list gen7_11_0_snapshot_block_list; + +static const struct adreno_gen7_core adreno_gpu_core_gen7_11_0 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_GEN7_11_0, + UINT_MAX, UINT_MAX, UINT_MAX, ANY_ID), + .compatible = "qcom,adreno-gpu-gen7-11-0", + .features = ADRENO_APRIV | ADRENO_IOCOHERENT | ADRENO_CONTENT_PROTECTION | + ADRENO_IFPC | ADRENO_PREEMPTION | ADRENO_L3_VOTE | + ADRENO_DMS | ADRENO_BCL, + .gpudev = &adreno_gen7_hwsched_gpudev.base, + .perfcounters = &adreno_gen7_hwsched_perfcounters, + .uche_gmem_alignment = SZ_16M, + .gmem_size = SZ_1M + SZ_512K, + .bus_width = 32, + .snapshot_size = SZ_4M, + .num_ddr_channels = 4, + }, + .sqefw_name = "gen71100_sqe.fw", + .gmufw_name = "gen71100_gmu.bin", + .zap_name = "gen71100_zap.mbn", + .hwcg = gen7_2_0_hwcg_regs, + .hwcg_count = ARRAY_SIZE(gen7_2_0_hwcg_regs), + .ao_hwcg = gen7_2_0_ao_hwcg_regs, + .ao_hwcg_count = ARRAY_SIZE(gen7_2_0_ao_hwcg_regs), + .gbif = gen7_2_0_gbif_regs, + .gbif_count = ARRAY_SIZE(gen7_2_0_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = gen7_0_0_protected_regs, + .highest_bank_bit = 16, + .gmu_hub_clk_freq = 200000000, + .gen7_snapshot_block_list = &gen7_11_0_snapshot_block_list, + .preempt_level = 1, + .acv_perfmode_vote = BIT(2), + .bcl_data = 1, + .fast_bus_hint = true, + .ctxt_record_size = (2196 * SZ_1K), +}; + +static const struct kgsl_regmap_list a663_hwcg_regs[] = { + {A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222}, + {A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220}, + {A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF}, + {A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222}, + {A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00}, + {A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022}, + {A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {A6XX_RBBM_CLOCK_CNTL, 0x8AA8AA82}, + {A6XX_RBBM_ISDB_CNT, 0x00000182}, + {A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000}, + {A6XX_RBBM_SP_HYST_CNT, 0x00000000}, + {A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555}, + {A6XX_GMUAO_GMU_CGC_MODE_CNTL, 0x00020200}, + {A6XX_GMUAO_GMU_CGC_DELAY_CNTL, 0x00010111}, + {A6XX_GMUAO_GMU_CGC_HYST_CNTL, 0x00005555}, + {A6XX_GMUCX_GMU_WFI_CONFIG, 0x00000000}, +}; + +/* A633 protected register list */ +static const struct adreno_protected_regs a663_protected_regs[] = { + { A6XX_CP_PROTECT_REG + 0, 0x00000, 0x004ff, 0 }, + { A6XX_CP_PROTECT_REG + 1, 0x00501, 0x00506, 0 }, + { A6XX_CP_PROTECT_REG + 2, 0x0050b, 0x007ff, 0 }, + { A6XX_CP_PROTECT_REG + 3, 0x0050e, 0x0050e, 1 }, + { A6XX_CP_PROTECT_REG + 4, 0x00510, 0x00510, 1 }, + { A6XX_CP_PROTECT_REG + 5, 0x00534, 0x00534, 1 }, + { A6XX_CP_PROTECT_REG + 6, 0x00800, 0x00882, 1 }, + { A6XX_CP_PROTECT_REG + 7, 0x008a0, 0x008a8, 1 }, + { A6XX_CP_PROTECT_REG + 8, 0x008ab, 0x008cf, 1 }, + { A6XX_CP_PROTECT_REG + 9, 0x008d0, 0x0098c, 0 }, + { A6XX_CP_PROTECT_REG + 10, 0x00900, 0x0094d, 1 }, + { A6XX_CP_PROTECT_REG + 11, 0x0098d, 0x00bff, 1 }, + { A6XX_CP_PROTECT_REG + 12, 0x00e00, 0x00e01, 1 }, + { A6XX_CP_PROTECT_REG + 13, 0x00e03, 0x00e0f, 1 }, + { A6XX_CP_PROTECT_REG + 14, 0x03c00, 0x03cc3, 1 }, + { A6XX_CP_PROTECT_REG + 15, 0x03cc4, 0x05cc3, 0 }, + { A6XX_CP_PROTECT_REG + 16, 0x08630, 0x087ff, 1 }, + { A6XX_CP_PROTECT_REG + 17, 0x08e00, 0x08e00, 1 }, + { A6XX_CP_PROTECT_REG + 18, 0x08e08, 0x08e08, 1 }, + { A6XX_CP_PROTECT_REG + 19, 0x08e50, 0x08e6f, 1 }, + { A6XX_CP_PROTECT_REG + 20, 0x08e80, 0x090ff, 1 }, + { A6XX_CP_PROTECT_REG + 21, 0x09624, 0x097ff, 1 }, + { A6XX_CP_PROTECT_REG + 22, 0x09e60, 0x09e71, 1 }, + { A6XX_CP_PROTECT_REG + 23, 0x09e78, 0x09fff, 1 }, + { A6XX_CP_PROTECT_REG + 24, 0x0a630, 0x0a7ff, 1 }, + { A6XX_CP_PROTECT_REG + 25, 0x0ae02, 0x0ae02, 1 }, + { A6XX_CP_PROTECT_REG + 26, 0x0ae50, 0x0af7f, 1 }, + { A6XX_CP_PROTECT_REG + 27, 0x0b604, 0x0b604, 1 }, + { A6XX_CP_PROTECT_REG + 28, 0x0b608, 0x0b60e, 1 }, + { A6XX_CP_PROTECT_REG + 29, 0x0be02, 0x0be03, 1 }, + { A6XX_CP_PROTECT_REG + 30, 0x0be20, 0x0bf7f, 1 }, + { A6XX_CP_PROTECT_REG + 31, 0x0d000, 0x0d5ff, 1 }, + { A6XX_CP_PROTECT_REG + 32, 0x0f000, 0x0fbff, 1 }, + { A6XX_CP_PROTECT_REG + 33, 0x0fc00, 0x11bff, 0 }, + /* Note1: lastspanunbound feature is enabled in + * CP_PROTECT_CNTL and hence this last + * protect register(REG_47) has infinite + * span. + * + * Note2: Although we are protecting the SMMU + * range here the CP register protection + * interrupt will not fire for this range + * as GPU RAP can only cover the GPU 18-bit + * DW address space. So max address offset + * is 0x3FFFF. Also note that the max number + * of bits for address in violation in + * CP_PROT_STATUS is only 18. + */ + { A6XX_CP_PROTECT_REG + 47, 0x11c00, 0x00000, 1 }, + { 0 }, +}; + +static const struct adreno_a6xx_core adreno_gpu_core_a663 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_A663, 6, 6, 3, ANY_ID), + .features = ADRENO_APRIV | + ADRENO_IOCOHERENT | ADRENO_CONTENT_PROTECTION | + ADRENO_PREEMPTION | ADRENO_ACD, + .gpudev = &adreno_a6xx_gmu_gpudev.base, + .perfcounters = &adreno_a6xx_perfcounters, + .gmem_size = SZ_1M + SZ_512K, + .bus_width = 32, + .snapshot_size = SZ_2M, + }, + .prim_fifo_threshold = 0x00300000, + .gmu_major = 2, + .gmu_minor = 0, + .sqefw_name = "a660_sqe.fw", + .gmufw_name = "a663_gmu.bin", + .zap_name = "a663_zap.mdt", + .hwcg = a663_hwcg_regs, + .hwcg_count = ARRAY_SIZE(a663_hwcg_regs), + .vbif = a650_gbif_regs, + .vbif_count = ARRAY_SIZE(a650_gbif_regs), + .hang_detect_cycles = 0xcfffff, + .veto_fal10 = true, + .protected_regs = a663_protected_regs, + .disable_tseskip = true, + .highest_bank_bit = 13, + .pdc_in_aop = true, + .ctxt_record_size = 2496 * 1024, +}; + +extern const struct gen8_snapshot_block_list gen8_3_0_snapshot_block_list; + +static const struct kgsl_regmap_list gen8_3_0_gbif_cx_regs[] = { + { GEN8_GBIF_QSB_SIDE0, 0x00071e20 }, + { GEN8_GBIF_QSB_SIDE1, 0x00071e20 }, + { GEN8_GBIF_QSB_SIDE2, 0x00071e20 }, + { GEN8_GBIF_QSB_SIDE3, 0x00071e20 }, + { GEN8_GBIF_CX_CONFIG, 0x20023000 }, +}; + +/* GEN8_3_0 noncontext register list */ +static const struct gen8_nonctxt_regs gen8_3_0_nonctxt_regs[] = { + { GEN8_CP_SMMU_STREAM_ID_LPAC, 0x00000101, BIT(PIPE_NONE) }, + { GEN8_GRAS_DBG_ECO_CNTL, 0x00f80800, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_PC_AUTO_VERTEX_STRIDE, 0x00000001, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_PC_VIS_STREAM_CNTL, 0x10010000, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_PC_CONTEXT_SWITCH_STABILIZE_CNTL_1, 0x00000002, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_PC_CHICKEN_BITS_1, 0x00000003, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_PC_CHICKEN_BITS_2, 0x00000200, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_PC_CHICKEN_BITS_3, 0x00500000, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_PC_CHICKEN_BITS_4, 0x00500050, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_RB_CCU_CNTL, 0x00000068, BIT(PIPE_BR) }, + { GEN8_RB_RESOLVE_PREFETCH_CNTL, 0x00000007, BIT(PIPE_BR) }, + { GEN8_RB_CMP_DBG_ECO_CNTL, 0x00004000, BIT(PIPE_BR) }, + { GEN8_RBBM_NC_MODE_CNTL, 0x00000001, BIT(PIPE_NONE) }, + { GEN8_RBBM_SLICE_NC_MODE_CNTL, 0x00000001, BIT(PIPE_NONE) }, + { GEN8_RBBM_WAIT_IDLE_CLOCKS_CNTL, 0x00000030, BIT(PIPE_NONE) }, + { GEN8_RBBM_WAIT_IDLE_CLOCKS_CNTL2, 0x00000030, BIT(PIPE_NONE) }, + { GEN8_UCHE_GBIF_GX_CONFIG, 0x010240e0, BIT(PIPE_NONE) }, + { GEN8_RBBM_GBIF_CLIENT_QOS_CNTL, 0x22122212, BIT(PIPE_NONE) }, + { GEN8_RBBM_CGC_P2S_CNTL, 0x00000040, BIT(PIPE_NONE) }, + /* + * BIT(22): Disable PS out of order retire + * BIT(23): Enable half wave mode and MM instruction src&dst is half precision + */ + { GEN8_SP_CHICKEN_BITS_2, BIT(22) | BIT(23), BIT(PIPE_NONE) }, + { GEN8_SP_CHICKEN_BITS_3, 0x00300000, BIT(PIPE_NONE) }, + { GEN8_SP_PERFCTR_SHADER_MASK, 0x0000003f, BIT(PIPE_NONE) }, + { GEN8_SP_HLSQ_TIMEOUT_THRESHOLD_DP, 0x00000080, BIT(PIPE_NONE) }, + { GEN8_SP_READ_SEL, 0x0001ff00, BIT(PIPE_NONE) }, + { GEN8_TPL1_DBG_ECO_CNTL, 0x10000000, BIT(PIPE_NONE) }, + { GEN8_TPL1_DBG_ECO_CNTL1, 0x00000724, BIT(PIPE_NONE) }, + { GEN8_UCHE_MODE_CNTL, 0x00020000, BIT(PIPE_NONE) }, + { GEN8_UCHE_CCHE_MODE_CNTL, 0x00001000, BIT(PIPE_NONE) }, + { GEN8_UCHE_CCHE_CACHE_WAYS, 0x00000800, BIT(PIPE_NONE) }, + { GEN8_UCHE_CACHE_WAYS, 0x00080000, BIT(PIPE_NONE) }, + { GEN8_UCHE_VARB_IDLE_TIMEOUT, 0x00000020, BIT(PIPE_NONE) }, + { GEN8_VFD_DBG_ECO_CNTL, 0x00008000, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_VFD_CB_BV_THRESHOLD, 0x00500050, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_VFD_CB_BR_THRESHOLD, 0x00600060, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_VFD_CB_BUSY_REQ_CNT, 0x00200020, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_VFD_CB_LP_REQ_CNT, 0x00100020, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_VPC_FLATSHADE_MODE_CNTL, 0x00000001, BIT(PIPE_BV) | BIT(PIPE_BR) }, + { GEN8_VSC_BIN_SIZE, 0x00010001, BIT(PIPE_NONE) }, + { GEN8_RB_GC_GMEM_PROTECT, 0x00900000, BIT(PIPE_BR) }, + { 0 }, +}; + +static const struct kgsl_regmap_list gen8_ao_hwcg_regs[] = { + { GEN8_GMUAO_CGC_MODE_CNTL, 0x00020000 }, + { GEN8_GMUAO_CGC_DELAY_CNTL, 0x00010111 }, + { GEN8_GMUAO_CGC_HYST_CNTL, 0x00005555 }, +}; + +/* GEN8_3_0 protected register list */ +static const struct gen8_protected_regs gen8_3_0_protected_regs[] = { + { GEN8_CP_PROTECT_REG_GLOBAL + 0, 0x00000, 0x003a3, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 1, 0x003b4, 0x0043f, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 2, 0x00440, 0x0045f, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 3, 0x00580, 0x005df, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 4, 0x005e0, 0x006ff, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 5, 0x0074a, 0x0074f, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 6, 0x00759, 0x0077f, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 7, 0x00789, 0x00789, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 8, 0x0078c, 0x0079f, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 9, 0x00800, 0x00829, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 10, 0x00837, 0x008e6, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 11, 0x008e7, 0x009b0, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 12, 0x008ec, 0x009af, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 13, 0x009b1, 0x00c01, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 14, 0x00ce0, 0x00ce1, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 15, 0x00df0, 0x00df0, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 16, 0x00df1, 0x00df1, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 17, 0x00e01, 0x00e01, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 18, 0x00e03, 0x02e02, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 19, 0x03c00, 0x03cc5, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 20, 0x03cc6, 0x05cc5, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 21, 0x08600, 0x087ff, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 22, 0x08e00, 0x08eff, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 23, 0x08f00, 0x08f00, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 24, 0x08f01, 0x090bf, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 25, 0x09600, 0x097ff, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 26, 0x0981a, 0x09aff, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 27, 0x09e00, 0x09fff, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 28, 0x0a600, 0x0a7ff, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 29, 0x0ae00, 0x0ae06, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 30, 0x0ae08, 0x0ae0e, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 31, 0x0ae10, 0x0b17f, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 32, 0x0b600, 0x0d5ff, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 33, 0x0dc00, 0x0fbff, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 34, 0x0fc00, 0x11bff, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 35, 0x18400, 0x1843f, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 36, 0x18440, 0x1857f, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 37, 0x18580, 0x1a57f, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 38, 0x1b400, 0x1d3ff, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 39, 0x1f400, 0x1f877, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 40, 0x1f878, 0x1ffff, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 41, 0x1f930, 0x1fc59, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 42, 0x20000, 0x21fff, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 43, 0x27800, 0x2787f, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 44, 0x27880, 0x27c01, 0 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 45, 0x27882, 0x27883, 1 }, + { GEN8_CP_PROTECT_REG_GLOBAL + 63, 0x27c02, 0x27c02, 1 }, + { 0 }, +}; + +static const struct adreno_gen8_core adreno_gpu_core_gen8_3_0 = { + .base = { + DEFINE_ADRENO_REV(ADRENO_REV_GEN8_3_0, + UINT_MAX, UINT_MAX, UINT_MAX, ANY_ID), + .compatible = "qcom,adreno-gpu-gen8-3-0", + .features = ADRENO_APRIV | ADRENO_IOCOHERENT | + ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | ADRENO_BCL | + ADRENO_PREEMPTION | ADRENO_ACD, + .gpudev = &adreno_gen8_hwsched_gpudev.base, + .perfcounters = &adreno_gen8_perfcounters, + .uche_gmem_alignment = SZ_64M, + .gmem_size = (SZ_512K + SZ_64K), + .bus_width = 32, + .snapshot_size = SZ_8M, + .num_ddr_channels = 2, + }, + .sqefw_name = "gen80300_sqe.fw", + .gmufw_name = "gen80300_gmu.bin", + .zap_name = "gen80300_zap.mbn", + .ao_hwcg = gen8_ao_hwcg_regs, + .ao_hwcg_count = ARRAY_SIZE(gen8_ao_hwcg_regs), + .gbif = gen8_3_0_gbif_cx_regs, + .gbif_count = ARRAY_SIZE(gen8_3_0_gbif_cx_regs), + .hang_detect_cycles = 0xcfffff, + .protected_regs = gen8_3_0_protected_regs, + .nonctxt_regs = gen8_3_0_nonctxt_regs, + .highest_bank_bit = 15, + .gmu_hub_clk_freq = 200000000, + .gen8_snapshot_block_list = &gen8_3_0_snapshot_block_list, + .ctxt_record_size = (4558 * SZ_1K), + .bcl_data = 1, + .noc_timeout_us = 6800, /* 6.8 msec */ +}; + +static const struct adreno_gpu_core *adreno_gpulist[] = { + &adreno_gpu_core_a306.base, + &adreno_gpu_core_a306a.base, + &adreno_gpu_core_a304.base, + &adreno_gpu_core_a405, /* Deprecated */ + &adreno_gpu_core_a418, /* Deprecated */ + &adreno_gpu_core_a420, /* Deprecated */ + &adreno_gpu_core_a430, /* Deprecated */ + &adreno_gpu_core_a530v1, /* Deprecated */ + &adreno_gpu_core_a530v2.base, + &adreno_gpu_core_a530v3.base, + &adreno_gpu_core_a505.base, + &adreno_gpu_core_a506.base, + &adreno_gpu_core_a510.base, + &adreno_gpu_core_a540v1, /* Deprecated */ + &adreno_gpu_core_a540v2.base, + &adreno_gpu_core_a512.base, + &adreno_gpu_core_a508.base, + &adreno_gpu_core_a630v1, /* Deprecated */ + &adreno_gpu_core_a630v2.base, + &adreno_gpu_core_a615.base, + &adreno_gpu_core_a618.base, + &adreno_gpu_core_a619.base, + &adreno_gpu_core_a619_variant.base, + &adreno_gpu_core_a620.base, + &adreno_gpu_core_a621.base, + &adreno_gpu_core_a635.base, + &adreno_gpu_core_a640.base, + &adreno_gpu_core_a650.base, + &adreno_gpu_core_a650v2.base, + &adreno_gpu_core_a660.base, + &adreno_gpu_core_a660v2.base, + &adreno_gpu_core_a663.base, + &adreno_gpu_core_a680.base, + &adreno_gpu_core_a612.base, + &adreno_gpu_core_a616.base, + &adreno_gpu_core_a610.base, + &adreno_gpu_core_a611.base, + &adreno_gpu_core_a660_shima.base, + &adreno_gpu_core_a702.base, + &adreno_gpu_core_gen7_0_0.base, + &adreno_gpu_core_gen7_0_1.base, + &adreno_gpu_core_a662.base, + &adreno_gpu_core_gen7_2_0.base, + &adreno_gpu_core_gen7_2_1.base, + &adreno_gpu_core_gen7_4_0.base, + &adreno_gpu_core_gen7_9_0.base, + &adreno_gpu_core_gen7_9_1.base, + &adreno_gpu_core_gen7_11_0.base, + &adreno_gpu_core_gen8_3_0.base, + +}; diff --git a/qcom/opensource/graphics-kernel/adreno.c b/qcom/opensource/graphics-kernel/adreno.c new file mode 100644 index 0000000000..2f818ec7a5 --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno.c @@ -0,0 +1,3837 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2002,2007-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "adreno.h" +#include "adreno_a3xx.h" +#include "adreno_a5xx.h" +#include "adreno_a6xx.h" +#include "adreno_compat.h" +#include "adreno_pm4types.h" +#include "adreno_trace.h" +#include "kgsl_bus.h" +#include "kgsl_reclaim.h" +#include "kgsl_trace.h" +#include "kgsl_util.h" + +/* Include the master list of GPU cores that are supported */ +#include "adreno-gpulist.h" + +static void adreno_unbind(struct device *dev); +static void adreno_input_work(struct work_struct *work); +static int adreno_soft_reset(struct kgsl_device *device); +static unsigned int counter_delta(struct kgsl_device *device, + unsigned int reg, unsigned int *counter); +static struct device_node * + adreno_get_gpu_model_node(struct platform_device *pdev); + +static struct adreno_device device_3d0; +static bool adreno_preemption_enable; + +/* Nice level for the higher priority GPU start thread */ +int adreno_wake_nice = -7; + +/* Number of milliseconds to stay active after a wake on touch */ +unsigned int adreno_wake_timeout = 100; + +static u32 get_ucode_version(const u32 *data) +{ + u32 version; + + version = data[1]; + + if ((version & 0xf) != 0xa) + return version; + + version &= ~0xfff; + return version | ((data[3] & 0xfff000) >> 12); +} + +int adreno_get_firmware(struct adreno_device *adreno_dev, + const char *fwfile, struct adreno_firmware *firmware) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct firmware *fw = NULL; + int ret; + + if (!IS_ERR_OR_NULL(firmware->memdesc)) + return 0; + + ret = request_firmware(&fw, fwfile, &device->pdev->dev); + + if (ret) { + dev_err(device->dev, "request_firmware(%s) failed: %d\n", + fwfile, ret); + return ret; + } + + firmware->memdesc = kgsl_allocate_global(device, fw->size - 4, 0, + KGSL_MEMFLAGS_GPUREADONLY, KGSL_MEMDESC_UCODE, + "ucode"); + + ret = PTR_ERR_OR_ZERO(firmware->memdesc); + if (!ret) { + memcpy(firmware->memdesc->hostptr, &fw->data[4], fw->size - 4); + firmware->size = (fw->size - 4) / sizeof(u32); + firmware->version = get_ucode_version((u32 *)fw->data); + } + + release_firmware(fw); + return ret; +} + + +int adreno_zap_shader_load(struct adreno_device *adreno_dev, + const char *name) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + if (!name || adreno_dev->zap_loaded) + return 0; + + ret = kgsl_zap_shader_load(&device->pdev->dev, name); + if (!ret) + adreno_dev->zap_loaded = true; + + return ret; +} + +#if (IS_ENABLED(CONFIG_QCOM_KGSL_HIBERNATION) || IS_ENABLED(CONFIG_DEEPSLEEP)) +static void adreno_zap_shader_unload(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + if (adreno_dev->zap_loaded) { + ret = kgsl_zap_shader_unload(&device->pdev->dev); + if (!ret) + adreno_dev->zap_loaded = false; + } +} +#endif + +/** + * adreno_readreg64() - Read a 64bit register by getting its offset from the + * offset array defined in gpudev node + * @adreno_dev: Pointer to the adreno device + * @lo: lower 32bit register enum that is to be read + * @hi: higher 32bit register enum that is to be read + * @val: 64 bit Register value read is placed here + */ +void adreno_readreg64(struct adreno_device *adreno_dev, + enum adreno_regs lo, enum adreno_regs hi, uint64_t *val) +{ + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int val_lo = 0, val_hi = 0; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (adreno_checkreg_off(adreno_dev, lo)) + kgsl_regread(device, gpudev->reg_offsets[lo], &val_lo); + if (adreno_checkreg_off(adreno_dev, hi)) + kgsl_regread(device, gpudev->reg_offsets[hi], &val_hi); + + *val = (val_lo | ((uint64_t)val_hi << 32)); +} + +/** + * adreno_get_rptr() - Get the current ringbuffer read pointer + * @rb: Pointer the ringbuffer to query + * + * Get the latest rptr + */ +unsigned int adreno_get_rptr(struct adreno_ringbuffer *rb) +{ + struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 rptr = 0; + + if (adreno_is_a3xx(adreno_dev)) + kgsl_regread(device, A3XX_CP_RB_RPTR, &rptr); + else + kgsl_sharedmem_readl(device->scratch, &rptr, + SCRATCH_RB_OFFSET(rb->id, rptr)); + + return rptr; +} + +static void adreno_touch_wakeup(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* + * Don't schedule adreno_start in a high priority workqueue, we are + * already in a workqueue which should be sufficient + */ + kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); + + /* + * When waking up from a touch event we want to stay active long enough + * for the user to send a draw command. The default idle timer timeout + * is shorter than we want so go ahead and push the idle timer out + * further for this special case + */ + mod_timer(&device->idle_timer, + jiffies + msecs_to_jiffies(adreno_wake_timeout)); + +} + +/* + * A workqueue callback responsible for actually turning on the GPU after a + * touch event. kgsl_pwrctrl_change_state(ACTIVE) is used without any + * active_count protection to avoid the need to maintain state. Either + * somebody will start using the GPU or the idle timer will fire and put the + * GPU back into slumber. + */ +static void adreno_input_work(struct work_struct *work) +{ + struct adreno_device *adreno_dev = container_of(work, + struct adreno_device, input_work); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + mutex_lock(&device->mutex); + + device->pwrctrl.wake_on_touch = true; + + ops->touch_wakeup(adreno_dev); + + mutex_unlock(&device->mutex); +} + +/* Wake up the touch event kworker to initiate GPU wakeup */ +void adreno_touch_wake(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + /* + * Don't do anything if anything hasn't been rendered since we've been + * here before + */ + + if (device->pwrctrl.wake_on_touch) + return; + + if (gmu_core_isenabled(device) || (device->state == KGSL_STATE_SLUMBER)) + schedule_work(&adreno_dev->input_work); +} + +/* + * Process input events and schedule work if needed. At this point we are only + * interested in groking EV_ABS touchscreen events + */ +static void adreno_input_event(struct input_handle *handle, unsigned int type, + unsigned int code, int value) +{ + struct kgsl_device *device = handle->handler->private; + + /* Only consider EV_ABS (touch) events */ + if (type == EV_ABS) + adreno_touch_wake(device); +} + +#ifdef CONFIG_INPUT +static int adreno_input_connect(struct input_handler *handler, + struct input_dev *dev, const struct input_device_id *id) +{ + struct input_handle *handle; + int ret; + + handle = kzalloc(sizeof(*handle), GFP_KERNEL); + if (handle == NULL) + return -ENOMEM; + + handle->dev = dev; + handle->handler = handler; + handle->name = handler->name; + + ret = input_register_handle(handle); + if (ret) { + kfree(handle); + return ret; + } + + ret = input_open_device(handle); + if (ret) { + input_unregister_handle(handle); + kfree(handle); + } + + return ret; +} + +static void adreno_input_disconnect(struct input_handle *handle) +{ + input_close_device(handle); + input_unregister_handle(handle); + kfree(handle); +} +#else +static int adreno_input_connect(struct input_handler *handler, + struct input_dev *dev, const struct input_device_id *id) +{ + return 0; +} +static void adreno_input_disconnect(struct input_handle *handle) {} +#endif + +/* + * We are only interested in EV_ABS events so only register handlers for those + * input devices that have EV_ABS events + */ +static const struct input_device_id adreno_input_ids[] = { + { + .flags = INPUT_DEVICE_ID_MATCH_EVBIT, + .evbit = { BIT_MASK(EV_ABS) }, + /* assumption: MT_.._X & MT_.._Y are in the same long */ + .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] = + BIT_MASK(ABS_MT_POSITION_X) | + BIT_MASK(ABS_MT_POSITION_Y) }, + }, + { }, +}; + +static struct input_handler adreno_input_handler = { + .event = adreno_input_event, + .connect = adreno_input_connect, + .disconnect = adreno_input_disconnect, + .name = "kgsl", + .id_table = adreno_input_ids, +}; + +/* + * _soft_reset() - Soft reset GPU + * @adreno_dev: Pointer to adreno device + * + * Soft reset the GPU by doing a AHB write of value 1 to RBBM_SW_RESET + * register. This is used when we want to reset the GPU without + * turning off GFX power rail. The reset when asserted resets + * all the HW logic, restores GPU registers to default state and + * flushes out pending VBIF transactions. + */ +static void _soft_reset(struct adreno_device *adreno_dev) +{ + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + unsigned int reg; + + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_SW_RESET_CMD, 1); + /* + * Do a dummy read to get a brief read cycle delay for the + * reset to take effect + */ + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_SW_RESET_CMD, ®); + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_SW_RESET_CMD, 0); + + /* The SP/TP regulator gets turned off after a soft reset */ + + clear_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, &adreno_dev->priv); + if (gpudev->regulator_enable) + gpudev->regulator_enable(adreno_dev); +} + +/** + * adreno_irqctrl() - Enables/disables the RBBM interrupt mask + * @adreno_dev: Pointer to an adreno_device + * @state: 1 for masked or 0 for unmasked + * Power: The caller of this function must make sure to use OOBs + * so that we know that the GPU is powered on + */ +void adreno_irqctrl(struct adreno_device *adreno_dev, int state) +{ + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (!adreno_dev->irq_mask) + return; + + adreno_writereg(adreno_dev, ADRENO_REG_RBBM_INT_0_MASK, + state ? adreno_dev->irq_mask : 0); + + if (gpudev->swfuse_irqctrl) + gpudev->swfuse_irqctrl(adreno_dev, state); +} + +/* + * adreno_hang_int_callback() - Isr for fatal interrupts that hang GPU + * @adreno_dev: Pointer to device + * @bit: Interrupt bit + */ +void adreno_hang_int_callback(struct adreno_device *adreno_dev, int bit) +{ + dev_crit_ratelimited(KGSL_DEVICE(adreno_dev)->dev, + "MISC: GPU hang detected\n"); + adreno_irqctrl(adreno_dev, 0); + + /* Trigger a fault in the dispatcher - this will effect a restart */ + adreno_dispatcher_fault(adreno_dev, ADRENO_HARD_FAULT); +} + +/* + * adreno_cp_callback() - CP interrupt handler + * @adreno_dev: Adreno device pointer + * @irq: irq number + * + * Handle the cp interrupt generated by GPU. + */ +void adreno_cp_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + adreno_dispatcher_schedule(device); +} + +static irqreturn_t adreno_irq_handler(int irq, void *data) +{ + struct kgsl_device *device = data; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + irqreturn_t ret; + + atomic_inc(&adreno_dev->pending_irq_refcnt); + /* Ensure this increment is done before the IRQ status is updated */ + smp_mb__after_atomic(); + + ret = gpudev->irq_handler(adreno_dev); + + /* Make sure the regwrites are done before the decrement */ + smp_mb__before_atomic(); + atomic_dec(&adreno_dev->pending_irq_refcnt); + /* Ensure other CPUs see the decrement */ + smp_mb__after_atomic(); + + return ret; +} + +static irqreturn_t adreno_freq_limiter_irq_handler(int irq, void *data) +{ + struct kgsl_device *device = data; + + KGSL_PWRCTRL_LOG_FREQLIM(device); + + reset_control_reset(device->freq_limiter_irq_clear); + + return IRQ_HANDLED; +} + +irqreturn_t adreno_irq_callbacks(struct adreno_device *adreno_dev, + const struct adreno_irq_funcs *funcs, u32 status) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + irqreturn_t ret = IRQ_NONE; + + /* Loop through all set interrupts and call respective handlers */ + while (status) { + int i = fls(status) - 1; + + if (funcs[i].func) { + if (adreno_dev->irq_mask & BIT(i)) + funcs[i].func(adreno_dev, i); + } else + dev_crit_ratelimited(device->dev, + "Unhandled interrupt bit %x\n", i); + + ret = IRQ_HANDLED; + + status &= ~BIT(i); + } + + return ret; +} + +static int adreno_get_chipid(struct platform_device *pdev, u32 *chipid); + +static inline bool _rev_match(unsigned int id, unsigned int entry) +{ + return (entry == ANY_ID || entry == id); +} + +static const struct adreno_gpu_core * +_get_gpu_core(struct platform_device *pdev, u32 *chipid) +{ + int i; + struct device_node *node; + + /* + * When "qcom,gpu-models" is defined, use gpu model node to match + * on a compatible string, otherwise match using legacy way. + */ + node = adreno_get_gpu_model_node(pdev); + if (!node || !of_find_property(node, "compatible", NULL)) + node = pdev->dev.of_node; + + *chipid = 0; + + /* Check to see if any of the entries match on a compatible string */ + for (i = 0; i < ARRAY_SIZE(adreno_gpulist); i++) { + if (adreno_gpulist[i]->compatible && + of_device_is_compatible(node, + adreno_gpulist[i]->compatible)) { + /* + * We matched compat string, set chipid based on + * dtsi, else fail. + */ + if (!adreno_get_chipid(pdev, chipid)) + return adreno_gpulist[i]; + + dev_crit(&pdev->dev, + "No chipid associated with %s\n", + adreno_gpulist[i]->compatible); + return NULL; + } + } + + /* No compatible string so try and match on chipid */ + if (!adreno_get_chipid(pdev, chipid)) { + unsigned int core = ADRENO_CHIPID_CORE(*chipid); + unsigned int major = ADRENO_CHIPID_MAJOR(*chipid); + unsigned int minor = ADRENO_CHIPID_MINOR(*chipid); + unsigned int patchid = ADRENO_CHIPID_PATCH(*chipid); + + for (i = 0; i < ARRAY_SIZE(adreno_gpulist); i++) { + if (core == adreno_gpulist[i]->core && + _rev_match(major, adreno_gpulist[i]->major) && + _rev_match(minor, adreno_gpulist[i]->minor) && + _rev_match(patchid, adreno_gpulist[i]->patchid)) + return adreno_gpulist[i]; + } + } + + dev_crit(&pdev->dev, "Unknown GPU chip ID %8.8x\n", *chipid); + return NULL; +} + +static struct { + unsigned int quirk; + const char *prop; +} adreno_quirks[] = { + { ADRENO_QUIRK_TWO_PASS_USE_WFI, "qcom,gpu-quirk-two-pass-use-wfi" }, + { ADRENO_QUIRK_CRITICAL_PACKETS, "qcom,gpu-quirk-critical-packets" }, + { ADRENO_QUIRK_FAULT_DETECT_MASK, "qcom,gpu-quirk-fault-detect-mask" }, + { ADRENO_QUIRK_DISABLE_RB_DP2CLOCKGATING, + "qcom,gpu-quirk-dp2clockgating-disable" }, + { ADRENO_QUIRK_DISABLE_LMLOADKILL, + "qcom,gpu-quirk-lmloadkill-disable" }, + { ADRENO_QUIRK_HFI_USE_REG, "qcom,gpu-quirk-hfi-use-reg" }, + { ADRENO_QUIRK_SECVID_SET_ONCE, "qcom,gpu-quirk-secvid-set-once" }, + { ADRENO_QUIRK_LIMIT_UCHE_GBIF_RW, + "qcom,gpu-quirk-limit-uche-gbif-rw" }, + { ADRENO_QUIRK_CX_GDSC, "qcom,gpu-quirk-cx-gdsc" }, +}; + +static int adreno_get_chipid(struct platform_device *pdev, u32 *chipid) +{ + u32 id; + + if (!of_property_read_u32(pdev->dev.of_node, "qcom,chipid", chipid)) + return 0; + + id = socinfo_get_partinfo_chip_id(SOCINFO_PART_GPU); + if (id) + *chipid = id; + + return id ? 0 : -EINVAL; +} + +static void +adreno_update_soc_hw_revision_quirks(struct adreno_device *adreno_dev, + struct platform_device *pdev) +{ + struct device_node *node = pdev->dev.of_node; + int i; + + /* update quirk */ + for (i = 0; i < ARRAY_SIZE(adreno_quirks); i++) { + if (of_property_read_bool(node, adreno_quirks[i].prop)) + adreno_dev->quirks |= adreno_quirks[i].quirk; + } +} + +static const struct adreno_gpu_core * +adreno_identify_gpu(struct platform_device *pdev, u32 *chipid) +{ + const struct adreno_gpu_core *gpucore; + + gpucore = _get_gpu_core(pdev, chipid); + if (!gpucore) + return ERR_PTR(-ENODEV); + + /* + * Identify non-longer supported targets and spins and print a helpful + * message + */ + if (gpucore->features & ADRENO_DEPRECATED) { + if (gpucore->compatible) + dev_err(&pdev->dev, + "Support for GPU %s has been deprecated\n", + gpucore->compatible); + else + dev_err(&pdev->dev, + "Support for GPU %x.%d.%x.%d has been deprecated\n", + gpucore->core, gpucore->major, + gpucore->minor, gpucore->patchid); + return ERR_PTR(-ENODEV); + } + + return gpucore; +} + +static const struct of_device_id adreno_match_table[] = { + { .compatible = "qcom,kgsl-3d0", .data = &device_3d0 }, + { }, +}; + +MODULE_DEVICE_TABLE(of, adreno_match_table); + +/* Dynamically build the OPP table for the GPU device */ +static void adreno_build_opp_table(struct device *dev, struct kgsl_pwrctrl *pwr) +{ + int i; + + /* Skip if the table has already been populated */ + if (dev_pm_opp_get_opp_count(dev) > 0) + return; + + /* Add all the supported frequencies into the tree */ + for (i = 0; i < pwr->num_pwrlevels; i++) + dev_pm_opp_add(dev, pwr->pwrlevels[i].gpu_freq, 0); +} + +static int adreno_of_parse_pwrlevels(struct adreno_device *adreno_dev, + struct device_node *node) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct device_node *child; + int ret; + + pwr->num_pwrlevels = 0; + + for_each_child_of_node(node, child) { + u32 index, freq = 0, voltage, bus; + struct kgsl_pwrlevel *level; + + ret = of_property_read_u32(child, "reg", &index); + if (ret) { + dev_err(device->dev, "%pOF: powerlevel index not found\n", + child); + goto out; + } + + ret = of_property_read_u32(child, "qcom,gpu-freq", &freq); + if (ret) { + dev_err(device->dev, "%pOF: Unable to read qcom,gpu-freq\n", + child); + goto out; + } + + /* Ignore "zero" powerlevels */ + if (!freq) + continue; + + ret = of_property_read_u32(child, "qcom,level", &voltage); + if (ret) { + dev_err(device->dev, "%pOF: Unable to read qcom,level\n", + child); + goto out; + } + + ret = kgsl_of_property_read_ddrtype(child, "qcom,bus-freq", + &bus); + if (ret) { + dev_err(device->dev, "%pOF:Unable to read qcom,bus-freq\n", + child); + goto out; + } + + if (index >= ARRAY_SIZE(pwr->pwrlevels)) { + dev_err(device->dev, "%pOF: Pwrlevel index %d is out of range\n", + child, index); + continue; + } + + if (index >= pwr->num_pwrlevels) + pwr->num_pwrlevels = index + 1; + + level = &pwr->pwrlevels[index]; + + level->gpu_freq = freq; + level->bus_freq = bus; + level->voltage_level = voltage; + level->cx_level = 0xffffffff; + + of_property_read_u32(child, "qcom,acd-level", + &level->acd_level); + + of_property_read_u32(child, "qcom,cx-level", + &level->cx_level); + + level->bus_min = level->bus_freq; + kgsl_of_property_read_ddrtype(child, + "qcom,bus-min", &level->bus_min); + + level->bus_max = level->bus_freq; + kgsl_of_property_read_ddrtype(child, + "qcom,bus-max", &level->bus_max); + } + + adreno_build_opp_table(&device->pdev->dev, pwr); + return 0; +out: + of_node_put(child); + return ret; +} + +static void adreno_of_get_initial_pwrlevels(struct kgsl_pwrctrl *pwr, + struct device_node *node) +{ + int level; + + /* Get and set the initial power level */ + if (of_property_read_u32(node, "qcom,initial-pwrlevel", &level)) + level = 1; + + if (level < 0 || level >= pwr->num_pwrlevels) + level = 1; + + pwr->active_pwrlevel = level; + pwr->default_pwrlevel = level; + + /* Set the max power level */ + pwr->max_pwrlevel = 0; + + /* Get and set the min power level */ + if (of_property_read_u32(node, "qcom,initial-min-pwrlevel", &level)) + level = pwr->num_pwrlevels - 1; + + if (level < 0 || level >= pwr->num_pwrlevels || level < pwr->default_pwrlevel) + level = pwr->num_pwrlevels - 1; + + pwr->min_render_pwrlevel = level; + pwr->min_pwrlevel = level; +} + +static void adreno_of_get_limits(struct adreno_device *adreno_dev, + struct device_node *node) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwrctrl = &device->pwrctrl; + unsigned int throttle_level; + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_LM) || of_property_read_u32(node, + "qcom,throttle-pwrlevel", &throttle_level)) + return; + + throttle_level = min(throttle_level, pwrctrl->num_pwrlevels - 1); + + pwrctrl->throttle_mask = GENMASK(pwrctrl->num_pwrlevels - 1, + pwrctrl->num_pwrlevels - 1 - throttle_level); + + adreno_dev->lm_enabled = true; +} + +static int adreno_of_get_legacy_pwrlevels(struct adreno_device *adreno_dev, + struct device_node *parent) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct device_node *node; + int ret; + + node = of_find_node_by_name(parent, "qcom,gpu-pwrlevels"); + + if (node == NULL) { + dev_err(&device->pdev->dev, + "Unable to find 'qcom,gpu-pwrlevels'\n"); + return -EINVAL; + } + + ret = adreno_of_parse_pwrlevels(adreno_dev, node); + + if (!ret) { + adreno_of_get_initial_pwrlevels(&device->pwrctrl, parent); + adreno_of_get_limits(adreno_dev, parent); + } + + of_node_put(node); + return ret; +} + +static int adreno_of_get_pwrlevels(struct adreno_device *adreno_dev, + struct device_node *parent) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct device_node *node, *child; + int feature_code, pcode; + + node = of_find_node_by_name(parent, "qcom,gpu-pwrlevel-bins"); + if (node == NULL) + return adreno_of_get_legacy_pwrlevels(adreno_dev, parent); + + feature_code = max_t(int, socinfo_get_feature_code(), SOCINFO_FC_UNKNOWN); + pcode = (feature_code >= SOCINFO_FC_Y0 && feature_code < SOCINFO_FC_INT_RESERVE) ? + max_t(int, socinfo_get_pcode(), SOCINFO_PCODE_UNKNOWN) : SOCINFO_PCODE_UNKNOWN; + + device->soc_code = FIELD_PREP(GENMASK(31, 16), pcode) | + FIELD_PREP(GENMASK(15, 0), feature_code); + + for_each_child_of_node(node, child) { + bool match = false; + int tbl_size; + u32 bin = 0; + + /* Check if the bin has a speed-bin requirement */ + if (!of_property_read_u32(child, "qcom,speed-bin", &bin)) + match = (bin == device->speed_bin); + + /* Check if the bin has a sku-code requirement */ + if (of_get_property(child, "qcom,sku-codes", &tbl_size)) { + int num_codes = tbl_size / sizeof(u32); + int i; + u32 sku_code; + + /* + * If we have a speed-bin requirement that did not match + * keep searching. + */ + if (bin && !match) + continue; + + /* Check if the soc_code matches any of the sku codes */ + match = false; + for (i = 0; i < num_codes; i++) { + if (!of_property_read_u32_index(child, "qcom,sku-codes", + i, &sku_code) && + (sku_code == 0 || device->soc_code == sku_code)) { + match = true; + break; + } + } + } + + if (match) { + int ret; + + ret = adreno_of_parse_pwrlevels(adreno_dev, child); + if (ret) { + of_node_put(child); + return ret; + } + + adreno_of_get_initial_pwrlevels(&device->pwrctrl, child); + + /* + * Check for global throttle-pwrlevel first and override + * with speedbin specific one if found. + */ + adreno_of_get_limits(adreno_dev, parent); + adreno_of_get_limits(adreno_dev, child); + + of_node_put(child); + return 0; + } + } + + dev_err(&device->pdev->dev, + "No match for speed_bin:%d and soc_code:0x%x\n", + device->speed_bin, device->soc_code); + return -ENODEV; +} + +static int register_l3_voter(struct kgsl_device *device) +{ + int ret = 0; + + mutex_lock(&device->mutex); + + if (!device->l3_vote) + goto done; + + /* This indicates that we are already set up */ + if (device->num_l3_pwrlevels != 0) + goto done; + + memset(device->l3_freq, 0x0, sizeof(device->l3_freq)); + + ret = qcom_dcvs_register_voter(KGSL_L3_DEVICE, DCVS_L3, DCVS_SLOW_PATH); + if (ret) { + dev_err_once(&device->pdev->dev, + "Unable to register l3 dcvs voter: %d\n", ret); + goto done; + } + + ret = qcom_dcvs_hw_minmax_get(DCVS_L3, &device->l3_freq[1], + &device->l3_freq[2]); + if (ret) { + dev_err_once(&device->pdev->dev, + "Unable to get min/max for l3 dcvs: %d\n", ret); + qcom_dcvs_unregister_voter(KGSL_L3_DEVICE, DCVS_L3, + DCVS_SLOW_PATH); + memset(device->l3_freq, 0x0, sizeof(device->l3_freq)); + goto done; + } + + device->num_l3_pwrlevels = 3; + +done: + mutex_unlock(&device->mutex); + + return ret; +} + +static int adreno_of_get_power(struct adreno_device *adreno_dev, + struct platform_device *pdev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + ret = adreno_of_get_pwrlevels(adreno_dev, pdev->dev.of_node); + if (ret) + return ret; + + device->pwrctrl.interval_timeout = CONFIG_QCOM_KGSL_IDLE_TIMEOUT; + + /* Set default bus control to true on all targets */ + device->pwrctrl.bus_control = true; + + return 0; +} + +static void adreno_cx_misc_probe(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct resource *res; + + res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM, + "cx_misc"); + + if (res == NULL) + return; + + adreno_dev->cx_misc_len = resource_size(res); + adreno_dev->cx_misc_virt = devm_ioremap(&device->pdev->dev, + res->start, adreno_dev->cx_misc_len); +} + +static void adreno_isense_probe(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct resource *res; + + res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM, + "isense_cntl"); + if (res == NULL) + return; + + adreno_dev->isense_base = res->start - device->regmap.base->start; + adreno_dev->isense_len = resource_size(res); + adreno_dev->isense_virt = devm_ioremap(&device->pdev->dev, res->start, + adreno_dev->isense_len); + if (adreno_dev->isense_virt == NULL) + dev_warn(device->dev, "isense ioremap failed\n"); +} + +/* Read the fuse through the new and fancy nvmem method */ +static int adreno_read_speed_bin(struct platform_device *pdev) +{ + struct nvmem_cell *cell = nvmem_cell_get(&pdev->dev, "speed_bin"); + int ret = PTR_ERR_OR_ZERO(cell); + void *buf; + int val = 0; + size_t len; + + if (ret) { + if (ret == -ENOENT) + return 0; + + return ret; + } + + buf = nvmem_cell_read(cell, &len); + nvmem_cell_put(cell); + + if (IS_ERR(buf)) + return PTR_ERR(buf); + + memcpy(&val, buf, min(len, sizeof(val))); + kfree(buf); + + return val; +} + +static int adreno_read_gpu_model_fuse(struct platform_device *pdev) +{ + struct nvmem_cell *cell = nvmem_cell_get(&pdev->dev, "gpu_model"); + void *buf; + int val = 0; + size_t len; + + if (IS_ERR(cell)) + return PTR_ERR(cell); + + buf = nvmem_cell_read(cell, &len); + nvmem_cell_put(cell); + + if (IS_ERR(buf)) + return PTR_ERR(buf); + + memcpy(&val, buf, min(len, sizeof(val))); + kfree(buf); + + return val; +} + +static struct device_node * +adreno_get_gpu_model_node(struct platform_device *pdev) +{ + struct device_node *node, *child; + int fuse_model = adreno_read_gpu_model_fuse(pdev); + + if (fuse_model < 0) + return NULL; + + node = of_find_node_by_name(pdev->dev.of_node, "qcom,gpu-models"); + if (node == NULL) + return NULL; + + for_each_child_of_node(node, child) { + u32 model; + + if (of_property_read_u32(child, "qcom,gpu-model-id", &model)) + continue; + + if (model == fuse_model) { + of_node_put(node); + return child; + } + } + + of_node_put(node); + + return NULL; +} + +const char *adreno_get_gpu_model(struct kgsl_device *device) +{ + struct device_node *node; + static char gpu_model[32]; + const char *model; + int ret; + + if (strlen(gpu_model)) + return gpu_model; + + node = adreno_get_gpu_model_node(device->pdev); + if (!node) + node = of_node_get(device->pdev->dev.of_node); + + ret = of_property_read_string(node, "qcom,gpu-model", &model); + of_node_put(node); + + if (!ret) + goto done; + + model = socinfo_get_partinfo_part_name(SOCINFO_PART_GPU); + if (model) + goto done; + + scnprintf(gpu_model, sizeof(gpu_model), "Adreno%u%u%uv%u", + (u32)ADRENO_CHIPID_CORE(ADRENO_DEVICE(device)->chipid), + (u32)ADRENO_CHIPID_MAJOR(ADRENO_DEVICE(device)->chipid), + (u32)ADRENO_CHIPID_MINOR(ADRENO_DEVICE(device)->chipid), + (u32)ADRENO_CHIPID_PATCH(ADRENO_DEVICE(device)->chipid) + 1); + + return gpu_model; + +done: + strscpy(gpu_model, model, sizeof(gpu_model)); + return gpu_model; +} + +static u32 adreno_get_vk_device_id(struct kgsl_device *device) +{ + struct device_node *node; + static u32 device_id; + u32 vk_id; + int ret; + + if (device_id) + return device_id; + + node = adreno_get_gpu_model_node(device->pdev); + if (!node) + node = of_node_get(device->pdev->dev.of_node); + + ret = of_property_read_u32(node, "qcom,vk-device-id", &device_id); + of_node_put(node); + if (!ret) + return device_id; + + vk_id = socinfo_get_partinfo_vulkan_id(SOCINFO_PART_GPU); + device_id = vk_id ? vk_id : ADRENO_DEVICE(device)->chipid; + + return device_id; +} + +#if IS_ENABLED(CONFIG_QCOM_LLCC) +static int adreno_probe_llcc(struct adreno_device *adreno_dev, + struct platform_device *pdev) +{ + int ret; + + /* Get the system cache slice descriptor for GPU */ + adreno_dev->gpu_llc_slice = llcc_slice_getd(LLCC_GPU); + ret = PTR_ERR_OR_ZERO(adreno_dev->gpu_llc_slice); + + if (ret) { + /* Propagate EPROBE_DEFER back to the probe function */ + if (ret == -EPROBE_DEFER) + return ret; + + if (ret != -ENOENT) + dev_warn(&pdev->dev, + "Unable to get the GPU LLC slice: %d\n", ret); + } else + adreno_dev->gpu_llc_slice_enable = true; + + /* Get the system cache slice descriptor for GPU pagetables */ + adreno_dev->gpuhtw_llc_slice = llcc_slice_getd(LLCC_GPUHTW); + ret = PTR_ERR_OR_ZERO(adreno_dev->gpuhtw_llc_slice); + if (ret) { + if (ret == -EPROBE_DEFER) { + llcc_slice_putd(adreno_dev->gpu_llc_slice); + return ret; + } + + if (ret != -ENOENT) + dev_warn(&pdev->dev, + "Unable to get GPU HTW LLC slice: %d\n", ret); + } else + adreno_dev->gpuhtw_llc_slice_enable = true; + + return 0; +} +#else +static int adreno_probe_llcc(struct adreno_device *adreno_dev, + struct platform_device *pdev) +{ + return 0; +} +#endif + +static void adreno_regmap_op_preaccess(struct kgsl_regmap_region *region) +{ + struct kgsl_device *device = region->priv; + /* + * kgsl panic notifier will be called in atomic context to get + * GPU snapshot. Also panic handler will skip snapshot dumping + * incase GPU is in SLUMBER state. So we can safely ignore the + * kgsl_pre_hwaccess(). + */ + if (!device->snapshot_atomic && !in_interrupt()) + kgsl_pre_hwaccess(device); +} + +static const struct kgsl_regmap_ops adreno_regmap_ops = { + .preaccess = adreno_regmap_op_preaccess, +}; + +static const struct kgsl_functable adreno_functable; + +static void adreno_setup_device(struct adreno_device *adreno_dev) +{ + u32 i; + + adreno_dev->dev.name = "kgsl-3d0"; + adreno_dev->dev.ftbl = &adreno_functable; + + init_completion(&adreno_dev->dev.hwaccess_gate); + init_completion(&adreno_dev->dev.halt_gate); + + idr_init(&adreno_dev->dev.context_idr); + + mutex_init(&adreno_dev->dev.mutex); + INIT_LIST_HEAD(&adreno_dev->dev.globals); + + /* Set the fault tolerance policy to replay, skip, throttle */ + adreno_dev->ft_policy = BIT(KGSL_FT_REPLAY) | + BIT(KGSL_FT_SKIPCMD) | BIT(KGSL_FT_THROTTLE); + + /* Enable command timeouts by default */ + adreno_dev->long_ib_detect = true; + + INIT_WORK(&adreno_dev->input_work, adreno_input_work); + + INIT_LIST_HEAD(&adreno_dev->active_list); + spin_lock_init(&adreno_dev->active_list_lock); + + for (i = 0; i < ARRAY_SIZE(adreno_dev->ringbuffers); i++) { + struct adreno_ringbuffer *rb = &adreno_dev->ringbuffers[i]; + + INIT_LIST_HEAD(&rb->events.group); + } + + /* + * Some GPUs needs specific alignment for UCHE GMEM base address. + * Configure UCHE GMEM base based on GMEM size and align it accordingly. + * This needs to be done based on GMEM size to avoid overlap between + * RB and UCHE GMEM range. + */ + if (adreno_dev->gpucore->uche_gmem_alignment) + adreno_dev->uche_gmem_base = + ALIGN(adreno_dev->gpucore->gmem_size, + adreno_dev->gpucore->uche_gmem_alignment); +} + +static const struct of_device_id adreno_component_match[] = { + { .compatible = "qcom,gen8-gmu" }, + { .compatible = "qcom,gen7-gmu" }, + { .compatible = "qcom,gpu-gmu" }, + { .compatible = "qcom,gpu-rgmu" }, + { .compatible = "qcom,kgsl-smmu-v2" }, + { .compatible = "qcom,smmu-kgsl-cb" }, + {}, +}; + +static int adreno_irq_setup(struct platform_device *pdev, + struct adreno_device *adreno_dev) +{ + if (!adreno_dev->irq_mask) + return 0; + + return kgsl_request_irq(pdev, "kgsl_3d0_irq", adreno_irq_handler, KGSL_DEVICE(adreno_dev)); +} + +int adreno_device_probe(struct platform_device *pdev, + struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct device *dev = &pdev->dev; + unsigned int priv = 0; + int status; + u32 size; + + KGSL_BOOT_MARKER("GPU Init"); + + /* Initialize the adreno device structure */ + adreno_setup_device(adreno_dev); + + dev_set_drvdata(dev, device); + + device->pdev = pdev; + + adreno_update_soc_hw_revision_quirks(adreno_dev, pdev); + + status = adreno_read_speed_bin(pdev); + if (status < 0) + goto err; + + device->speed_bin = status; + + status = adreno_of_get_power(adreno_dev, pdev); + if (status) + goto err; + + status = kgsl_bus_init(device, pdev); + if (status) + goto err; + + status = kgsl_regmap_init(pdev, &device->regmap, "kgsl_3d0_reg_memory", + &adreno_regmap_ops, device); + if (status) + goto err_bus_close; + + /* + * The SMMU APIs use unsigned long for virtual addresses which means + * that we cannot use 64 bit virtual addresses on a 32 bit kernel even + * though the hardware and the rest of the KGSL driver supports it. + */ + if (adreno_support_64bit(adreno_dev)) + kgsl_mmu_set_feature(device, KGSL_MMU_64BIT); + + /* + * Set the SMMU aperture on A6XX/Gen7 targets to use per-process + * pagetables. + */ + if (ADRENO_GPUREV(adreno_dev) >= 600) + kgsl_mmu_set_feature(device, KGSL_MMU_SMMU_APERTURE); + + if (ADRENO_FEATURE(adreno_dev, ADRENO_IOCOHERENT)) + kgsl_mmu_set_feature(device, KGSL_MMU_IO_COHERENT); + + /* + * Support VBOs on hardware where HLOS has access to PRR registers + * configuration. + */ + if (!adreno_is_a650(adreno_dev)) + kgsl_mmu_set_feature(device, KGSL_MMU_SUPPORT_VBO); + + if (adreno_preemption_enable) + adreno_dev->preempt_override = true; + + device->pwrctrl.bus_width = adreno_dev->gpucore->bus_width; + + device->mmu.secured = (IS_ENABLED(CONFIG_QCOM_SECURE_BUFFER) && + ADRENO_FEATURE(adreno_dev, ADRENO_CONTENT_PROTECTION)); + + /* Probe the LLCC - this could return -EPROBE_DEFER */ + status = adreno_probe_llcc(adreno_dev, pdev); + if (status) + goto err_bus_close; + + /* + * IF the GPU HTW slice was successsful set the MMU feature so the + * domain can set the appropriate attributes + */ + if (!IS_ERR_OR_NULL(adreno_dev->gpuhtw_llc_slice)) + kgsl_mmu_set_feature(device, KGSL_MMU_LLCC_ENABLE); + + /* Bind the components before doing the KGSL platform probe. */ + status = component_bind_all(dev, NULL); + if (status) + goto err_remove_llcc; + + status = adreno_irq_setup(pdev, adreno_dev); + if (status < 0) + goto err_unbind; + + device->pwrctrl.interrupt_num = status; + + device->freq_limiter_intr_num = kgsl_request_irq_optional(pdev, "freq_limiter_irq", + adreno_freq_limiter_irq_handler, device); + + device->freq_limiter_irq_clear = + devm_reset_control_get(&pdev->dev, "freq_limiter_irq_clear"); + + status = kgsl_device_platform_probe(device); + if (status) + goto err_unbind; + + adreno_fence_trace_array_init(device); + + /* Add CX_DBGC block to the regmap*/ + kgsl_regmap_add_region(&device->regmap, pdev, "cx_dbgc", NULL, NULL); + + /* Probe for the optional CX_MISC block */ + adreno_cx_misc_probe(device); + + adreno_isense_probe(device); + + /* Allocate the memstore for storing timestamps and other useful info */ + + if (ADRENO_FEATURE(adreno_dev, ADRENO_APRIV)) + priv |= KGSL_MEMDESC_PRIVILEGED; + + device->memstore = kgsl_allocate_global(device, + KGSL_MEMSTORE_SIZE, 0, 0, priv, "memstore"); + + status = PTR_ERR_OR_ZERO(device->memstore); + if (status) { + trace_array_put(device->fence_trace_array); + kgsl_device_platform_remove(device); + goto err_unbind; + } + + /* Initialize the snapshot engine */ + size = adreno_dev->gpucore->snapshot_size; + + /* + * Use a default size if one wasn't specified, but print a warning so + * the developer knows to fix it + */ + + if (WARN(!size, "The snapshot size was not specified in the gpucore\n")) + size = SZ_1M; + + kgsl_device_snapshot_probe(device, size); + + adreno_debugfs_init(adreno_dev); + adreno_profile_init(adreno_dev); + + adreno_dev->perfcounter = false; + + adreno_sysfs_init(adreno_dev); + + /* Ignore return value, as driver can still function without pwrscale enabled */ + kgsl_pwrscale_init(device, pdev, CONFIG_QCOM_ADRENO_DEFAULT_GOVERNOR); + + if (ADRENO_FEATURE(adreno_dev, ADRENO_L3_VOTE)) + device->l3_vote = true; + +#ifdef CONFIG_INPUT + + if (!of_property_read_bool(pdev->dev.of_node, + "qcom,disable-wake-on-touch")) { + adreno_input_handler.private = device; + /* + * It isn't fatal if we cannot register the input handler. Sad, + * perhaps, but not fatal + */ + if (input_register_handler(&adreno_input_handler)) { + adreno_input_handler.private = NULL; + dev_err(device->dev, + "Unable to register the input handler\n"); + } + } +#endif + + kgsl_qcom_va_md_register(device); + + KGSL_BOOT_MARKER("GPU Ready"); + + return 0; + +err_unbind: + component_unbind_all(dev, NULL); + +err_remove_llcc: + if (!IS_ERR_OR_NULL(adreno_dev->gpu_llc_slice)) + llcc_slice_putd(adreno_dev->gpu_llc_slice); + + if (!IS_ERR_OR_NULL(adreno_dev->gpuhtw_llc_slice)) + llcc_slice_putd(adreno_dev->gpuhtw_llc_slice); + +err_bus_close: + kgsl_bus_close(device); + +err: + device->pdev = NULL; + dev_err_probe(&pdev->dev, status, "adreno device probe failed\n"); + return status; +} + +static int adreno_bind(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + const struct adreno_gpu_core *gpucore; + int ret; + u32 chipid; + + gpucore = adreno_identify_gpu(pdev, &chipid); + if (IS_ERR(gpucore)) + return PTR_ERR(gpucore); + + ret = gpucore->gpudev->probe(pdev, chipid, gpucore); + + if (!ret) { + struct kgsl_device *device = dev_get_drvdata(dev); + + device->pdev_loaded = true; + srcu_init_notifier_head(&device->nh); + } else { + /* + * Handle resource clean up through unbind, instead of a + * lengthy goto error path. + */ + adreno_unbind(dev); + } + + return ret; +} + +static void adreno_unbind(struct device *dev) +{ + struct adreno_device *adreno_dev; + struct kgsl_device *device; + const struct adreno_gpudev *gpudev; + + device = dev_get_drvdata(dev); + if (!device) + return; + + /* Return if cleanup happens in adreno_device_probe */ + if (!device->pdev) + return; + + if (device->pdev_loaded) { + srcu_cleanup_notifier_head(&device->nh); + device->pdev_loaded = false; + } + + adreno_dev = ADRENO_DEVICE(device); + gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + trace_array_put(device->fence_trace_array); + + if (gpudev->remove != NULL) + gpudev->remove(adreno_dev); + +#ifdef CONFIG_INPUT + if (adreno_input_handler.private) + input_unregister_handler(&adreno_input_handler); +#endif + + kgsl_qcom_va_md_unregister(device); + adreno_coresight_remove(adreno_dev); + adreno_profile_close(adreno_dev); + + /* Release the system cache slice descriptor */ + if (!IS_ERR_OR_NULL(adreno_dev->gpu_llc_slice)) + llcc_slice_putd(adreno_dev->gpu_llc_slice); + + if (!IS_ERR_OR_NULL(adreno_dev->gpuhtw_llc_slice)) + llcc_slice_putd(adreno_dev->gpuhtw_llc_slice); + + kgsl_pwrscale_close(device); + + if (adreno_dev->dispatch_ops && adreno_dev->dispatch_ops->close) + adreno_dev->dispatch_ops->close(adreno_dev); + + kgsl_device_platform_remove(device); + + component_unbind_all(dev, NULL); + + kgsl_bus_close(device); + device->pdev = NULL; + + if (device->num_l3_pwrlevels != 0) + qcom_dcvs_unregister_voter(KGSL_L3_DEVICE, DCVS_L3, + DCVS_SLOW_PATH); + + clear_bit(ADRENO_DEVICE_PWRON_FIXUP, &adreno_dev->priv); + clear_bit(ADRENO_DEVICE_INITIALIZED, &adreno_dev->priv); +} + +static void adreno_resume(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (device->state == KGSL_STATE_SUSPEND) { + adreno_put_gpu_halt(adreno_dev); + kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); + } else if (device->state != KGSL_STATE_INIT) { + /* + * This is an error situation so wait for the device to idle and + * then put the device in SLUMBER state. This will get us to + * the right place when we resume. + */ + if (device->state == KGSL_STATE_ACTIVE) + adreno_idle(device); + kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); + dev_err(device->dev, "resume invoked without a suspend\n"); + } +} + +static int adreno_pm_resume(struct device *dev) +{ + struct kgsl_device *device = dev_get_drvdata(dev); + struct adreno_device *adreno_dev; + const struct adreno_power_ops *ops; + + if (!device) + return 0; + + adreno_dev = ADRENO_DEVICE(device); + ops = ADRENO_POWER_OPS(adreno_dev); + +#if IS_ENABLED(CONFIG_DEEPSLEEP) + if (pm_suspend_via_firmware()) { + struct kgsl_iommu *iommu = &device->mmu.iommu; + int status = kgsl_set_smmu_aperture(device, &iommu->user_context); + + if (status) + return status; + + status = kgsl_set_smmu_lpac_aperture(device, &iommu->lpac_context); + if (status < 0) + return status; + } +#endif + + mutex_lock(&device->mutex); + ops->pm_resume(adreno_dev); + mutex_unlock(&device->mutex); + + kgsl_reclaim_start(); + return 0; +} + +static int adreno_suspend(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int status = kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND); + + if (!status && device->state == KGSL_STATE_SUSPEND) + adreno_get_gpu_halt(adreno_dev); + + return status; +} + +static int adreno_pm_suspend(struct device *dev) +{ + struct kgsl_device *device = dev_get_drvdata(dev); + struct adreno_device *adreno_dev; + const struct adreno_power_ops *ops; + int status; + + if (!device) + return 0; + + adreno_dev = ADRENO_DEVICE(device); + ops = ADRENO_POWER_OPS(adreno_dev); + + mutex_lock(&device->mutex); + status = ops->pm_suspend(adreno_dev); + +#if IS_ENABLED(CONFIG_DEEPSLEEP) + if (!status && pm_suspend_via_firmware()) + adreno_zap_shader_unload(adreno_dev); +#endif + + mutex_unlock(&device->mutex); + + if (status) + return status; + + /* + * When the device enters in suspend state, the CX can be collapsed causing + * the GPU CX timer to pause. Clear the ADRENO_DEVICE_CX_TIMER_INITIALIZED + * flag to ensure that the CX timer is reseeded during resume. + */ + clear_bit(ADRENO_DEVICE_CX_TIMER_INITIALIZED, &adreno_dev->priv); + kgsl_reclaim_close(); + kthread_flush_worker(device->events_worker); + flush_workqueue(kgsl_driver.lockless_workqueue); + + return status; +} + +void adreno_create_profile_buffer(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int priv = 0; + + if (ADRENO_FEATURE(adreno_dev, ADRENO_APRIV)) + priv = KGSL_MEMDESC_PRIVILEGED; + + adreno_allocate_global(device, &adreno_dev->profile_buffer, + PAGE_SIZE, 0, 0, priv, "alwayson"); + + adreno_dev->profile_index = 0; + + if (!IS_ERR(adreno_dev->profile_buffer)) + set_bit(ADRENO_DEVICE_DRAWOBJ_PROFILE, + &adreno_dev->priv); +} + +static int adreno_init(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + int ret; + + ret = kgsl_pwrctrl_change_state(device, KGSL_STATE_INIT); + if (ret) + return ret; + + /* + * initialization only needs to be done once initially until + * device is shutdown + */ + if (test_bit(ADRENO_DEVICE_INITIALIZED, &adreno_dev->priv)) + return 0; + + ret = gpudev->init(adreno_dev); + if (ret) + return ret; + + set_bit(ADRENO_DEVICE_INITIALIZED, &adreno_dev->priv); + + return 0; +} + +static bool regulators_left_on(struct kgsl_device *device) +{ + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + + if (gmu_core_gpmu_isenabled(device)) + return false; + + if (!IS_ERR_OR_NULL(pwr->cx_gdsc)) + if (regulator_is_enabled(pwr->cx_gdsc)) + return true; + + if (!IS_ERR_OR_NULL(pwr->gx_gdsc)) + return regulator_is_enabled(pwr->gx_gdsc); + + return false; +} + +void adreno_set_active_ctxs_null(struct adreno_device *adreno_dev) +{ + int i; + struct adreno_ringbuffer *rb; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + if (rb->drawctxt_active) + kgsl_context_put(&(rb->drawctxt_active->base)); + rb->drawctxt_active = NULL; + + kgsl_sharedmem_writel(device->scratch, + SCRATCH_RB_OFFSET(rb->id, current_rb_ptname), + 0); + } +} + +static int adreno_open(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + /* + * active_cnt special case: we are starting up for the first + * time, so use this sequence instead of the kgsl_pwrctrl_wake() + * which will be called by adreno_active_count_get(). + */ + atomic_inc(&device->active_cnt); + + memset(device->memstore->hostptr, 0, device->memstore->size); + + ret = adreno_init(device); + if (ret) + goto err; + + ret = adreno_start(device, 0); + if (ret) + goto err; + + complete_all(&device->hwaccess_gate); + kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); + adreno_active_count_put(adreno_dev); + + return 0; +err: + kgsl_pwrctrl_change_state(device, KGSL_STATE_INIT); + atomic_dec(&device->active_cnt); + + return ret; +} + +static int adreno_first_open(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + if (!device->pdev_loaded) + return -ENODEV; + + return ops->first_open(adreno_dev); +} + +static int adreno_close(struct adreno_device *adreno_dev) +{ + return kgsl_pwrctrl_change_state(KGSL_DEVICE(adreno_dev), + KGSL_STATE_INIT); +} + +static int adreno_last_close(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + /* + * Wait up to 1 second for the active count to go low + * and then start complaining about it + */ + if (kgsl_active_count_wait(device, 0, HZ)) { + dev_err(device->dev, + "Waiting for the active count to become 0\n"); + + while (kgsl_active_count_wait(device, 0, HZ)) + dev_err(device->dev, + "Still waiting for the active count\n"); + } + + return ops->last_close(adreno_dev); +} + +static int adreno_pwrctrl_active_count_get(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret = 0; + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return -EINVAL; + + if ((atomic_read(&device->active_cnt) == 0) && + (device->state != KGSL_STATE_ACTIVE)) { + mutex_unlock(&device->mutex); + wait_for_completion(&device->hwaccess_gate); + mutex_lock(&device->mutex); + device->pwrctrl.superfast = true; + ret = kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); + } + if (ret == 0) + atomic_inc(&device->active_cnt); + trace_kgsl_active_count(device, + (unsigned long) __builtin_return_address(0)); + return ret; +} + +static void adreno_pwrctrl_active_count_put(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return; + + if (WARN(atomic_read(&device->active_cnt) == 0, + "Unbalanced get/put calls to KGSL active count\n")) + return; + + if (atomic_dec_and_test(&device->active_cnt)) { + kgsl_pwrscale_update_stats(device); + kgsl_pwrscale_update(device); + + kgsl_start_idle_timer(device); + } + + trace_kgsl_active_count(device, + (unsigned long) __builtin_return_address(0)); + + wake_up(&device->active_cnt_wq); +} + +int adreno_active_count_get(struct adreno_device *adreno_dev) +{ + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + return ops->active_count_get(adreno_dev); +} + +void adreno_active_count_put(struct adreno_device *adreno_dev) +{ + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + ops->active_count_put(adreno_dev); +} + +void adreno_get_bus_counters(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret = 0; + + if (!device->pwrctrl.bus_control) + return; + + /* VBIF waiting for RAM */ + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF_PWR, 0, + &adreno_dev->starved_ram_lo, NULL); + + /* Target has GBIF */ + if (adreno_is_gen8(adreno_dev) || adreno_is_gen7(adreno_dev) || + (adreno_is_a6xx(adreno_dev) && !adreno_is_a630(adreno_dev))) { + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF_PWR, 1, + &adreno_dev->starved_ram_lo_ch1, NULL); + + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI0_READ_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo, NULL); + + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI1_READ_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo_ch1_read, NULL); + + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI0_WRITE_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo_ch0_write, NULL); + + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI1_WRITE_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo_ch1_write, NULL); + } else { + /* VBIF DDR cycles */ + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + VBIF_AXI_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo, NULL); + } + + if (ret) + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "Unable to get perf counters for bus DCVS\n"); +} + +#define ADRENO_AHB_MIN_TIMEOUT_VAL_USEC 1000 + +u32 adreno_get_ahb_timeout_val(struct adreno_device *adreno_dev, u32 noc_timeout_us) +{ + u64 cycles, hub_clk_freq = adreno_dev->gmu_hub_clk_freq; + u32 timeout_val; + + if (!noc_timeout_us) + return 0; + + do_div(hub_clk_freq, HZ_PER_MHZ); + cycles = hub_clk_freq * noc_timeout_us; + + /* + * Get max possible AHB timeout value which is less than the GPU NOC timeout value. + * When cycles are exact power of two, the calculated AHB timeout value will be same + * as GPU config NOC timeout. Just reduce one cycle to make sure we do not program AHB + * timeout same as GPU config NOC timeout. + */ + if (is_power_of_2(cycles)) + cycles -= 1; + + timeout_val = ilog2(cycles); + + /* + * Make sure, AHB timeout value fits into bit fields and it is not too low + * which can cause false timeouts. + */ + if ((timeout_val > GENMASK(4, 0)) || + ((ADRENO_AHB_MIN_TIMEOUT_VAL_USEC * hub_clk_freq) > (1 << timeout_val))) { + dev_warn(adreno_dev->dev.dev, "Invalid AHB timeout_val %u\n", timeout_val); + return 0; + } + + /* + * Return (timeout_val - 1). Based on timeout_val programmed, a timeout will occur if + * an AHB transaction is not completed in 2 ^ (timeout_val + 1) cycles. + */ + return (timeout_val - 1); +} + +/** + * _adreno_start - Power up the GPU and prepare to accept commands + * @adreno_dev: Pointer to an adreno_device structure + * + * The core function that powers up and initalizes the GPU. This function is + * called at init and after coming out of SLUMBER + */ +static int _adreno_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + int status; + unsigned int state = device->state; + bool regulator_left_on; + + /* make sure ADRENO_DEVICE_STARTED is not set here */ + WARN_ON(test_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv)); + + regulator_left_on = regulators_left_on(device); + + /* Clear any GPU faults that might have been left over */ + adreno_clear_gpu_fault(adreno_dev); + + /* Put the GPU in a responsive state */ + status = kgsl_pwrctrl_change_state(device, KGSL_STATE_AWARE); + if (status) + goto error_pwr_off; + + /* Set any stale active contexts to NULL */ + adreno_set_active_ctxs_null(adreno_dev); + + /* Set the bit to indicate that we've just powered on */ + set_bit(ADRENO_DEVICE_PWRON, &adreno_dev->priv); + + /* Clear the busy_data stats - we're starting over from scratch */ + memset(&adreno_dev->busy_data, 0, sizeof(adreno_dev->busy_data)); + + /* Soft reset the GPU if a regulator is stuck on*/ + if (regulator_left_on) + _soft_reset(adreno_dev); + + /* Start the GPU */ + status = gpudev->start(adreno_dev); + if (status) + goto error_pwr_off; + + /* Re-initialize the coresight registers if applicable */ + adreno_coresight_start(adreno_dev); + + adreno_irqctrl(adreno_dev, 1); + + adreno_perfcounter_start(adreno_dev); + + /* Clear FSR here in case it is set from a previous pagefault */ + kgsl_mmu_clear_fsr(&device->mmu); + + status = gpudev->rb_start(adreno_dev); + if (status) + goto error_pwr_off; + + /* + * At this point it is safe to assume that we recovered. Setting + * this field allows us to take a new snapshot for the next failure + * if we are prioritizing the first unrecoverable snapshot. + */ + if (device->snapshot) + device->snapshot->recovered = true; + + /* Start the dispatcher */ + adreno_dispatcher_start(device); + + device->reset_counter++; + + set_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + + /* + * There is a possible deadlock scenario during kgsl firmware reading + * (request_firmware) and devfreq update calls. During first boot, kgsl + * device mutex is held and then request_firmware is called for reading + * firmware. request_firmware internally takes dev_pm_qos_mtx lock. + * Whereas in case of devfreq update calls triggered by thermal/bcl or + * devfreq sysfs, it first takes the same dev_pm_qos_mtx lock and then + * tries to take kgsl device mutex as part of get_dev_status/target + * calls. This results in deadlock when both thread are unable to acquire + * the mutex held by other thread. Enable devfreq updates now as we are + * done reading all firmware files. + */ + device->pwrscale.devfreq_enabled = true; + + return 0; + +error_pwr_off: + /* set the state back to original state */ + kgsl_pwrctrl_change_state(device, state); + + return status; +} + +/** + * adreno_start() - Power up and initialize the GPU + * @device: Pointer to the KGSL device to power up + * @priority: Boolean flag to specify of the start should be scheduled in a low + * latency work queue + * + * Power up the GPU and initialize it. If priority is specified then elevate + * the thread priority for the duration of the start operation + */ +int adreno_start(struct kgsl_device *device, int priority) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + int nice = task_nice(current); + int ret; + + if (priority && (adreno_wake_nice < nice)) + set_user_nice(current, adreno_wake_nice); + + ret = _adreno_start(adreno_dev); + + if (priority) + set_user_nice(current, nice); + + return ret; +} + +static int adreno_stop(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + int error = 0; + + if (!test_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv)) + return 0; + + kgsl_pwrscale_update_stats(device); + + adreno_irqctrl(adreno_dev, 0); + + /* Save active coresight registers if applicable */ + adreno_coresight_stop(adreno_dev); + + /* Save physical performance counter values before GPU power down*/ + adreno_perfcounter_save(adreno_dev); + + if (gpudev->clear_pending_transactions) + gpudev->clear_pending_transactions(adreno_dev); + + adreno_dispatcher_stop(adreno_dev); + + adreno_ringbuffer_stop(adreno_dev); + + adreno_llcc_slice_deactivate(adreno_dev); + + adreno_set_active_ctxs_null(adreno_dev); + + clear_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + + return error; +} + +/** + * adreno_reset() - Helper function to reset the GPU + * @device: Pointer to the KGSL device structure for the GPU + * @fault: Type of fault. Needed to skip soft reset for MMU fault + * + * Try to reset the GPU to recover from a fault. First, try to do a low latency + * soft reset. If the soft reset fails for some reason, then bring out the big + * guns and toggle the footswitch. + */ +int adreno_reset(struct kgsl_device *device, int fault) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + int ret = -EINVAL; + int i; + + if (gpudev->reset) + return gpudev->reset(adreno_dev); + + /* + * Try soft reset first Do not do soft reset for a IOMMU fault (because + * the IOMMU hardware needs a reset too) + */ + + if (!(fault & ADRENO_IOMMU_PAGE_FAULT)) + ret = adreno_soft_reset(device); + + if (ret) { + /* If soft reset failed/skipped, then pull the power */ + kgsl_pwrctrl_change_state(device, KGSL_STATE_INIT); + /* since device is officially off now clear start bit */ + clear_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + + /* Try to reset the device */ + ret = adreno_start(device, 0); + + for (i = 0; ret && i < 4; i++) { + msleep(20); + ret = adreno_start(device, 0); + } + + if (ret) + return ret; + + if (i != 0) + dev_warn(device->dev, + "Device hard reset tried %d tries\n", i); + } + + /* + * If active_cnt is non-zero then the system was active before + * going into a reset - put it back in that state + */ + + if (atomic_read(&device->active_cnt)) + kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); + + return ret; +} + +static int copy_prop(struct kgsl_device_getproperty *param, + void *src, size_t size) +{ + if (copy_to_user(param->value, src, + min_t(u32, size, param->sizebytes))) + return -EFAULT; + + return 0; +} + +static int adreno_prop_device_info(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_devinfo devinfo = { + .device_id = device->id + 1, + .chip_id = adreno_dev->chipid, + .mmu_enabled = kgsl_mmu_has_feature(device, KGSL_MMU_PAGED), + .gmem_gpubaseaddr = 0, + .gmem_sizebytes = adreno_dev->gpucore->gmem_size, + }; + + return copy_prop(param, &devinfo, sizeof(devinfo)); +} + +static int adreno_prop_gpu_model(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + struct kgsl_gpu_model model = {0}; + + strscpy(model.gpu_model, adreno_get_gpu_model(device), + sizeof(model.gpu_model)); + + return copy_prop(param, &model, sizeof(model)); +} + +static int adreno_prop_device_shadow(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + struct kgsl_shadowprop shadowprop = { 0 }; + + if (device->memstore->hostptr) { + /* Pass a dummy address to identify memstore */ + shadowprop.gpuaddr = KGSL_MEMSTORE_TOKEN_ADDRESS; + shadowprop.size = device->memstore->size; + + shadowprop.flags = KGSL_FLAGS_INITIALIZED | + KGSL_FLAGS_PER_CONTEXT_TIMESTAMPS; + } + + return copy_prop(param, &shadowprop, sizeof(shadowprop)); +} + +static int adreno_prop_device_qdss_stm(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + struct kgsl_qdss_stm_prop qdssprop = {0}; + + if (!IS_ERR_OR_NULL(device->qdss_desc)) { + qdssprop.gpuaddr = device->qdss_desc->gpuaddr; + qdssprop.size = device->qdss_desc->size; + } + + return copy_prop(param, &qdssprop, sizeof(qdssprop)); +} + +static int adreno_prop_device_qtimer(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + struct kgsl_qtimer_prop qtimerprop = {0}; + + if (!IS_ERR_OR_NULL(device->qtimer_desc)) { + qtimerprop.gpuaddr = device->qtimer_desc->gpuaddr; + qtimerprop.size = device->qtimer_desc->size; + } + + return copy_prop(param, &qtimerprop, sizeof(qtimerprop)); +} + +static int adreno_prop_s32(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + int val = 0; + + if (param->type == KGSL_PROP_MMU_ENABLE) + val = kgsl_mmu_has_feature(device, KGSL_MMU_PAGED); + else if (param->type == KGSL_PROP_INTERRUPT_WAITS) + val = 1; + + return copy_prop(param, &val, sizeof(val)); +} + +static int adreno_prop_uche_gmem_addr(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + return copy_prop(param, &adreno_dev->uche_gmem_base, + sizeof(adreno_dev->uche_gmem_base)); +} + +static int adreno_prop_ucode_version(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_ucode_version ucode = { + .pfp = adreno_dev->fw[ADRENO_FW_PFP].version, + .pm4 = adreno_dev->fw[ADRENO_FW_PM4].version, + }; + + return copy_prop(param, &ucode, sizeof(ucode)); +} + +static int adreno_prop_gaming_bin(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + void *buf; + size_t len; + int ret; + struct nvmem_cell *cell; + + cell = nvmem_cell_get(&device->pdev->dev, "gaming_bin"); + if (IS_ERR(cell)) + return -EINVAL; + + buf = nvmem_cell_read(cell, &len); + nvmem_cell_put(cell); + + if (!IS_ERR(buf)) { + ret = copy_prop(param, buf, len); + kfree(buf); + return ret; + } + + dev_err(device->dev, "failed to read gaming_bin nvmem cell\n"); + return -EINVAL; +} + +static int adreno_prop_u32(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + u32 val = 0; + + if (param->type == KGSL_PROP_HIGHEST_BANK_BIT) { + val = adreno_dev->highest_bank_bit; + } else if (param->type == KGSL_PROP_MIN_ACCESS_LENGTH) + of_property_read_u32(device->pdev->dev.of_node, + "qcom,min-access-length", &val); + else if (param->type == KGSL_PROP_UBWC_MODE) + of_property_read_u32(device->pdev->dev.of_node, + "qcom,ubwc-mode", &val); + else if (param->type == KGSL_PROP_DEVICE_BITNESS) + val = adreno_support_64bit(adreno_dev) ? 48 : 32; + else if (param->type == KGSL_PROP_SPEED_BIN) + val = device->speed_bin; + else if (param->type == KGSL_PROP_VK_DEVICE_ID) + val = adreno_get_vk_device_id(device); + else if (param->type == KGSL_PROP_IS_LPAC_ENABLED) + val = adreno_dev->lpac_enabled ? 1 : 0; + else if (param->type == KGSL_PROP_IS_RAYTRACING_ENABLED) + val = adreno_dev->raytracing_enabled ? 1 : 0; + else if (param->type == KGSL_PROP_IS_FASTBLEND_ENABLED) + val = adreno_dev->fastblend_enabled ? 1 : 0; + else if (param->type == KGSL_PROP_IS_AQE_ENABLED) + val = ADRENO_FEATURE(adreno_dev, ADRENO_AQE) ? 1 : 0; + + return copy_prop(param, &val, sizeof(val)); +} + +static int adreno_prop_uche_trap_base(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + u64 val = 0; + + if (!gpudev->get_uche_trap_base) + return -EINVAL; + + val = gpudev->get_uche_trap_base(); + + return copy_prop(param, &val, sizeof(val)); +} + +static const struct { + int type; + int (*func)(struct kgsl_device *device, + struct kgsl_device_getproperty *param); +} adreno_property_funcs[] = { + { KGSL_PROP_DEVICE_INFO, adreno_prop_device_info }, + { KGSL_PROP_DEVICE_SHADOW, adreno_prop_device_shadow }, + { KGSL_PROP_DEVICE_QDSS_STM, adreno_prop_device_qdss_stm }, + { KGSL_PROP_DEVICE_QTIMER, adreno_prop_device_qtimer }, + { KGSL_PROP_MMU_ENABLE, adreno_prop_s32 }, + { KGSL_PROP_INTERRUPT_WAITS, adreno_prop_s32 }, + { KGSL_PROP_UCHE_GMEM_VADDR, adreno_prop_uche_gmem_addr }, + { KGSL_PROP_UCODE_VERSION, adreno_prop_ucode_version }, + { KGSL_PROP_HIGHEST_BANK_BIT, adreno_prop_u32 }, + { KGSL_PROP_MIN_ACCESS_LENGTH, adreno_prop_u32 }, + { KGSL_PROP_UBWC_MODE, adreno_prop_u32 }, + { KGSL_PROP_DEVICE_BITNESS, adreno_prop_u32 }, + { KGSL_PROP_SPEED_BIN, adreno_prop_u32 }, + { KGSL_PROP_GAMING_BIN, adreno_prop_gaming_bin }, + { KGSL_PROP_GPU_MODEL, adreno_prop_gpu_model}, + { KGSL_PROP_VK_DEVICE_ID, adreno_prop_u32}, + { KGSL_PROP_IS_LPAC_ENABLED, adreno_prop_u32 }, + { KGSL_PROP_IS_RAYTRACING_ENABLED, adreno_prop_u32}, + { KGSL_PROP_IS_FASTBLEND_ENABLED, adreno_prop_u32}, + { KGSL_PROP_UCHE_TRAP_BASE, adreno_prop_uche_trap_base }, + { KGSL_PROP_IS_AQE_ENABLED, adreno_prop_u32 }, +}; + +static int adreno_getproperty(struct kgsl_device *device, + struct kgsl_device_getproperty *param) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(adreno_property_funcs); i++) { + if (param->type == adreno_property_funcs[i].type) + return adreno_property_funcs[i].func(device, param); + } + + return -ENODEV; +} + +static int adreno_query_property_list(struct kgsl_device *device, u32 *list, + u32 count) +{ + int i; + + if (!list) + return ARRAY_SIZE(adreno_property_funcs); + + for (i = 0; i < count && i < ARRAY_SIZE(adreno_property_funcs); i++) + list[i] = adreno_property_funcs[i].type; + + return i; +} + +int adreno_set_constraint(struct kgsl_device *device, + struct kgsl_context *context, + struct kgsl_device_constraint *constraint) +{ + int status = 0; + + switch (constraint->type) { + case KGSL_CONSTRAINT_PWRLEVEL: { + struct kgsl_device_constraint_pwrlevel pwr; + + if (constraint->size != sizeof(pwr)) { + status = -EINVAL; + break; + } + + if (copy_from_user(&pwr, + (void __user *)constraint->data, + sizeof(pwr))) { + status = -EFAULT; + break; + } + if (pwr.level >= KGSL_CONSTRAINT_PWR_MAXLEVELS) { + status = -EINVAL; + break; + } + + context->pwr_constraint.type = + KGSL_CONSTRAINT_PWRLEVEL; + context->pwr_constraint.sub_type = pwr.level; + trace_kgsl_user_pwrlevel_constraint(device, + context->id, + context->pwr_constraint.type, + context->pwr_constraint.sub_type); + } + break; + case KGSL_CONSTRAINT_NONE: + if (context->pwr_constraint.type == KGSL_CONSTRAINT_PWRLEVEL) + trace_kgsl_user_pwrlevel_constraint(device, + context->id, + KGSL_CONSTRAINT_NONE, + context->pwr_constraint.sub_type); + context->pwr_constraint.type = KGSL_CONSTRAINT_NONE; + break; + case KGSL_CONSTRAINT_L3_PWRLEVEL: { + struct kgsl_device_constraint_pwrlevel pwr; + + if (constraint->size != sizeof(pwr)) { + status = -EINVAL; + break; + } + + if (copy_from_user(&pwr, constraint->data, sizeof(pwr))) { + status = -EFAULT; + break; + } + + status = register_l3_voter(device); + if (status) + break; + + if (pwr.level >= KGSL_CONSTRAINT_PWR_MAXLEVELS) + pwr.level = KGSL_CONSTRAINT_PWR_MAXLEVELS - 1; + + context->l3_pwr_constraint.type = KGSL_CONSTRAINT_L3_PWRLEVEL; + context->l3_pwr_constraint.sub_type = pwr.level; + trace_kgsl_user_pwrlevel_constraint(device, context->id, + context->l3_pwr_constraint.type, + context->l3_pwr_constraint.sub_type); + } + break; + case KGSL_CONSTRAINT_L3_NONE: { + unsigned int type = context->l3_pwr_constraint.type; + + if (type == KGSL_CONSTRAINT_L3_PWRLEVEL) + trace_kgsl_user_pwrlevel_constraint(device, context->id, + KGSL_CONSTRAINT_L3_NONE, + context->l3_pwr_constraint.sub_type); + context->l3_pwr_constraint.type = KGSL_CONSTRAINT_L3_NONE; + } + break; + default: + status = -EINVAL; + break; + } + + /* If a new constraint has been set for a context, cancel the old one */ + if ((status == 0) && + (context->id == device->pwrctrl.constraint.owner_id)) { + trace_kgsl_constraint(device, device->pwrctrl.constraint.type, + device->pwrctrl.active_pwrlevel, 0); + device->pwrctrl.constraint.type = KGSL_CONSTRAINT_NONE; + } + + return status; +} + +static int adreno_setproperty(struct kgsl_device_private *dev_priv, + unsigned int type, + void __user *value, + unsigned int sizebytes) +{ + int status = -EINVAL; + struct kgsl_device *device = dev_priv->device; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + switch (type) { + case KGSL_PROP_PWR_CONSTRAINT: + case KGSL_PROP_L3_PWR_CONSTRAINT: { + struct kgsl_device_constraint constraint; + struct kgsl_context *context; + + if (sizebytes != sizeof(constraint)) + break; + + if (copy_from_user(&constraint, value, + sizeof(constraint))) { + status = -EFAULT; + break; + } + + context = kgsl_context_get_owner(dev_priv, + constraint.context_id); + + if (context == NULL) + break; + + status = adreno_set_constraint(device, context, + &constraint); + + kgsl_context_put(context); + } + break; + default: + status = gpudev->setproperty(dev_priv, type, value, sizebytes); + break; + } + + return status; +} + +/* + * adreno_soft_reset - Do a soft reset of the GPU hardware + * @device: KGSL device to soft reset + * + * "soft reset" the GPU hardware - this is a fast path GPU reset + * The GPU hardware is reset but we never pull power so we can skip + * a lot of the standard adreno_stop/adreno_start sequence + */ +static int adreno_soft_reset(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + int ret; + + /* + * Don't allow a soft reset for a304 because the SMMU needs to be hard + * reset + */ + if (adreno_is_a304(adreno_dev)) + return -ENODEV; + + if (gpudev->clear_pending_transactions) { + ret = gpudev->clear_pending_transactions(adreno_dev); + if (ret) + return ret; + } + + kgsl_pwrctrl_change_state(device, KGSL_STATE_AWARE); + adreno_set_active_ctxs_null(adreno_dev); + + adreno_irqctrl(adreno_dev, 0); + + adreno_clear_gpu_fault(adreno_dev); + /* since device is oficially off now clear start bit */ + clear_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + + /* save physical performance counter values before GPU soft reset */ + adreno_perfcounter_save(adreno_dev); + + _soft_reset(adreno_dev); + + /* Clear the busy_data stats - we're starting over from scratch */ + adreno_dev->busy_data.gpu_busy = 0; + adreno_dev->busy_data.bif_ram_cycles = 0; + adreno_dev->busy_data.bif_ram_cycles_read_ch1 = 0; + adreno_dev->busy_data.bif_ram_cycles_write_ch0 = 0; + adreno_dev->busy_data.bif_ram_cycles_write_ch1 = 0; + adreno_dev->busy_data.bif_starved_ram = 0; + adreno_dev->busy_data.bif_starved_ram_ch1 = 0; + + /* Reinitialize the GPU */ + gpudev->start(adreno_dev); + + /* Re-initialize the coresight registers if applicable */ + adreno_coresight_start(adreno_dev); + + /* Enable IRQ */ + adreno_irqctrl(adreno_dev, 1); + + /* stop all ringbuffers to cancel RB events */ + adreno_ringbuffer_stop(adreno_dev); + + /* Start the ringbuffer(s) again */ + ret = gpudev->rb_start(adreno_dev); + if (ret == 0) { + device->reset_counter++; + set_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + } + + /* Restore physical performance counter values after soft reset */ + adreno_perfcounter_restore(adreno_dev); + + if (ret) + dev_err(device->dev, "Device soft reset failed: %d\n", ret); + + return ret; +} + +bool adreno_isidle(struct adreno_device *adreno_dev) +{ + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + struct adreno_ringbuffer *rb; + int i; + + if (!kgsl_state_is_awake(KGSL_DEVICE(adreno_dev))) + return true; + + /* + * wptr is updated when we add commands to ringbuffer, add a barrier + * to make sure updated wptr is compared to rptr + */ + smp_mb(); + + /* + * ringbuffer is truly idle when all ringbuffers read and write + * pointers are equal + */ + + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + if (!adreno_rb_empty(rb)) + return false; + } + + return gpudev->hw_isidle(adreno_dev); +} + +/** + * adreno_spin_idle() - Spin wait for the GPU to idle + * @adreno_dev: Pointer to an adreno device + * @timeout: milliseconds to wait before returning error + * + * Spin the CPU waiting for the RBBM status to return idle + */ +int adreno_spin_idle(struct adreno_device *adreno_dev, unsigned int timeout) +{ + unsigned long wait = jiffies + msecs_to_jiffies(timeout); + + do { + /* + * If we fault, stop waiting and return an error. The dispatcher + * will clean up the fault from the work queue, but we need to + * make sure we don't block it by waiting for an idle that + * will never come. + */ + + if (adreno_gpu_fault(adreno_dev) != 0) + return -EDEADLK; + + if (adreno_isidle(adreno_dev)) + return 0; + + } while (time_before(jiffies, wait)); + + /* + * Under rare conditions, preemption can cause the while loop to exit + * without checking if the gpu is idle. check one last time before we + * return failure. + */ + if (adreno_gpu_fault(adreno_dev) != 0) + return -EDEADLK; + + if (adreno_isidle(adreno_dev)) + return 0; + + return -ETIMEDOUT; +} + +/** + * adreno_idle() - wait for the GPU hardware to go idle + * @device: Pointer to the KGSL device structure for the GPU + * + * Wait up to ADRENO_IDLE_TIMEOUT milliseconds for the GPU hardware to go quiet. + * Caller must hold the device mutex, and must not hold the dispatcher mutex. + */ + +int adreno_idle(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + int ret; + + /* + * Make sure the device mutex is held so the dispatcher can't send any + * more commands to the hardware + */ + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return -EDEADLK; + + /* Check if we are already idle before idling dispatcher */ + if (adreno_isidle(adreno_dev)) + return 0; + /* + * Wait for dispatcher to finish completing commands + * already submitted + */ + ret = adreno_dispatcher_idle(adreno_dev); + if (ret) + return ret; + + return adreno_spin_idle(adreno_dev, ADRENO_IDLE_TIMEOUT); +} + +static int adreno_drain_and_idle(struct kgsl_device *device) +{ + int ret; + + reinit_completion(&device->halt_gate); + + ret = kgsl_active_count_wait(device, 0, HZ); + if (ret) + return ret; + + return adreno_idle(device); +} + +/* Caller must hold the device mutex. */ +int adreno_suspend_context(struct kgsl_device *device) +{ + /* process any profiling results that are available */ + adreno_profile_process_results(ADRENO_DEVICE(device)); + + /* Wait for the device to go idle */ + return adreno_idle(device); +} + +void adreno_cx_misc_regread(struct adreno_device *adreno_dev, + unsigned int offsetwords, unsigned int *value) +{ + unsigned int cx_misc_offset; + + WARN_ONCE(!adreno_dev->cx_misc_virt, + "cx_misc region is not defined in device tree"); + + cx_misc_offset = (offsetwords << 2); + if (!adreno_dev->cx_misc_virt || + (cx_misc_offset >= adreno_dev->cx_misc_len)) + return; + + *value = __raw_readl(adreno_dev->cx_misc_virt + cx_misc_offset); + + /* + * ensure this read finishes before the next one. + * i.e. act like normal readl() + */ + rmb(); +} + +void adreno_isense_regread(struct adreno_device *adreno_dev, + unsigned int offsetwords, unsigned int *value) +{ + unsigned int isense_offset; + + isense_offset = (offsetwords << 2); + if (!adreno_dev->isense_virt || + (isense_offset >= adreno_dev->isense_len)) + return; + + *value = __raw_readl(adreno_dev->isense_virt + isense_offset); + + /* + * ensure this read finishes before the next one. + * i.e. act like normal readl() + */ + rmb(); +} + +bool adreno_gx_is_on(struct adreno_device *adreno_dev) +{ + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + return gpudev->gx_is_on(adreno_dev); +} + +void adreno_cx_misc_regwrite(struct adreno_device *adreno_dev, + unsigned int offsetwords, unsigned int value) +{ + unsigned int cx_misc_offset; + + WARN_ONCE(!adreno_dev->cx_misc_virt, + "cx_misc region is not defined in device tree"); + + cx_misc_offset = (offsetwords << 2); + if (!adreno_dev->cx_misc_virt || + (cx_misc_offset >= adreno_dev->cx_misc_len)) + return; + + /* + * ensure previous writes post before this one, + * i.e. act like normal writel() + */ + wmb(); + __raw_writel(value, adreno_dev->cx_misc_virt + cx_misc_offset); +} + +void adreno_cx_misc_regrmw(struct adreno_device *adreno_dev, + unsigned int offsetwords, + unsigned int mask, unsigned int bits) +{ + unsigned int val = 0; + + adreno_cx_misc_regread(adreno_dev, offsetwords, &val); + val &= ~mask; + adreno_cx_misc_regwrite(adreno_dev, offsetwords, val | bits); +} + +void adreno_profile_submit_time(struct adreno_submit_time *time) +{ + struct kgsl_drawobj *drawobj; + struct kgsl_drawobj_cmd *cmdobj; + struct kgsl_mem_entry *entry; + struct kgsl_drawobj_profiling_buffer *profile_buffer; + + if (!time) + return; + + drawobj = time->drawobj; + if (drawobj == NULL) + return; + + cmdobj = CMDOBJ(drawobj); + entry = cmdobj->profiling_buf_entry; + if (!entry) + return; + + profile_buffer = kgsl_gpuaddr_to_vaddr(&entry->memdesc, + cmdobj->profiling_buffer_gpuaddr); + + if (profile_buffer == NULL) + return; + + /* Return kernel clock time to the client if requested */ + if (drawobj->flags & KGSL_DRAWOBJ_PROFILING_KTIME) { + u64 secs = time->ktime; + + profile_buffer->wall_clock_ns = + do_div(secs, NSEC_PER_SEC); + profile_buffer->wall_clock_s = secs; + } else { + profile_buffer->wall_clock_s = time->utime.tv_sec; + profile_buffer->wall_clock_ns = time->utime.tv_nsec; + } + + profile_buffer->gpu_ticks_queued = time->ticks; + + kgsl_memdesc_unmap(&entry->memdesc); +} + +/** + * adreno_waittimestamp - sleep while waiting for the specified timestamp + * @device - pointer to a KGSL device structure + * @context - pointer to the active kgsl context + * @timestamp - GPU timestamp to wait for + * @msecs - amount of time to wait (in milliseconds) + * + * Wait up to 'msecs' milliseconds for the specified timestamp to expire. + */ +static int adreno_waittimestamp(struct kgsl_device *device, + struct kgsl_context *context, + unsigned int timestamp, + unsigned int msecs) +{ + int ret; + + if (context == NULL) { + /* If they are doing then complain once */ + dev_WARN_ONCE(device->dev, 1, + "IOCTL_KGSL_DEVICE_WAITTIMESTAMP is deprecated\n"); + return -ENOTTY; + } + + /* Return -ENOENT if the context has been detached */ + if (kgsl_context_detached(context)) + return -ENOENT; + + ret = adreno_drawctxt_wait(ADRENO_DEVICE(device), context, + timestamp, msecs); + + /* If the context got invalidated then return a specific error */ + if (kgsl_context_invalid(context)) + ret = -EDEADLK; + + /* + * Return -EPROTO if the device has faulted since the last time we + * checked. Userspace uses this as a marker for performing post + * fault activities + */ + + if (!ret && test_and_clear_bit(ADRENO_CONTEXT_FAULT, &context->priv)) + ret = -EPROTO; + + return ret; +} + +/** + * __adreno_readtimestamp() - Reads the timestamp from memstore memory + * @adreno_dev: Pointer to an adreno device + * @index: Index into the memstore memory + * @type: Type of timestamp to read + * @timestamp: The out parameter where the timestamp is read + */ +static int __adreno_readtimestamp(struct adreno_device *adreno_dev, int index, + int type, unsigned int *timestamp) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int status = 0; + + switch (type) { + case KGSL_TIMESTAMP_CONSUMED: + kgsl_sharedmem_readl(device->memstore, timestamp, + KGSL_MEMSTORE_OFFSET(index, soptimestamp)); + break; + case KGSL_TIMESTAMP_RETIRED: + kgsl_sharedmem_readl(device->memstore, timestamp, + KGSL_MEMSTORE_OFFSET(index, eoptimestamp)); + break; + default: + status = -EINVAL; + *timestamp = 0; + break; + } + return status; +} + +/** + * adreno_rb_readtimestamp(): Return the value of given type of timestamp + * for a RB + * @adreno_dev: adreno device whose timestamp values are being queried + * @priv: The object being queried for a timestamp (expected to be a rb pointer) + * @type: The type of timestamp (one of 3) to be read + * @timestamp: Pointer to where the read timestamp is to be written to + * + * CONSUMED and RETIRED type timestamps are sorted by id and are constantly + * updated by the GPU through shared memstore memory. QUEUED type timestamps + * are read directly from context struct. + + * The function returns 0 on success and timestamp value at the *timestamp + * address and returns -EINVAL on any read error/invalid type and timestamp = 0. + */ +int adreno_rb_readtimestamp(struct adreno_device *adreno_dev, + void *priv, enum kgsl_timestamp_type type, + unsigned int *timestamp) +{ + int status = 0; + struct adreno_ringbuffer *rb = priv; + + if (type == KGSL_TIMESTAMP_QUEUED) + *timestamp = rb->timestamp; + else + status = __adreno_readtimestamp(adreno_dev, + rb->id + KGSL_MEMSTORE_MAX, + type, timestamp); + + return status; +} + +/** + * adreno_readtimestamp(): Return the value of given type of timestamp + * @device: GPU device whose timestamp values are being queried + * @priv: The object being queried for a timestamp (expected to be a context) + * @type: The type of timestamp (one of 3) to be read + * @timestamp: Pointer to where the read timestamp is to be written to + * + * CONSUMED and RETIRED type timestamps are sorted by id and are constantly + * updated by the GPU through shared memstore memory. QUEUED type timestamps + * are read directly from context struct. + + * The function returns 0 on success and timestamp value at the *timestamp + * address and returns -EINVAL on any read error/invalid type and timestamp = 0. + */ +static int adreno_readtimestamp(struct kgsl_device *device, + void *priv, enum kgsl_timestamp_type type, + unsigned int *timestamp) +{ + int status = 0; + struct kgsl_context *context = priv; + + if (type == KGSL_TIMESTAMP_QUEUED) { + struct adreno_context *ctxt = ADRENO_CONTEXT(context); + + *timestamp = ctxt->timestamp; + } else + status = __adreno_readtimestamp(ADRENO_DEVICE(device), + context->id, type, timestamp); + + return status; +} + +/** + * adreno_device_private_create(): Allocate an adreno_device_private structure + */ +static struct kgsl_device_private *adreno_device_private_create(void) +{ + struct adreno_device_private *adreno_priv = + kzalloc(sizeof(*adreno_priv), GFP_KERNEL); + + if (adreno_priv) { + INIT_LIST_HEAD(&adreno_priv->perfcounter_list); + return &adreno_priv->dev_priv; + } + return NULL; +} + +/** + * adreno_device_private_destroy(): Destroy an adreno_device_private structure + * and release the perfcounters held by the kgsl fd. + * @dev_priv: The kgsl device private structure + */ +static void adreno_device_private_destroy(struct kgsl_device_private *dev_priv) +{ + struct kgsl_device *device = dev_priv->device; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_device_private *adreno_priv = + container_of(dev_priv, struct adreno_device_private, + dev_priv); + struct adreno_perfcounter_list_node *p, *tmp; + + mutex_lock(&device->mutex); + list_for_each_entry_safe(p, tmp, &adreno_priv->perfcounter_list, node) { + adreno_perfcounter_put(adreno_dev, p->groupid, + p->countable, PERFCOUNTER_FLAG_NONE); + list_del(&p->node); + kfree(p); + } + mutex_unlock(&device->mutex); + + kfree(adreno_priv); +} + +/** + * adreno_power_stats() - Reads the counters needed for freq decisions + * @device: Pointer to device whose counters are read + * @stats: Pointer to stats set that needs updating + * Power: The caller is expected to be in a clock enabled state as this + * function does reg reads + */ +static void adreno_power_stats(struct kgsl_device *device, + struct kgsl_power_stats *stats) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + return gpudev->power_stats(adreno_dev, stats); +} + +static int adreno_regulator_enable(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (gpudev->regulator_enable) + return gpudev->regulator_enable(adreno_dev); + + return 0; +} + +static bool adreno_is_hw_collapsible(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (!gpudev->is_hw_collapsible(adreno_dev)) + return false; + + if (gpudev->clear_pending_transactions(adreno_dev)) + return false; + + adreno_dispatcher_stop_fault_timer(device); + + return true; +} + +static void adreno_regulator_disable(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (gpudev->regulator_disable) + gpudev->regulator_disable(adreno_dev); +} + +static void adreno_pwrlevel_change_settings(struct kgsl_device *device, + unsigned int prelevel, unsigned int postlevel, bool post) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (gpudev->pwrlevel_change_settings) + gpudev->pwrlevel_change_settings(adreno_dev, prelevel, + postlevel, post); +} + +static bool adreno_is_hwcg_on(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + return adreno_dev->hwcg_enabled; +} + +static int adreno_queue_cmds(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, struct kgsl_drawobj *drawobj[], + u32 count, u32 *timestamp) +{ + struct kgsl_device *device = dev_priv->device; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + if (WARN_ON(!adreno_dev->dispatch_ops || !adreno_dev->dispatch_ops->queue_cmds)) + return -ENODEV; + + return adreno_dev->dispatch_ops->queue_cmds(dev_priv, context, drawobj, + count, timestamp); +} + +static inline bool _verify_ib(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, struct kgsl_memobj_node *ib) +{ + struct kgsl_device *device = dev_priv->device; + struct kgsl_process_private *private = dev_priv->process_priv; + + /* The maximum allowable size for an IB in the CP is 0xFFFFF dwords */ + if (ib->size == 0 || ((ib->size >> 2) > 0xFFFFF)) { + pr_context(device, context, "ctxt %u invalid ib size %lld\n", + context->id, ib->size); + return false; + } + + /* Make sure that the address is in range and dword aligned */ + if (!kgsl_mmu_gpuaddr_in_range(private->pagetable, ib->gpuaddr, + ib->size) || !IS_ALIGNED(ib->gpuaddr, 4)) { + pr_context(device, context, "ctxt %u invalid ib gpuaddr %llX\n", + context->id, ib->gpuaddr); + return false; + } + + return true; +} + +int adreno_verify_cmdobj(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, struct kgsl_drawobj *drawobj[], + uint32_t count) +{ + struct kgsl_device *device = dev_priv->device; + struct kgsl_memobj_node *ib; + unsigned int i; + + for (i = 0; i < count; i++) { + /* Verify the IBs before they get queued */ + if (drawobj[i]->type == CMDOBJ_TYPE) { + struct kgsl_drawobj_cmd *cmdobj = CMDOBJ(drawobj[i]); + + list_for_each_entry(ib, &cmdobj->cmdlist, node) + if (!_verify_ib(dev_priv, + &ADRENO_CONTEXT(context)->base, ib)) + return -EINVAL; + + /* + * Clear the wake on touch bit to indicate an IB has + * been submitted since the last time we set it. + * But only clear it when we have rendering commands. + */ + device->pwrctrl.wake_on_touch = false; + } + + /* A3XX does not have support for drawobj profiling */ + if (adreno_is_a3xx(ADRENO_DEVICE(device)) && + (drawobj[i]->flags & KGSL_DRAWOBJ_PROFILING)) + return -EOPNOTSUPP; + } + + return 0; +} + +static int adreno_queue_recurring_cmd(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, struct kgsl_drawobj *drawobj) +{ + struct kgsl_device *device = dev_priv->device; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_hwsched *hwsched = &adreno_dev->hwsched; + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + struct kgsl_drawobj_cmd *cmdobj = CMDOBJ(drawobj); + int ret; + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_LSR)) + return -EOPNOTSUPP; + + if (!gpudev->send_recurring_cmdobj) + return -ENODEV; + + ret = adreno_verify_cmdobj(dev_priv, context, &drawobj, 1); + if (ret) + return ret; + + mutex_lock(&device->mutex); + + /* Only one recurring command allowed */ + if (hwsched->recurring_cmdobj) { + mutex_unlock(&device->mutex); + return -EINVAL; + } + + ret = kgsl_check_context_state(context); + if (ret) { + mutex_unlock(&device->mutex); + return ret; + } + + set_bit(CMDOBJ_RECURRING_START, &cmdobj->priv); + + ret = gpudev->send_recurring_cmdobj(adreno_dev, cmdobj); + mutex_unlock(&device->mutex); + + if (!ret) + srcu_notifier_call_chain(&device->nh, GPU_GMU_READY, NULL); + + return ret; +} + +static int adreno_dequeue_recurring_cmd(struct kgsl_device *device, + struct kgsl_context *context) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_hwsched *hwsched = &adreno_dev->hwsched; + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + struct kgsl_drawobj *recurring_drawobj; + int ret; + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_LSR)) + return -EOPNOTSUPP; + + if (!gpudev->send_recurring_cmdobj) + return -ENODEV; + + mutex_lock(&device->mutex); + + /* We can safely return here as recurring wokload is already untracked */ + if (hwsched->recurring_cmdobj == NULL) { + mutex_unlock(&device->mutex); + return -EINVAL; + } + + recurring_drawobj = DRAWOBJ(hwsched->recurring_cmdobj); + + /* Check if the recurring command is for same context or not*/ + if (recurring_drawobj->context != context) { + mutex_unlock(&device->mutex); + return -EINVAL; + } + + ret = kgsl_check_context_state(context); + if (ret) { + mutex_unlock(&device->mutex); + return ret; + } + + clear_bit(CMDOBJ_RECURRING_START, &hwsched->recurring_cmdobj->priv); + set_bit(CMDOBJ_RECURRING_STOP, &hwsched->recurring_cmdobj->priv); + + ret = gpudev->send_recurring_cmdobj(adreno_dev, hwsched->recurring_cmdobj); + + mutex_unlock(&device->mutex); + + if (!ret) + srcu_notifier_call_chain(&device->nh, GPU_GMU_STOP, NULL); + + return ret; +} + +static void adreno_set_isdb_breakpoint_registers(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (gpudev->set_isdb_breakpoint_registers) + gpudev->set_isdb_breakpoint_registers(adreno_dev); +} + +static void adreno_drawctxt_sched(struct kgsl_device *device, + struct kgsl_context *context) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + if (WARN_ON(!adreno_dev->dispatch_ops || !adreno_dev->dispatch_ops->queue_context)) + return; + + adreno_dev->dispatch_ops->queue_context(adreno_dev, + ADRENO_CONTEXT(context)); +} + +void adreno_mark_for_coldboot(struct adreno_device *adreno_dev) +{ + if (!adreno_dev->warmboot_enabled) + return; + + set_bit(ADRENO_DEVICE_FORCE_COLDBOOT, &adreno_dev->priv); +} + +bool adreno_smmu_is_stalled(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_mmu *mmu = &device->mmu; + u32 fault, val; + + /* + * RBBM_STATUS3:SMMU_STALLED_ON_FAULT (BIT 24) to tells if GPU + * encoutnered a pagefault. Gen8 page fault status checked from + * the software condition as RBBM_STATS3 is not available. + */ + if (ADRENO_GPUREV(adreno_dev) < 0x080000) { + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_STATUS3, &val); + return (val & BIT(24)); + } + + if (WARN_ON(!adreno_dev->dispatch_ops || !adreno_dev->dispatch_ops->get_fault)) + return false; + + fault = adreno_dev->dispatch_ops->get_fault(adreno_dev); + + return ((fault & ADRENO_IOMMU_PAGE_FAULT) && + test_bit(KGSL_FT_PAGEFAULT_GPUHALT_ENABLE, &mmu->pfpolicy)) ? true : false; +} + +int adreno_power_cycle(struct adreno_device *adreno_dev, + void (*callback)(struct adreno_device *adreno_dev, void *priv), + void *priv) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + int ret; + + mutex_lock(&device->mutex); + ret = ops->pm_suspend(adreno_dev); + + if (!ret) { + callback(adreno_dev, priv); + adreno_mark_for_coldboot(adreno_dev); + ops->pm_resume(adreno_dev); + } + + mutex_unlock(&device->mutex); + + return ret; +} + +struct cycle_data { + void *ptr; + void *val; +}; + +static void cycle_set_bool(struct adreno_device *adreno_dev, void *priv) +{ + struct cycle_data *data = priv; + + *((bool *) data->ptr) = *((bool *) data->val); +} + +int adreno_power_cycle_bool(struct adreno_device *adreno_dev, + bool *flag, bool val) +{ + struct cycle_data data = { .ptr = flag, .val = &val }; + + return adreno_power_cycle(adreno_dev, cycle_set_bool, &data); +} + +static void cycle_set_u32(struct adreno_device *adreno_dev, void *priv) +{ + struct cycle_data *data = priv; + + *((u32 *) data->ptr) = *((u32 *) data->val); +} + +int adreno_power_cycle_u32(struct adreno_device *adreno_dev, + u32 *flag, u32 val) +{ + struct cycle_data data = { .ptr = flag, .val = &val }; + + return adreno_power_cycle(adreno_dev, cycle_set_u32, &data); +} + +static int adreno_gpu_clock_set(struct kgsl_device *device, u32 pwrlevel) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct kgsl_pwrlevel *pl = &pwr->pwrlevels[pwrlevel]; + int ret; + + if (ops->gpu_clock_set) + return ops->gpu_clock_set(adreno_dev, pwrlevel); + + ret = clk_set_rate(pwr->grp_clks[0], pl->gpu_freq); + if (ret) + dev_err(device->dev, "GPU clk freq set failure: %d\n", ret); + + return ret; +} + +static int adreno_interconnect_bus_set(struct adreno_device *adreno_dev, + int level, u32 ab) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + + if ((level == pwr->cur_buslevel) && (ab == pwr->cur_ab)) + return 0; + + kgsl_icc_set_tag(pwr, level); + pwr->cur_buslevel = level; + pwr->cur_ab = ab; + + icc_set_bw(pwr->icc_path, MBps_to_icc(ab), + kBps_to_icc(pwr->ddr_table[level])); + + trace_kgsl_buslevel(device, pwr->active_pwrlevel, level, ab); + + return 0; +} + +static int adreno_gpu_bus_set(struct kgsl_device *device, int level, u32 ab) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + if (ops->gpu_bus_set) + return ops->gpu_bus_set(adreno_dev, level, ab); + + return adreno_interconnect_bus_set(adreno_dev, level, ab); +} + +static void adreno_deassert_gbif_halt(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (gpudev->deassert_gbif_halt) + gpudev->deassert_gbif_halt(adreno_dev); +} + +static void adreno_create_hw_fence(struct kgsl_device *device, struct kgsl_sync_fence *kfence) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + if (WARN_ON(!adreno_dev->dispatch_ops)) + return; + + if (adreno_dev->dispatch_ops->create_hw_fence) + adreno_dev->dispatch_ops->create_hw_fence(adreno_dev, kfence); +} + +u64 adreno_read_cx_timer(struct adreno_device *adreno_dev) +{ + /* Check if the CX timer is initialized */ + if (!test_bit(ADRENO_DEVICE_CX_TIMER_INITIALIZED, &adreno_dev->priv)) + return 0; + + /* Since the GPU CX and CPU timers are synchronized return the CPU timer */ + return arch_timer_read_counter(); +} + +static const struct kgsl_functable adreno_functable = { + /* Mandatory functions */ + .suspend_context = adreno_suspend_context, + .first_open = adreno_first_open, + .start = adreno_start, + .stop = adreno_stop, + .last_close = adreno_last_close, + .getproperty = adreno_getproperty, + .getproperty_compat = adreno_getproperty_compat, + .waittimestamp = adreno_waittimestamp, + .readtimestamp = adreno_readtimestamp, + .queue_cmds = adreno_queue_cmds, + .ioctl = adreno_ioctl, + .compat_ioctl = adreno_compat_ioctl, + .power_stats = adreno_power_stats, + .snapshot = adreno_snapshot, + .drain_and_idle = adreno_drain_and_idle, + .device_private_create = adreno_device_private_create, + .device_private_destroy = adreno_device_private_destroy, + /* Optional functions */ + .drawctxt_create = adreno_drawctxt_create, + .drawctxt_detach = adreno_drawctxt_detach, + .drawctxt_destroy = adreno_drawctxt_destroy, + .drawctxt_dump = adreno_drawctxt_dump, + .setproperty = adreno_setproperty, + .setproperty_compat = adreno_setproperty_compat, + .drawctxt_sched = adreno_drawctxt_sched, + .resume = adreno_dispatcher_start, + .regulator_enable = adreno_regulator_enable, + .is_hw_collapsible = adreno_is_hw_collapsible, + .regulator_disable = adreno_regulator_disable, + .pwrlevel_change_settings = adreno_pwrlevel_change_settings, + .query_property_list = adreno_query_property_list, + .is_hwcg_on = adreno_is_hwcg_on, + .gpu_clock_set = adreno_gpu_clock_set, + .gpu_bus_set = adreno_gpu_bus_set, + .deassert_gbif_halt = adreno_deassert_gbif_halt, + .queue_recurring_cmd = adreno_queue_recurring_cmd, + .dequeue_recurring_cmd = adreno_dequeue_recurring_cmd, + .set_isdb_breakpoint_registers = adreno_set_isdb_breakpoint_registers, + .create_hw_fence = adreno_create_hw_fence, +}; + +static const struct component_master_ops adreno_ops = { + .bind = adreno_bind, + .unbind = adreno_unbind, +}; + +const struct adreno_power_ops adreno_power_operations = { + .first_open = adreno_open, + .last_close = adreno_close, + .active_count_get = adreno_pwrctrl_active_count_get, + .active_count_put = adreno_pwrctrl_active_count_put, + .pm_suspend = adreno_suspend, + .pm_resume = adreno_resume, + .touch_wakeup = adreno_touch_wakeup, +}; + +static int _compare_of(struct device *dev, void *data) +{ + return (dev->of_node == data); +} + +static void _release_of(struct device *dev, void *data) +{ + of_node_put(data); +} + +static void adreno_add_components(struct device *dev, + struct component_match **match) +{ + struct device_node *node; + + /* + * Add kgsl-smmu, context banks and gmu as components, if supported. + * Master bind (adreno_bind) will be called only once all added + * components are available. + */ + for_each_matching_node(node, adreno_component_match) { + if (!of_device_is_available(node)) + continue; + + component_match_add_release(dev, match, _release_of, _compare_of, node); + } +} + +static int adreno_probe(struct platform_device *pdev) +{ + struct component_match *match = NULL; + + adreno_add_components(&pdev->dev, &match); + + if (!match) + return -ENODEV; + + return component_master_add_with_match(&pdev->dev, + &adreno_ops, match); +} + +static int adreno_remove(struct platform_device *pdev) +{ + component_master_del(&pdev->dev, &adreno_ops); + + return 0; +} + +#if IS_ENABLED(CONFIG_QCOM_KGSL_HIBERNATION) +#if IS_ENABLED(CONFIG_QCOM_SECURE_BUFFER) +/* + * Issue hyp_assign call to assign non-used internal/userspace secure + * buffers to kernel. + */ +static int adreno_secure_pt_hibernate(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_process_private *process; + struct kgsl_mem_entry *entry; + struct kgsl_global_memdesc *md; + struct kgsl_memdesc *memdesc; + int ret, id; + + read_lock(&kgsl_driver.proclist_lock); + list_for_each_entry(process, &kgsl_driver.process_list, list) { + idr_for_each_entry(&process->mem_idr, entry, id) { + memdesc = &entry->memdesc; + if (!kgsl_memdesc_is_secured(memdesc) || + (memdesc->flags & KGSL_MEMFLAGS_USERMEM_ION) || + (memdesc->priv & KGSL_MEMDESC_HYPASSIGNED_HLOS)) + continue; + + read_unlock(&kgsl_driver.proclist_lock); + + if (kgsl_unlock_sgt(memdesc->sgt)) + dev_err(device->dev, "kgsl_unlock_sgt failed\n"); + + memdesc->priv |= KGSL_MEMDESC_HYPASSIGNED_HLOS; + + read_lock(&kgsl_driver.proclist_lock); + } + } + read_unlock(&kgsl_driver.proclist_lock); + + list_for_each_entry(md, &device->globals, node) { + memdesc = &md->memdesc; + if (kgsl_memdesc_is_secured(memdesc) && + !(memdesc->priv & KGSL_MEMDESC_HYPASSIGNED_HLOS)) { + ret = kgsl_unlock_sgt(memdesc->sgt); + if (ret) { + dev_err(device->dev, "kgsl_unlock_sgt failed ret %d\n", ret); + goto fail; + } + memdesc->priv |= KGSL_MEMDESC_HYPASSIGNED_HLOS; + } + } + + return 0; + +fail: + list_for_each_entry(md, &device->globals, node) { + memdesc = &md->memdesc; + if (kgsl_memdesc_is_secured(memdesc) && + (memdesc->priv & KGSL_MEMDESC_HYPASSIGNED_HLOS)) { + kgsl_lock_sgt(memdesc->sgt, memdesc->size); + memdesc->priv &= ~KGSL_MEMDESC_HYPASSIGNED_HLOS; + } + } + + return -EBUSY; +} + +static int adreno_secure_pt_restore(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_process_private *process; + struct kgsl_mem_entry *entry; + struct kgsl_memdesc *memdesc; + struct kgsl_global_memdesc *md; + int ret, id; + + list_for_each_entry(md, &device->globals, node) { + memdesc = &md->memdesc; + if (kgsl_memdesc_is_secured(memdesc) && + (memdesc->priv & KGSL_MEMDESC_HYPASSIGNED_HLOS)) { + ret = kgsl_lock_sgt(memdesc->sgt, memdesc->size); + if (ret) { + dev_err(device->dev, "kgsl_lock_sgt failed ret %d\n", ret); + return ret; + } + memdesc->priv &= ~KGSL_MEMDESC_HYPASSIGNED_HLOS; + } + } + + read_lock(&kgsl_driver.proclist_lock); + list_for_each_entry(process, &kgsl_driver.process_list, list) { + idr_for_each_entry(&process->mem_idr, entry, id) { + memdesc = &entry->memdesc; + if (!kgsl_memdesc_is_secured(memdesc) || + (memdesc->flags & KGSL_MEMFLAGS_USERMEM_ION) || + !(memdesc->priv & KGSL_MEMDESC_HYPASSIGNED_HLOS)) + continue; + + read_unlock(&kgsl_driver.proclist_lock); + + ret = kgsl_lock_sgt(memdesc->sgt, memdesc->size); + if (ret) { + dev_err(device->dev, "kgsl_lock_sgt failed ret %d\n", ret); + return ret; + } + memdesc->priv &= ~KGSL_MEMDESC_HYPASSIGNED_HLOS; + + read_lock(&kgsl_driver.proclist_lock); + } + } + read_unlock(&kgsl_driver.proclist_lock); + + return 0; +} +#else +static int adreno_secure_pt_hibernate(struct adreno_device *adreno_dev) +{ + return 0; +} + +static int adreno_secure_pt_restore(struct adreno_device *adreno_dev) +{ + return 0; +} +#endif /* IS_ENABLED(CONFIG_QCOM_SECURE_BUFFER) */ + +static int adreno_hibernation_suspend(struct device *dev) +{ + struct kgsl_device *device = dev_get_drvdata(dev); + struct adreno_device *adreno_dev; + const struct adreno_power_ops *ops; + int status; + + if (!device) + return 0; + + adreno_dev = ADRENO_DEVICE(device); + ops = ADRENO_POWER_OPS(adreno_dev); + + mutex_lock(&device->mutex); + + status = ops->pm_suspend(adreno_dev); + if (status) + goto err; + + /* + * When the device enters in hibernation state, the CX will be collapsed causing + * the GPU CX timer to pause. Clear the ADRENO_DEVICE_CX_TIMER_INITIALIZED flag + * to ensure that the CX timer is reseeded during resume. + */ + clear_bit(ADRENO_DEVICE_CX_TIMER_INITIALIZED, &adreno_dev->priv); + + /* + * Unload zap shader during device hibernation and reload it + * during resume as there is possibility that TZ driver + * is not aware of the hibernation. + */ + adreno_zap_shader_unload(adreno_dev); + status = adreno_secure_pt_hibernate(adreno_dev); + +err: + mutex_unlock(&device->mutex); + return status; +} + +static int adreno_hibernation_resume(struct device *dev) +{ + struct kgsl_device *device = dev_get_drvdata(dev); + struct kgsl_iommu *iommu; + struct kgsl_pwrscale *pwrscale; + struct adreno_device *adreno_dev; + const struct adreno_power_ops *ops; + int ret; + + if (!device) + return 0; + + iommu = &device->mmu.iommu; + pwrscale = &device->pwrscale; + adreno_dev = ADRENO_DEVICE(device); + ops = ADRENO_POWER_OPS(adreno_dev); + + mutex_lock(&device->mutex); + + ret = adreno_secure_pt_restore(adreno_dev); + if (ret) + goto err; + + ret = kgsl_set_smmu_aperture(device, &iommu->user_context); + if (ret) + goto err; + + ret = kgsl_set_smmu_lpac_aperture(device, &iommu->lpac_context); + if (ret < 0) + goto err; + + gmu_core_dev_force_first_boot(device); + + msm_adreno_tz_reinit(pwrscale->devfreqptr); + + ops->pm_resume(adreno_dev); + +err: + mutex_unlock(&device->mutex); + return ret; +} + +static const struct dev_pm_ops adreno_pm_ops = { + .suspend = adreno_pm_suspend, + .resume = adreno_pm_resume, + .freeze = adreno_hibernation_suspend, + .thaw = adreno_hibernation_resume, + .poweroff = adreno_hibernation_suspend, + .restore = adreno_hibernation_resume, +}; +#else +static const struct dev_pm_ops adreno_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(adreno_pm_suspend, adreno_pm_resume) +}; +#endif /* IS_ENABLED(CONFIG_QCOM_KGSL_HIBERNATION) */ + +static struct platform_driver adreno_platform_driver = { + .probe = adreno_probe, + .remove = adreno_remove, + .driver = { + .name = "kgsl-3d", + .pm = &adreno_pm_ops, + .of_match_table = of_match_ptr(adreno_match_table), + } +}; + +static int __init kgsl_3d_init(void) +{ + int ret; + + ret = kgsl_core_init(); + if (ret) + return ret; + + ret = kgsl_mmu_init(); + if (ret) { + kgsl_core_exit(); + return ret; + } + + gmu_core_register(); + ret = platform_driver_register(&adreno_platform_driver); + if (ret) { + gmu_core_unregister(); + kgsl_mmu_exit(); + kgsl_core_exit(); + } + + return ret; +} + +static void __exit kgsl_3d_exit(void) +{ + platform_driver_unregister(&adreno_platform_driver); + gmu_core_unregister(); + kgsl_mmu_exit(); + kgsl_core_exit(); +} + +module_param_named(preempt_enable, adreno_preemption_enable, bool, 0600); +MODULE_PARM_DESC(preempt_enable, "Enable GPU HW Preemption"); + +module_init(kgsl_3d_init); +module_exit(kgsl_3d_exit); + +MODULE_DESCRIPTION("3D Graphics driver"); +MODULE_LICENSE("GPL v2"); +MODULE_SOFTDEP("pre: arm_smmu nvmem_qfprom socinfo"); +#if (KERNEL_VERSION(5, 18, 0) <= LINUX_VERSION_CODE) +MODULE_IMPORT_NS(DMA_BUF); +#endif diff --git a/qcom/opensource/graphics-kernel/adreno.h b/qcom/opensource/graphics-kernel/adreno.h new file mode 100644 index 0000000000..2abdeaa07e --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno.h @@ -0,0 +1,2067 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2008-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ +#ifndef __ADRENO_H +#define __ADRENO_H + +#include +#include +#include +#include "adreno_coresight.h" +#include "adreno_dispatch.h" +#include "adreno_drawctxt.h" +#include "adreno_hfi.h" +#include "adreno_hwsched.h" +#include "adreno_perfcounter.h" +#include "adreno_profile.h" +#include "adreno_ringbuffer.h" +#include "kgsl_sharedmem.h" + +/* Used to point CP to the SMMU record during preemption */ +#define SET_PSEUDO_SMMU_INFO 0 +/* Used to inform CP where to save preemption data at the time of switch out */ +#define SET_PSEUDO_PRIV_NON_SECURE_SAVE_ADDR 1 +/* Used to inform CP where to save secure preemption data at the time of switch out */ +#define SET_PSEUDO_PRIV_SECURE_SAVE_ADDR 2 +/* Used to inform CP where to save per context non-secure data at the time of switch out */ +#define SET_PSEUDO_NON_PRIV_SAVE_ADDR 3 +/* Used to inform CP where to save preemption counter data at the time of switch out */ +#define SET_PSEUDO_COUNTER 4 + +/* Index to preemption scratch buffer to store current QOS value */ +#define QOS_VALUE_IDX KGSL_PRIORITY_MAX_RB_LEVELS + +/* ADRENO_DEVICE - Given a kgsl_device return the adreno device struct */ +#define ADRENO_DEVICE(device) \ + container_of(device, struct adreno_device, dev) + +/* KGSL_DEVICE - given an adreno_device, return the KGSL device struct */ +#define KGSL_DEVICE(_dev) (&((_dev)->dev)) + +/* ADRENO_CONTEXT - Given a context return the adreno context struct */ +#define ADRENO_CONTEXT(context) \ + container_of(context, struct adreno_context, base) + +/* ADRENO_GPU_DEVICE - Given an adreno device return the GPU specific struct */ +#define ADRENO_GPU_DEVICE(_a) ((_a)->gpucore->gpudev) + +/* + * ADRENO_POWER_OPS - Given an adreno device return the GPU specific power + * ops + */ +#define ADRENO_POWER_OPS(_a) ((_a)->gpucore->gpudev->power_ops) + +#define ADRENO_CHIPID_CORE(_id) FIELD_GET(GENMASK(31, 24), _id) +#define ADRENO_CHIPID_MAJOR(_id) FIELD_GET(GENMASK(23, 16), _id) +#define ADRENO_CHIPID_MINOR(_id) FIELD_GET(GENMASK(15, 8), _id) +#define ADRENO_CHIPID_PATCH(_id) FIELD_GET(GENMASK(7, 0), _id) + +#define ADRENO_GMU_CHIPID(_id) \ + (FIELD_PREP(GENMASK(31, 24), ADRENO_CHIPID_CORE(_id)) | \ + FIELD_PREP(GENMASK(23, 16), ADRENO_CHIPID_MAJOR(_id)) | \ + FIELD_PREP(GENMASK(15, 12), ADRENO_CHIPID_MINOR(_id)) | \ + FIELD_PREP(GENMASK(11, 8), ADRENO_CHIPID_PATCH(_id))) + +#define ADRENO_REV_MAJOR(_rev) FIELD_GET(GENMASK(23, 16), _rev) +#define ADRENO_REV_MINOR(_rev) FIELD_GET(GENMASK(15, 8), _rev) +#define ADRENO_REV_PATCH(_rev) FIELD_GET(GENMASK(7, 0), _rev) + +#define ADRENO_GMU_REV(_rev) \ + (FIELD_PREP(GENMASK(31, 24), ADRENO_REV_MAJOR(_rev)) | \ + FIELD_PREP(GENMASK(23, 16), ADRENO_REV_MINOR(_rev)) | \ + FIELD_PREP(GENMASK(15, 8), ADRENO_REV_PATCH(_rev))) + +/* ADRENO_GPUREV - Return the GPU ID for the given adreno_device */ +#define ADRENO_GPUREV(_a) ((_a)->gpucore->gpurev) + +/* + * ADRENO_FEATURE - return true if the specified feature is supported by the GPU + * core + */ +#define ADRENO_FEATURE(_dev, _bit) \ + ((_dev)->gpucore->features & (_bit)) + +/** + * ADRENO_QUIRK - return true if the specified quirk is required by the GPU + */ +#define ADRENO_QUIRK(_dev, _bit) \ + ((_dev)->quirks & (_bit)) + +#define ADRENO_FW(a, f) (&(a->fw[f])) + +/* Adreno core features */ +/* The core supports SP/TP hw controlled power collapse */ +#define ADRENO_SPTP_PC BIT(0) +/* The GPU supports content protection */ +#define ADRENO_CONTENT_PROTECTION BIT(1) +/* The GPU supports preemption */ +#define ADRENO_PREEMPTION BIT(2) +/* The GPMU supports Limits Management */ +#define ADRENO_LM BIT(3) +/* The GPU supports retention for cpz registers */ +#define ADRENO_CPZ_RETENTION BIT(4) +/* The core has soft fault detection available */ +#define ADRENO_SOFT_FAULT_DETECT BIT(5) +/* The GMU supports IFPC power management*/ +#define ADRENO_IFPC BIT(6) +/* The core supports IO-coherent memory */ +#define ADRENO_IOCOHERENT BIT(7) +/* + * The GMU supports Adaptive Clock Distribution (ACD) + * for droop mitigation + */ +#define ADRENO_ACD BIT(8) +/* Cooperative reset enabled GMU */ +#define ADRENO_COOP_RESET BIT(9) +/* Indicates that the specific target is no longer supported */ +#define ADRENO_DEPRECATED BIT(10) +/* The target supports ringbuffer level APRIV */ +#define ADRENO_APRIV BIT(11) +/* The GMU supports Battery Current Limiting */ +#define ADRENO_BCL BIT(12) +/* L3 voting is supported with L3 constraints */ +#define ADRENO_L3_VOTE BIT(13) +/* LPAC is supported */ +#define ADRENO_LPAC BIT(14) +/* Late Stage Reprojection (LSR) enablment for GMU */ +#define ADRENO_LSR BIT(15) +/* GMU and kernel supports hardware fences */ +#define ADRENO_HW_FENCE BIT(16) +/* Dynamic Mode Switching supported on this target */ +#define ADRENO_DMS BIT(17) +/* AQE supported on this target */ +#define ADRENO_AQE BIT(18) +/* Warm Boot supported on this target */ +#define ADRENO_GMU_WARMBOOT BIT(19) +/* The GPU supports CLX */ +#define ADRENO_CLX BIT(20) + +/* + * Adreno GPU quirks - control bits for various workarounds + */ + +/* Set TWOPASSUSEWFI in PC_DBG_ECO_CNTL (5XX/6XX) */ +#define ADRENO_QUIRK_TWO_PASS_USE_WFI BIT(0) +/* Submit critical packets at GPU wake up */ +#define ADRENO_QUIRK_CRITICAL_PACKETS BIT(1) +/* Mask out RB1-3 activity signals from HW hang detection logic */ +#define ADRENO_QUIRK_FAULT_DETECT_MASK BIT(2) +/* Disable RB sampler datapath clock gating optimization */ +#define ADRENO_QUIRK_DISABLE_RB_DP2CLOCKGATING BIT(3) +/* Disable local memory(LM) feature to avoid corner case error */ +#define ADRENO_QUIRK_DISABLE_LMLOADKILL BIT(4) +/* Allow HFI to use registers to send message to GMU */ +#define ADRENO_QUIRK_HFI_USE_REG BIT(5) +/* Only set protected SECVID registers once */ +#define ADRENO_QUIRK_SECVID_SET_ONCE BIT(6) +/* + * Limit number of read and write transactions from + * UCHE block to GBIF to avoid possible deadlock + * between GBIF, SMMU and MEMNOC. + */ +#define ADRENO_QUIRK_LIMIT_UCHE_GBIF_RW BIT(8) +/* Do explicit mode control of cx gdsc */ +#define ADRENO_QUIRK_CX_GDSC BIT(9) + +/* Command identifiers */ +#define CONTEXT_TO_MEM_IDENTIFIER 0x2EADBEEF +#define CMD_IDENTIFIER 0x2EEDFACE +#define CMD_INTERNAL_IDENTIFIER 0x2EEDD00D +#define START_IB_IDENTIFIER 0x2EADEABE +#define END_IB_IDENTIFIER 0x2ABEDEAD +#define START_PROFILE_IDENTIFIER 0x2DEFADE1 +#define END_PROFILE_IDENTIFIER 0x2DEFADE2 +#define PWRON_FIXUP_IDENTIFIER 0x2AFAFAFA + +/* One cannot wait forever for the core to idle, so set an upper limit to the + * amount of time to wait for the core to go idle + */ +#define ADRENO_IDLE_TIMEOUT (20 * 1000) + +#define ADRENO_FW_PFP 0 +#define ADRENO_FW_SQE 0 +#define ADRENO_FW_PM4 1 +#define ADRENO_FW_AQE 1 + +#define ADRENO_GPUREV_VALUE(_major, _minor, _patchid) (((_major & 0xFF) << 16) | \ + ((_minor & 0xFF) << 8) | \ + (_patchid & 0xFF)) +enum adreno_gpurev { + ADRENO_REV_UNKNOWN = 0, + ADRENO_REV_A304 = 304, + ADRENO_REV_A305 = 305, + ADRENO_REV_A305C = 306, + ADRENO_REV_A306 = 307, + ADRENO_REV_A306A = 308, + ADRENO_REV_A310 = 310, + ADRENO_REV_A320 = 320, + ADRENO_REV_A330 = 330, + ADRENO_REV_A305B = 335, + ADRENO_REV_A405 = 405, + ADRENO_REV_A418 = 418, + ADRENO_REV_A420 = 420, + ADRENO_REV_A430 = 430, + ADRENO_REV_A505 = 505, + ADRENO_REV_A506 = 506, + ADRENO_REV_A508 = 508, + ADRENO_REV_A510 = 510, + ADRENO_REV_A512 = 512, + ADRENO_REV_A530 = 530, + ADRENO_REV_A540 = 540, + ADRENO_REV_A610 = 610, + ADRENO_REV_A611 = 611, + ADRENO_REV_A612 = 612, + ADRENO_REV_A615 = 615, + ADRENO_REV_A616 = 616, + ADRENO_REV_A618 = 618, + ADRENO_REV_A619 = 619, + ADRENO_REV_A620 = 620, + ADRENO_REV_A621 = 621, + ADRENO_REV_A630 = 630, + ADRENO_REV_A635 = 635, + ADRENO_REV_A640 = 640, + ADRENO_REV_A650 = 650, + ADRENO_REV_A660 = 660, + ADRENO_REV_A662 = 662, + ADRENO_REV_A663 = 663, + ADRENO_REV_A680 = 680, + ADRENO_REV_A702 = 702, + /* + * Gen7 and higher version numbers may exceed 1 digit + * Bits 16-23: Major + * Bits 8-15: Minor + * Bits 0-7: Patch id + */ + ADRENO_REV_GEN7_0_0 = ADRENO_GPUREV_VALUE(7, 0, 0), + ADRENO_REV_GEN7_0_1 = ADRENO_GPUREV_VALUE(7, 0, 1), + ADRENO_REV_GEN7_2_0 = ADRENO_GPUREV_VALUE(7, 2, 0), + ADRENO_REV_GEN7_2_1 = ADRENO_GPUREV_VALUE(7, 2, 1), + ADRENO_REV_GEN7_4_0 = ADRENO_GPUREV_VALUE(7, 4, 0), + ADRENO_REV_GEN7_9_0 = ADRENO_GPUREV_VALUE(7, 9, 0), + ADRENO_REV_GEN7_9_1 = ADRENO_GPUREV_VALUE(7, 9, 1), + ADRENO_REV_GEN7_11_0 = ADRENO_GPUREV_VALUE(7, 11, 0), + ADRENO_REV_GEN8_3_0 = ADRENO_GPUREV_VALUE(8, 3, 0), +}; + +#define ADRENO_SOFT_FAULT BIT(0) +#define ADRENO_HARD_FAULT BIT(1) +#define ADRENO_TIMEOUT_FAULT BIT(2) +#define ADRENO_IOMMU_PAGE_FAULT BIT(3) +#define ADRENO_PREEMPT_FAULT BIT(4) +#define ADRENO_GMU_FAULT BIT(5) +#define ADRENO_CTX_DETATCH_TIMEOUT_FAULT BIT(6) +#define ADRENO_GMU_FAULT_SKIP_SNAPSHOT BIT(7) + +enum adreno_pipe_type { + PIPE_NONE = 0, + PIPE_BR = 1, + PIPE_BV = 2, + PIPE_LPAC = 3, + PIPE_AQE0 = 4, + PIPE_AQE1 = 5, + PIPE_DDE_BR = 6, + PIPE_DDE_BV = 7, +}; +/** + * Bit fields for GPU_CX_MISC_CX_AHB_*_CNTL registers + * AHB_TXFRTIMEOUTRELEASE [8:8] + * AHB_TXFRTIMEOUTENABLE [9:9] + * AHB_RESPONDERROR [11:11] + * AHB_ERRORSTATUSENABLE [12:12] + */ +#define ADRENO_AHB_CNTL_DEFAULT (BIT(12) | BIT(11) | BIT(9) | BIT(8)) + +/* number of throttle counters for DCVS adjustment */ +#define ADRENO_GPMU_THROTTLE_COUNTERS 4 + +struct adreno_gpudev; + +/* Time to allow preemption to complete (in ms) */ +#define ADRENO_PREEMPT_TIMEOUT 10000 + +#define PREEMPT_SCRATCH_OFFSET(id) (id * sizeof(u64)) + +#define PREEMPT_SCRATCH_ADDR(dev, id) \ + ((dev)->preempt.scratch->gpuaddr + PREEMPT_SCRATCH_OFFSET(id)) + +/** + * enum adreno_preempt_states + * ADRENO_PREEMPT_NONE: No preemption is scheduled + * ADRENO_PREEMPT_START: The S/W has started + * ADRENO_PREEMPT_TRIGGERED: A preeempt has been triggered in the HW + * ADRENO_PREEMPT_FAULTED: The preempt timer has fired + * ADRENO_PREEMPT_PENDING: The H/W has signaled preemption complete + * ADRENO_PREEMPT_COMPLETE: Preemption could not be finished in the IRQ handler, + * worker has been scheduled + */ +enum adreno_preempt_states { + ADRENO_PREEMPT_NONE = 0, + ADRENO_PREEMPT_START, + ADRENO_PREEMPT_TRIGGERED, + ADRENO_PREEMPT_FAULTED, + ADRENO_PREEMPT_PENDING, + ADRENO_PREEMPT_COMPLETE, +}; + +/** + * struct adreno_protected_regs - container for a protect register span + */ +struct adreno_protected_regs { + /** @reg: Physical protected mode register to write to */ + u32 reg; + /** @start: Dword offset of the starting register in the range */ + u32 start; + /** + * @end: Dword offset of the ending register in the range + * (inclusive) + */ + u32 end; + /** + * @noaccess: 1 if the register should not be accessible from + * userspace, 0 if it can be read (but not written) + */ + u32 noaccess; +}; + +/** + * struct adreno_preemption + * @state: The current state of preemption + * @scratch: Per-target scratch memory for implementation specific functionality + * @timer: A timer to make sure preemption doesn't stall + * @work: A work struct for the preemption worker (for 5XX) + * preempt_level: The level of preemption (for 6XX) + * skipsaverestore: To skip saverestore during L1 preemption (for 6XX) + * usesgmem: enable GMEM save/restore across preemption (for 6XX) + * count: Track the number of preemptions triggered + */ +struct adreno_preemption { + atomic_t state; + struct kgsl_memdesc *scratch; + struct timer_list timer; + struct work_struct work; + unsigned int preempt_level; + bool skipsaverestore; + bool usesgmem; + unsigned int count; + /* @postamble_len: Number of dwords in KMD postamble pm4 packet */ + u32 postamble_len; + /* + * @postamble_bootup_len: Number of dwords in KMD postamble pm4 packet + * that needs to be sent before first submission to GPU. + * Note: Postambles are not preserved across slumber. + */ + u32 postamble_bootup_len; +}; + +struct adreno_busy_data { + unsigned int gpu_busy; + unsigned int bif_ram_cycles; + unsigned int bif_ram_cycles_read_ch1; + unsigned int bif_ram_cycles_write_ch0; + unsigned int bif_ram_cycles_write_ch1; + unsigned int bif_starved_ram; + unsigned int bif_starved_ram_ch1; + unsigned int num_ifpc; + unsigned int throttle_cycles[ADRENO_GPMU_THROTTLE_COUNTERS]; + u32 bcl_throttle; +}; + +/** + * struct adreno_firmware - Struct holding fw details + * @fwvirt: Buffer which holds the ucode + * @size: Size of ucode buffer + * @version: Version of ucode + * @memdesc: Memory descriptor which holds ucode buffer info + */ +struct adreno_firmware { + unsigned int *fwvirt; + size_t size; + unsigned int version; + struct kgsl_memdesc *memdesc; +}; + +/** + * struct adreno_perfcounter_list_node - struct to store perfcounters + * allocated by a process on a kgsl fd. + * @groupid: groupid of the allocated perfcounter + * @countable: countable assigned to the allocated perfcounter + * @node: list node for perfcounter_list of a process + */ +struct adreno_perfcounter_list_node { + unsigned int groupid; + unsigned int countable; + struct list_head node; +}; + +/** + * struct adreno_device_private - Adreno private structure per fd + * @dev_priv: the kgsl device private structure + * @perfcounter_list: list of perfcounters used by the process + */ +struct adreno_device_private { + struct kgsl_device_private dev_priv; + struct list_head perfcounter_list; +}; + +/** + * struct adreno_reglist_list - A container for list of registers and + * number of registers in the list + */ +struct adreno_reglist_list { + /** @reg: List of register **/ + const u32 *regs; + /** @count: Number of registers in the list **/ + u32 count; +}; + +/** + * struct adreno_power_ops - Container for target specific power up/down + * sequences + */ +struct adreno_power_ops { + /** + * @first_open: Target specific function triggered when first kgsl + * instance is opened + */ + int (*first_open)(struct adreno_device *adreno_dev); + /** + * @last_close: Target specific function triggered when last kgsl + * instance is closed + */ + int (*last_close)(struct adreno_device *adreno_dev); + /** + * @active_count_get: Target specific function to keep gpu from power + * collapsing + */ + int (*active_count_get)(struct adreno_device *adreno_dev); + /** + * @active_count_put: Target specific function to allow gpu to power + * collapse + */ + void (*active_count_put)(struct adreno_device *adreno_dev); + /** @pm_suspend: Target specific function to suspend the driver */ + int (*pm_suspend)(struct adreno_device *adreno_dev); + /** @pm_resume: Target specific function to resume the driver */ + void (*pm_resume)(struct adreno_device *adreno_dev); + /** + * @touch_wakeup: Target specific function to start gpu on touch event + */ + void (*touch_wakeup)(struct adreno_device *adreno_dev); + /** @gpu_clock_set: Target specific function to set gpu frequency */ + int (*gpu_clock_set)(struct adreno_device *adreno_dev, u32 pwrlevel); + /** @gpu_bus_set: Target specific function to set gpu bandwidth */ + int (*gpu_bus_set)(struct adreno_device *adreno_dev, int bus_level, + u32 ab); +}; + +/** + * struct adreno_gpu_core - A specific GPU core definition + * @gpurev: Unique GPU revision identifier + * @core: Match for the core version of the GPU + * @major: Match for the major version of the GPU + * @minor: Match for the minor version of the GPU + * @patchid: Match for the patch revision of the GPU + * @features: Common adreno features supported by this core + * @gpudev: Pointer to the GPU family specific functions for this core + * @uche_gmem_alignment: Alignment required for UCHE GMEM base + * @gmem_size: Amount of binning memory (GMEM/OCMEM) to reserve for the core + * @bus_width: Bytes transferred in 1 cycle + */ +struct adreno_gpu_core { + enum adreno_gpurev gpurev; + unsigned int core, major, minor, patchid; + /** + * @compatible: If specified, use the compatible string to match the + * device + */ + const char *compatible; + unsigned long features; + const struct adreno_gpudev *gpudev; + const struct adreno_perfcounters *perfcounters; + u32 uche_gmem_alignment; + size_t gmem_size; + u32 bus_width; + /** @snapshot_size: Size of the static snapshot region in bytes */ + u32 snapshot_size; + /** @num_ddr_channels: Number of DDR channels */ + u32 num_ddr_channels; +}; + +/** + * struct adreno_dispatch_ops - Common functions for dispatcher operations + */ +struct adreno_dispatch_ops { + /* @close: Shut down the dispatcher */ + void (*close)(struct adreno_device *adreno_dev); + /* @queue_cmds: Queue a command on the context */ + int (*queue_cmds)(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, struct kgsl_drawobj *drawobj[], + u32 count, u32 *timestamp); + /* @queue_context: Queue a context to be dispatched */ + void (*queue_context)(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt); + void (*setup_context)(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt); + void (*fault)(struct adreno_device *adreno_dev, u32 fault); + /* @create_hw_fence: Create a hardware fence */ + void (*create_hw_fence)(struct adreno_device *adreno_dev, struct kgsl_sync_fence *kfence); + /* @get_fault: Get the GPU fault status */ + u32 (*get_fault)(struct adreno_device *adreno_dev); +}; + +/** + * struct adreno_device - The mothership structure for all adreno related info + * @dev: Reference to struct kgsl_device + * @priv: Holds the private flags specific to the adreno_device + * @chipid: Chip ID specific to the GPU + * @cx_misc_len: Length of the CX MISC register block + * @cx_misc_virt: Pointer where the CX MISC block is mapped + * @isense_base: Base physical address of isense block + * @isense_len: Length of the isense register block + * @isense_virt: Pointer where isense block is mapped + * @gpucore: Pointer to the adreno_gpu_core structure + * @gpmu_cmds_size: Length of gpmu cmd stream + * @gpmu_cmds: gpmu cmd stream + * @ringbuffers: Array of pointers to adreno_ringbuffers + * @num_ringbuffers: Number of ringbuffers for the GPU + * @cur_rb: Pointer to the current ringbuffer + * @next_rb: Ringbuffer we are switching to during preemption + * @prev_rb: Ringbuffer we are switching from during preemption + * @fast_hang_detect: Software fault detection availability + * @ft_policy: Defines the fault tolerance policy + * @long_ib_detect: Long IB detection availability + * @cooperative_reset: Indicates if graceful death handshake is enabled + * between GMU and GPU + * @profile: Container for adreno profiler information + * @dispatcher: Container for adreno GPU dispatcher + * @pwron_fixup: Command buffer to run a post-power collapse shader workaround + * @pwron_fixup_dwords: Number of dwords in the command buffer + * @input_work: Work struct for turning on the GPU after a touch event + * @busy_data: Struct holding GPU VBIF busy stats + * @ram_cycles_lo: Number of DDR clock cycles for the monitor session (Only + * DDR channel 0 read cycles in case of GBIF) + * @ram_cycles_lo_ch1_read: Number of DDR channel 1 Read clock cycles for + * the monitor session + * @ram_cycles_lo_ch0_write: Number of DDR channel 0 Write clock cycles for + * the monitor session + * @ram_cycles_lo_ch1_write: Number of DDR channel 0 Write clock cycles for + * the monitor session + * @starved_ram_lo: Number of cycles VBIF/GBIF is stalled by DDR (Only channel 0 + * stall cycles in case of GBIF) + * @starved_ram_lo_ch1: Number of cycles GBIF is stalled by DDR channel 1 + * @halt: Atomic variable to check whether the GPU is currently halted + * @pending_irq_refcnt: Atomic variable to keep track of running IRQ handlers + * @ctx_d_debugfs: Context debugfs node + * @profile_buffer: Memdesc holding the drawobj profiling buffer + * @profile_index: Index to store the start/stop ticks in the profiling + * buffer + * @pwrup_reglist: Memdesc holding the power up register list + * which is used by CP during preemption and IFPC + * @lm_sequence: Pointer to the start of the register write sequence for LM + * @lm_size: The dword size of the LM sequence + * @lm_limit: limiting value for LM + * @lm_threshold_count: register value for counter for lm threshold breakin + * @lm_threshold_cross: number of current peaks exceeding threshold + * @ifpc_count: Number of times the GPU went into IFPC + * @highest_bank_bit: Value of the highest bank bit + * @gpmu_throttle_counters - counters for number of throttled clocks + * @irq_storm_work: Worker to handle possible interrupt storms + * @active_list: List to track active contexts + * @active_list_lock: Lock to protect active_list + * @gpu_llc_slice: GPU system cache slice descriptor + * @gpu_llc_slice_enable: To enable the GPU system cache slice or not + * @gpuhtw_llc_slice: GPU pagetables system cache slice descriptor + * @gpuhtw_llc_slice_enable: To enable the GPUHTW system cache slice or not + * @zap_loaded: Used to track if zap was successfully loaded or not + */ +struct adreno_device { + struct kgsl_device dev; /* Must be first field in this struct */ + unsigned long priv; + unsigned int chipid; + /** @uche_gmem_base: Base address of GMEM for UCHE access */ + u64 uche_gmem_base; + unsigned int cx_misc_len; + void __iomem *cx_misc_virt; + unsigned long isense_base; + unsigned int isense_len; + void __iomem *isense_virt; + const struct adreno_gpu_core *gpucore; + struct adreno_firmware fw[2]; + size_t gpmu_cmds_size; + unsigned int *gpmu_cmds; + struct adreno_ringbuffer ringbuffers[KGSL_PRIORITY_MAX_RB_LEVELS]; + int num_ringbuffers; + struct adreno_ringbuffer *cur_rb; + struct adreno_ringbuffer *next_rb; + struct adreno_ringbuffer *prev_rb; + unsigned int fast_hang_detect; + unsigned long ft_policy; + bool long_ib_detect; + bool cooperative_reset; + struct adreno_profile profile; + struct adreno_dispatcher dispatcher; + struct kgsl_memdesc *pwron_fixup; + unsigned int pwron_fixup_dwords; + struct work_struct input_work; + struct adreno_busy_data busy_data; + unsigned int ram_cycles_lo; + unsigned int ram_cycles_lo_ch1_read; + unsigned int ram_cycles_lo_ch0_write; + unsigned int ram_cycles_lo_ch1_write; + unsigned int starved_ram_lo; + unsigned int starved_ram_lo_ch1; + atomic_t halt; + atomic_t pending_irq_refcnt; + struct dentry *ctx_d_debugfs; + /** @lm_enabled: True if limits management is enabled for this target */ + bool lm_enabled; + /** @acd_enabled: True if acd is enabled for this target */ + bool acd_enabled; + /** @hwcg_enabled: True if hardware clock gating is enabled */ + bool hwcg_enabled; + /** @throttling_enabled: True if LM throttling is enabled on a5xx */ + bool throttling_enabled; + /** @sptp_pc_enabled: True if SPTP power collapse is enabled on a5xx */ + bool sptp_pc_enabled; + /** @bcl_enabled: True if BCL is enabled */ + bool bcl_enabled; + /** @clx_enabled: True if CLX is enabled */ + bool clx_enabled; + /** @lpac_enabled: True if LPAC is enabled */ + bool lpac_enabled; + /** @dms_enabled: True if DMS is enabled */ + bool dms_enabled; + /** @warmboot_enabled: True if warmboot is enabled */ + bool warmboot_enabled; + /** @preempt_override: True if command line param enables preemption */ + bool preempt_override; + struct kgsl_memdesc *profile_buffer; + unsigned int profile_index; + struct kgsl_memdesc *pwrup_reglist; + uint32_t *lm_sequence; + uint32_t lm_size; + struct adreno_preemption preempt; + struct work_struct gpmu_work; + uint32_t lm_leakage; + uint32_t lm_limit; + uint32_t lm_threshold_count; + uint32_t lm_threshold_cross; + uint32_t ifpc_count; + + unsigned int highest_bank_bit; + unsigned int quirks; + +#ifdef CONFIG_QCOM_KGSL_CORESIGHT + /** @gx_coresight: A coresight instance for GX */ + struct adreno_coresight_device gx_coresight; + /** @gx_coresight: A coresight instance for CX */ + struct adreno_coresight_device cx_coresight; + /** @funnel_gfx: A coresight instance for gfx funnel */ + struct adreno_funnel_device funnel_gfx; +#endif + + uint32_t gpmu_throttle_counters[ADRENO_GPMU_THROTTLE_COUNTERS]; + struct work_struct irq_storm_work; + + struct list_head active_list; + spinlock_t active_list_lock; + + void *gpu_llc_slice; + bool gpu_llc_slice_enable; + void *gpuhtw_llc_slice; + bool gpuhtw_llc_slice_enable; + unsigned int zap_loaded; + /** + * @critpkts: Memory descriptor for 5xx critical packets if applicable + */ + struct kgsl_memdesc *critpkts; + /** + * @critpkts: Memory descriptor for 5xx secure critical packets + */ + struct kgsl_memdesc *critpkts_secure; + /** @irq_mask: The current interrupt mask for the GPU device */ + u32 irq_mask; + /* + * @soft_ft_regs: an array of registers for soft fault detection on a3xx + * targets + */ + u32 *soft_ft_regs; + /* + * @soft_ft_vals: an array of register values for soft fault detection + * on a3xx targets + */ + u32 *soft_ft_vals; + /* + * @soft_ft_vals: number of elements in @soft_ft_regs and @soft_ft_vals + */ + int soft_ft_count; + /* @dispatch_ops: A pointer to a set of adreno dispatch ops */ + const struct adreno_dispatch_ops *dispatch_ops; + /** @hwsched: Container for the hardware dispatcher */ + struct adreno_hwsched hwsched; + /* + * @perfcounter: Flag to clear perfcounters across contexts and + * controls perfcounter ioctl read + */ + bool perfcounter; + /** @gmu_hub_clk_freq: Gmu hub interface clock frequency */ + u64 gmu_hub_clk_freq; + /* @patch_reglist: If false power up register list needs to be patched */ + bool patch_reglist; + /* + * @uche_client_pf: uche_client_pf client register configuration + * for pf debugging + */ + u32 uche_client_pf; + /** + * @bcl_data: bit 0 contains response type for bcl alarms and bits 1:24 controls + * throttle level for bcl alarm levels 0-2. If not set, gmu fw sets default throttle levels. + */ + u32 bcl_data; + /* + * @bcl_debugfs_dir: Debugfs directory node for bcl related nodes + */ + struct dentry *bcl_debugfs_dir; + /** @bcl_throttle_time_us: Total time in us spent in BCL throttling */ + u32 bcl_throttle_time_us; + /* @preemption_debugfs_dir: Debugfs directory node for preemption related nodes */ + struct dentry *preemption_debugfs_dir; + /* @hwsched_enabled: If true, hwsched is enabled */ + bool hwsched_enabled; + /* @fastblend_enabled: True if fastblend feature is enabled */ + bool fastblend_enabled; + /* @raytracing_enabled: True if raytracing feature is enabled */ + bool raytracing_enabled; + /* @feature_fuse: feature fuse value read from HW */ + u32 feature_fuse; + /** @gmu_ab: Track if GMU supports ab vote */ + bool gmu_ab; + /** @ifpc_hyst: IFPC long hysteresis value */ + u32 ifpc_hyst; + /** @ifpc_hyst_floor: IFPC long hysteresis floor value */ + u32 ifpc_hyst_floor; + /** @cx_misc_base: CX MISC register block base offset */ + u32 cx_misc_base; + /* + * @no_restore_count: Keep track of perfcounter requests that don't have + * ADRENO_PERFCOUNTER_GROUP_RESTORE flag set + */ + u32 no_restore_count; + /* + * @ahb_timeout_val: AHB transaction timeout value. + * If set, a timeout will occur in 2 ^ (ahb_timeout_val + 1) cycles. + */ + u32 ahb_timeout_val; +}; + +/** + * enum adreno_device_flags - Private flags for the adreno_device + * @ADRENO_DEVICE_PWRON - Set during init after a power collapse + * @ADRENO_DEVICE_PWRON_FIXUP - Set if the target requires the shader fixup + * after power collapse + * @ADRENO_DEVICE_STARTED - Set if the device start sequence is in progress + * @ADRENO_DEVICE_FAULT - Set if the device is currently in fault (and shouldn't + * send any more commands to the ringbuffer) + * @ADRENO_DEVICE_DRAWOBJ_PROFILE - Set if the device supports drawobj + * profiling via the ALWAYSON counter + * @ADRENO_DEVICE_PREEMPTION - Turn on/off preemption + * @ADRENO_DEVICE_SOFT_FAULT_DETECT - Set if soft fault detect is enabled + * @ADRENO_DEVICE_GPMU_INITIALIZED - Set if GPMU firmware initialization succeed + * @ADRENO_DEVICE_ISDB_ENABLED - Set if the Integrated Shader DeBugger is + * attached and enabled + * @ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED - Set if a CACHE_FLUSH_TS irq storm + * is in progress + */ +enum adreno_device_flags { + ADRENO_DEVICE_PWRON = 0, + ADRENO_DEVICE_PWRON_FIXUP = 1, + ADRENO_DEVICE_INITIALIZED = 2, + ADRENO_DEVICE_STARTED = 5, + ADRENO_DEVICE_FAULT = 6, + ADRENO_DEVICE_DRAWOBJ_PROFILE = 7, + ADRENO_DEVICE_GPU_REGULATOR_ENABLED = 8, + ADRENO_DEVICE_PREEMPTION = 9, + ADRENO_DEVICE_SOFT_FAULT_DETECT = 10, + ADRENO_DEVICE_GPMU_INITIALIZED = 11, + ADRENO_DEVICE_ISDB_ENABLED = 12, + ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED = 13, + /** @ADRENO_DEVICE_DMS: Set if DMS is enabled */ + ADRENO_DEVICE_DMS = 14, + /** @ADRENO_DEVICE_GMU_AB: Set if AB vote via GMU is enabled */ + ADRENO_DEVICE_GMU_AB = 15, + /* + * @ADRENO_DEVICE_FORCE_COLDBOOT: Set if a feature is toggled + * via sysfs/debugfs or when we are doing fault recovery + */ + ADRENO_DEVICE_FORCE_COLDBOOT = 16, + /** @ADRENO_DEVICE_CX_TIMER_INITIALIZED: Set if the CX timer is initialized */ + ADRENO_DEVICE_CX_TIMER_INITIALIZED = 17, +}; + +/** + * struct adreno_drawobj_profile_entry - a single drawobj entry in the + * kernel profiling buffer + * @started: Number of GPU ticks at start of the drawobj + * @retired: Number of GPU ticks at the end of the drawobj + * @ctx_start: CP_ALWAYS_ON_CONTEXT tick at start of the drawobj + * @ctx_end: CP_ALWAYS_ON_CONTEXT tick at end of the drawobj + */ +struct adreno_drawobj_profile_entry { + uint64_t started; + uint64_t retired; + uint64_t ctx_start; + uint64_t ctx_end; +}; + +#define ADRENO_DRAWOBJ_PROFILE_OFFSET(_index, _member) \ + ((_index) * sizeof(struct adreno_drawobj_profile_entry) \ + + offsetof(struct adreno_drawobj_profile_entry, _member)) + + +/** + * adreno_regs: List of registers that are used in kgsl driver for all + * 3D devices. Each device type has different offset value for the same + * register, so an array of register offsets are declared for every device + * and are indexed by the enumeration values defined in this enum + */ +enum adreno_regs { + ADRENO_REG_CP_ME_RAM_DATA, + ADRENO_REG_CP_RB_BASE, + ADRENO_REG_CP_RB_BASE_HI, + ADRENO_REG_CP_RB_RPTR_ADDR_LO, + ADRENO_REG_CP_RB_RPTR_ADDR_HI, + ADRENO_REG_CP_RB_RPTR, + ADRENO_REG_CP_RB_WPTR, + ADRENO_REG_CP_ME_CNTL, + ADRENO_REG_CP_RB_CNTL, + ADRENO_REG_CP_IB1_BASE, + ADRENO_REG_CP_IB1_BASE_HI, + ADRENO_REG_CP_IB1_BUFSZ, + ADRENO_REG_CP_IB2_BASE, + ADRENO_REG_CP_IB2_BASE_HI, + ADRENO_REG_CP_IB2_BUFSZ, + ADRENO_REG_CP_TIMESTAMP, + ADRENO_REG_CP_SCRATCH_REG6, + ADRENO_REG_CP_SCRATCH_REG7, + ADRENO_REG_CP_PROTECT_STATUS, + ADRENO_REG_CP_PREEMPT, + ADRENO_REG_CP_PREEMPT_DEBUG, + ADRENO_REG_CP_PREEMPT_DISABLE, + ADRENO_REG_CP_PROTECT_REG_0, + ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_LO, + ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_HI, + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO, + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI, + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO, + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI, + ADRENO_REG_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO, + ADRENO_REG_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI, + ADRENO_REG_CP_PREEMPT_LEVEL_STATUS, + ADRENO_REG_RBBM_STATUS, + ADRENO_REG_RBBM_STATUS3, + ADRENO_REG_RBBM_PERFCTR_LOAD_CMD0, + ADRENO_REG_RBBM_PERFCTR_LOAD_CMD1, + ADRENO_REG_RBBM_PERFCTR_LOAD_CMD2, + ADRENO_REG_RBBM_PERFCTR_LOAD_CMD3, + ADRENO_REG_RBBM_PERFCTR_PWR_1_LO, + ADRENO_REG_RBBM_INT_0_MASK, + ADRENO_REG_RBBM_PM_OVERRIDE2, + ADRENO_REG_RBBM_SW_RESET_CMD, + ADRENO_REG_RBBM_CLOCK_CTL, + ADRENO_REG_PA_SC_AA_CONFIG, + ADRENO_REG_SQ_GPR_MANAGEMENT, + ADRENO_REG_SQ_INST_STORE_MANAGEMENT, + ADRENO_REG_TP0_CHICKEN, + ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_LO, + ADRENO_REG_RBBM_PERFCTR_LOAD_VALUE_HI, + ADRENO_REG_GMU_AO_HOST_INTERRUPT_MASK, + ADRENO_REG_GMU_AHB_FENCE_STATUS, + ADRENO_REG_GMU_GMU2HOST_INTR_MASK, + ADRENO_REG_GPMU_POWER_COUNTER_ENABLE, + ADRENO_REG_REGISTER_MAX, +}; + +#define ADRENO_REG_UNUSED 0xFFFFFFFF +#define ADRENO_REG_SKIP 0xFFFFFFFE +#define ADRENO_REG_DEFINE(_offset, _reg)[_offset] = _reg + +struct adreno_irq_funcs { + void (*func)(struct adreno_device *adreno_dev, int mask); +}; +#define ADRENO_IRQ_CALLBACK(_c) { .func = _c } + +/* + * struct adreno_debugbus_block - Holds info about debug buses of a chip + * @block_id: Bus identifier + * @dwords: Number of dwords of data that this block holds + */ +struct adreno_debugbus_block { + unsigned int block_id; + unsigned int dwords; +}; + +enum adreno_cp_marker_type { + IFPC_DISABLE, + IFPC_ENABLE, + IB1LIST_START, + IB1LIST_END, +}; + +struct adreno_gpudev { + /* + * These registers are in a different location on different devices, + * so define them in the structure and use them as variables. + */ + unsigned int *const reg_offsets; + + /* GPU specific function hooks */ + int (*probe)(struct platform_device *pdev, u32 chipid, + const struct adreno_gpu_core *gpucore); + void (*snapshot)(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot); + irqreturn_t (*irq_handler)(struct adreno_device *adreno_dev); + int (*init)(struct adreno_device *adreno_dev); + void (*remove)(struct adreno_device *adreno_dev); + int (*rb_start)(struct adreno_device *adreno_dev); + int (*start)(struct adreno_device *adreno_dev); + int (*regulator_enable)(struct adreno_device *adreno_dev); + void (*regulator_disable)(struct adreno_device *adreno_dev); + void (*pwrlevel_change_settings)(struct adreno_device *adreno_dev, + unsigned int prelevel, unsigned int postlevel, + bool post); + void (*preemption_schedule)(struct adreno_device *adreno_dev); + int (*preemption_context_init)(struct kgsl_context *context); + void (*context_detach)(struct adreno_context *drawctxt); + void (*pre_reset)(struct adreno_device *adreno_dev); + void (*gpu_keepalive)(struct adreno_device *adreno_dev, + bool state); + bool (*hw_isidle)(struct adreno_device *adreno_dev); + const char *(*iommu_fault_block)(struct kgsl_device *device, + unsigned int fsynr1); + int (*reset)(struct adreno_device *adreno_dev); + /** @read_alwayson: Return the current value of the alwayson counter */ + u64 (*read_alwayson)(struct adreno_device *adreno_dev); + /** + * @power_ops: Target specific function pointers to power up/down the + * gpu + */ + const struct adreno_power_ops *power_ops; + int (*clear_pending_transactions)(struct adreno_device *adreno_dev); + void (*deassert_gbif_halt)(struct adreno_device *adreno_dev); + int (*ringbuffer_submitcmd)(struct adreno_device *adreno_dev, + struct kgsl_drawobj_cmd *cmdobj, u32 flags, + struct adreno_submit_time *time); + /** + * @is_hw_collapsible: Return true if the hardware can be collapsed. + * Only used by non GMU/RGMU targets + */ + bool (*is_hw_collapsible)(struct adreno_device *adreno_dev); + /** + * @power_stats - Return the perfcounter statistics for DCVS + */ + void (*power_stats)(struct adreno_device *adreno_dev, + struct kgsl_power_stats *stats); + int (*setproperty)(struct kgsl_device_private *priv, u32 type, + void __user *value, u32 sizebytes); + int (*add_to_va_minidump)(struct adreno_device *adreno_dev); + /** + * @gx_is_on - Return true if both gfx clock and gxgdsc are enabled. + */ + bool (*gx_is_on)(struct adreno_device *adreno_dev); + /** + * @send_recurring_cmdobj - Target specific function to send recurring IBs to GMU + */ + int (*send_recurring_cmdobj)(struct adreno_device *adreno_dev, + struct kgsl_drawobj_cmd *cmdobj); + /** + * @perfcounter_remove: Remove perfcounter from the power up list + */ + int (*perfcounter_remove)(struct adreno_device *adreno_dev, + struct adreno_perfcount_register *reg, u32 groupid); + /** + * @set_isdb_breakpoint_registers - Program isdb registers to issue break command + */ + void (*set_isdb_breakpoint_registers)(struct adreno_device *adreno_dev); + /** + * @context_destroy: Target specific function called during context destruction + */ + void (*context_destroy)(struct adreno_device *adreno_dev, struct adreno_context *drawctxt); + /** + * @swfuse_irqctrl: To enable/disable sw fuse violation interrupt + */ + void (*swfuse_irqctrl)(struct adreno_device *adreno_dev, bool state); + /** + * @lpac_store: To enable/disable lpac at runtime + */ + int (*lpac_store)(struct adreno_device *adreno_dev, bool enable); + /* + * @get_uche_trap_base: Return the UCHE_TRAP_BASE value + */ + u64 (*get_uche_trap_base)(void); + /** + * @fault_header: Print fault header + */ + void (*fault_header)(struct adreno_device *adreno_dev, struct kgsl_drawobj *drawobj); + /** + * @lpac_fault_header: Print LPAC fault header + */ + void (*lpac_fault_header)(struct adreno_device *adreno_dev, struct kgsl_drawobj *drawobj); +}; + +/** + * enum kgsl_ft_policy_bits - KGSL fault tolerance policy bits + * @KGSL_FT_OFF: Disable fault detection (not used) + * @KGSL_FT_REPLAY: Replay the faulting command + * @KGSL_FT_SKIPIB: Skip the faulting indirect buffer + * @KGSL_FT_SKIPFRAME: Skip the frame containing the faulting IB + * @KGSL_FT_DISABLE: Tells the dispatcher to disable FT for the command obj + * @KGSL_FT_TEMP_DISABLE: Disables FT for all commands + * @KGSL_FT_THROTTLE: Disable the context if it faults too often + * @KGSL_FT_SKIPCMD: Skip the command containing the faulting IB + */ +enum kgsl_ft_policy_bits { + KGSL_FT_OFF = 0, + KGSL_FT_REPLAY, + KGSL_FT_SKIPIB, + KGSL_FT_SKIPFRAME, + KGSL_FT_DISABLE, + KGSL_FT_TEMP_DISABLE, + KGSL_FT_THROTTLE, + KGSL_FT_SKIPCMD, + /* KGSL_FT_MAX_BITS is used to calculate the mask */ + KGSL_FT_MAX_BITS, + /* Internal bits - set during GFT */ + /* Skip the PM dump on replayed command obj's */ + KGSL_FT_SKIP_PMDUMP = 31, +}; + +#define KGSL_FT_POLICY_MASK GENMASK(KGSL_FT_MAX_BITS - 1, 0) + +#define FOR_EACH_RINGBUFFER(_dev, _rb, _i) \ + for ((_i) = 0, (_rb) = &((_dev)->ringbuffers[0]); \ + (_i) < (_dev)->num_ringbuffers; \ + (_i)++, (_rb)++) + +extern const struct adreno_power_ops adreno_power_operations; + +extern const struct adreno_gpudev adreno_a3xx_gpudev; +extern const struct adreno_gpudev adreno_a5xx_gpudev; +extern const struct adreno_gpudev adreno_a6xx_gpudev; +extern const struct adreno_gpudev adreno_a6xx_rgmu_gpudev; +extern const struct adreno_gpudev adreno_a619_holi_gpudev; +extern const struct adreno_gpudev adreno_a611_gpudev; + +extern int adreno_wake_nice; +extern unsigned int adreno_wake_timeout; + +int adreno_start(struct kgsl_device *device, int priority); +long adreno_ioctl(struct kgsl_device_private *dev_priv, + unsigned int cmd, unsigned long arg); + +long adreno_ioctl_helper(struct kgsl_device_private *dev_priv, + unsigned int cmd, unsigned long arg, + const struct kgsl_ioctl *cmds, int len); + +int adreno_spin_idle(struct adreno_device *device, unsigned int timeout); +int adreno_idle(struct kgsl_device *device); + +int adreno_set_constraint(struct kgsl_device *device, + struct kgsl_context *context, + struct kgsl_device_constraint *constraint); + +void adreno_snapshot(struct kgsl_device *device, + struct kgsl_snapshot *snapshot, + struct kgsl_context *context, struct kgsl_context *context_lpac); + +int adreno_reset(struct kgsl_device *device, int fault); + +void adreno_fault_skipcmd_detached(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, + struct kgsl_drawobj *drawobj); + +void adreno_hang_int_callback(struct adreno_device *adreno_dev, int bit); +void adreno_cp_callback(struct adreno_device *adreno_dev, int bit); + +int adreno_sysfs_init(struct adreno_device *adreno_dev); + +void adreno_irqctrl(struct adreno_device *adreno_dev, int state); + +long adreno_ioctl_perfcounter_get(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); + +long adreno_ioctl_perfcounter_put(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); + +void adreno_cx_misc_regread(struct adreno_device *adreno_dev, + unsigned int offsetwords, unsigned int *value); +void adreno_cx_misc_regwrite(struct adreno_device *adreno_dev, + unsigned int offsetwords, unsigned int value); +void adreno_cx_misc_regrmw(struct adreno_device *adreno_dev, + unsigned int offsetwords, + unsigned int mask, unsigned int bits); +void adreno_isense_regread(struct adreno_device *adreno_dev, + unsigned int offsetwords, unsigned int *value); +bool adreno_gx_is_on(struct adreno_device *adreno_dev); + +u64 adreno_read_cx_timer(struct adreno_device *adreno_dev); + +/** + * adreno_active_count_get - Wrapper for target specific active count get + * @adreno_dev: pointer to the adreno device + * + * Increase the active count for the KGSL device and execute slumber exit + * sequence if this is the first reference. Code paths that need to touch the + * hardware or wait for the hardware to complete an operation must hold an + * active count reference until they are finished. The device mutex must be held + * while calling this function. + * + * Return: 0 on success or negative error on failure to wake up the device + */ +int adreno_active_count_get(struct adreno_device *adreno_dev); + +/** + * adreno_active_count_put - Wrapper for target specific active count put + * @adreno_dev: pointer to the adreno device + * + * Decrease the active or the KGSL device and schedule the idle thread to + * execute the slumber sequence if there are no remaining references. The + * device mutex must be held while calling this function. + */ +void adreno_active_count_put(struct adreno_device *adreno_dev); + +#define ADRENO_TARGET(_name, _id) \ +static inline int adreno_is_##_name(struct adreno_device *adreno_dev) \ +{ \ + return (ADRENO_GPUREV(adreno_dev) == (_id)); \ +} + +static inline int adreno_is_a3xx(struct adreno_device *adreno_dev) +{ + return ((ADRENO_GPUREV(adreno_dev) >= 300) && + (ADRENO_GPUREV(adreno_dev) < 400)); +} + +ADRENO_TARGET(a304, ADRENO_REV_A304) +ADRENO_TARGET(a306, ADRENO_REV_A306) +ADRENO_TARGET(a306a, ADRENO_REV_A306A) + +static inline int adreno_is_a5xx(struct adreno_device *adreno_dev) +{ + return ADRENO_GPUREV(adreno_dev) >= 500 && + ADRENO_GPUREV(adreno_dev) < 600; +} + +ADRENO_TARGET(a505, ADRENO_REV_A505) +ADRENO_TARGET(a506, ADRENO_REV_A506) +ADRENO_TARGET(a508, ADRENO_REV_A508) +ADRENO_TARGET(a510, ADRENO_REV_A510) +ADRENO_TARGET(a512, ADRENO_REV_A512) +ADRENO_TARGET(a530, ADRENO_REV_A530) +ADRENO_TARGET(a540, ADRENO_REV_A540) + +static inline int adreno_is_a530v2(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A530) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 1); +} + +static inline int adreno_is_a530v3(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A530) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 2); +} + +static inline int adreno_is_a505_or_a506(struct adreno_device *adreno_dev) +{ + return ADRENO_GPUREV(adreno_dev) >= 505 && + ADRENO_GPUREV(adreno_dev) <= 506; +} + +static inline int adreno_is_a6xx(struct adreno_device *adreno_dev) +{ + return ADRENO_GPUREV(adreno_dev) >= 600 && + ADRENO_GPUREV(adreno_dev) <= 702; +} + +static inline int adreno_is_a660_shima(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A660) && + (adreno_dev->gpucore->compatible && + !strcmp(adreno_dev->gpucore->compatible, + "qcom,adreno-gpu-a660-shima")); +} + +ADRENO_TARGET(a610, ADRENO_REV_A610) +ADRENO_TARGET(a611, ADRENO_REV_A611) +ADRENO_TARGET(a612, ADRENO_REV_A612) +ADRENO_TARGET(a618, ADRENO_REV_A618) +ADRENO_TARGET(a619, ADRENO_REV_A619) +ADRENO_TARGET(a621, ADRENO_REV_A621) +ADRENO_TARGET(a630, ADRENO_REV_A630) +ADRENO_TARGET(a635, ADRENO_REV_A635) +ADRENO_TARGET(a662, ADRENO_REV_A662) +ADRENO_TARGET(a640, ADRENO_REV_A640) +ADRENO_TARGET(a650, ADRENO_REV_A650) +ADRENO_TARGET(a663, ADRENO_REV_A663) +ADRENO_TARGET(a680, ADRENO_REV_A680) +ADRENO_TARGET(a702, ADRENO_REV_A702) + +/* A635 is derived from A660 and shares same logic */ +static inline int adreno_is_a660(struct adreno_device *adreno_dev) +{ + unsigned int rev = ADRENO_GPUREV(adreno_dev); + + return (rev == ADRENO_REV_A660 || rev == ADRENO_REV_A635 || + rev == ADRENO_REV_A662); +} + +/* + * All the derived chipsets from A615 needs to be added to this + * list such as A616, A618, A619 etc. + */ +static inline int adreno_is_a615_family(struct adreno_device *adreno_dev) +{ + unsigned int rev = ADRENO_GPUREV(adreno_dev); + + return (rev == ADRENO_REV_A615 || rev == ADRENO_REV_A616 || + rev == ADRENO_REV_A618 || rev == ADRENO_REV_A619); +} + +/* + * Derived GPUs from A640 needs to be added to this list. + * A640 and A680 belongs to this family. + */ +static inline int adreno_is_a640_family(struct adreno_device *adreno_dev) +{ + unsigned int rev = ADRENO_GPUREV(adreno_dev); + + return (rev == ADRENO_REV_A640 || rev == ADRENO_REV_A680); +} + +/* + * Derived GPUs from A650 needs to be added to this list. + * A650 is derived from A640 but register specs has been + * changed hence do not belongs to A640 family. A620, A621, + * A660, A663, A690 follows the register specs of A650. + * + */ +static inline int adreno_is_a650_family(struct adreno_device *adreno_dev) +{ + unsigned int rev = ADRENO_GPUREV(adreno_dev); + + return (rev == ADRENO_REV_A650 || rev == ADRENO_REV_A620 || + rev == ADRENO_REV_A660 || rev == ADRENO_REV_A635 || + rev == ADRENO_REV_A662 || rev == ADRENO_REV_A621 || + rev == ADRENO_REV_A663); +} + +static inline int adreno_is_a619_holi(struct adreno_device *adreno_dev) +{ + return of_device_is_compatible(adreno_dev->dev.pdev->dev.of_node, + "qcom,adreno-gpu-a619-holi"); +} + +static inline int adreno_is_a620(struct adreno_device *adreno_dev) +{ + unsigned int rev = ADRENO_GPUREV(adreno_dev); + + return (rev == ADRENO_REV_A620 || rev == ADRENO_REV_A621); +} + +static inline int adreno_is_a610_family(struct adreno_device *adreno_dev) +{ + unsigned int rev = ADRENO_GPUREV(adreno_dev); + + return (rev == ADRENO_REV_A610 || rev == ADRENO_REV_A611); +} + +static inline int adreno_is_a640v2(struct adreno_device *adreno_dev) +{ + return (ADRENO_GPUREV(adreno_dev) == ADRENO_REV_A640) && + (ADRENO_CHIPID_PATCH(adreno_dev->chipid) == 1); +} + +static inline int adreno_is_gen7(struct adreno_device *adreno_dev) +{ + return ADRENO_GPUREV(adreno_dev) >= 0x070000 && + ADRENO_GPUREV(adreno_dev) < 0x080000; +} + +static inline int adreno_is_gen8(struct adreno_device *adreno_dev) +{ + return ADRENO_GPUREV(adreno_dev) >= 0x080000 && + ADRENO_GPUREV(adreno_dev) < 0x090000; +} + +ADRENO_TARGET(gen7_0_0, ADRENO_REV_GEN7_0_0) +ADRENO_TARGET(gen7_0_1, ADRENO_REV_GEN7_0_1) +ADRENO_TARGET(gen7_2_0, ADRENO_REV_GEN7_2_0) +ADRENO_TARGET(gen7_2_1, ADRENO_REV_GEN7_2_1) +ADRENO_TARGET(gen7_4_0, ADRENO_REV_GEN7_4_0) +ADRENO_TARGET(gen7_9_0, ADRENO_REV_GEN7_9_0) +ADRENO_TARGET(gen7_9_1, ADRENO_REV_GEN7_9_1) +ADRENO_TARGET(gen7_11_0, ADRENO_REV_GEN7_11_0) +ADRENO_TARGET(gen8_3_0, ADRENO_REV_GEN8_3_0) + +static inline int adreno_is_gen7_9_x(struct adreno_device *adreno_dev) +{ + return adreno_is_gen7_9_0(adreno_dev) || adreno_is_gen7_9_1(adreno_dev); +} + +static inline int adreno_is_gen7_0_x_family(struct adreno_device *adreno_dev) +{ + return adreno_is_gen7_0_0(adreno_dev) || adreno_is_gen7_0_1(adreno_dev) || + adreno_is_gen7_4_0(adreno_dev); +} + +static inline int adreno_is_gen7_2_x_family(struct adreno_device *adreno_dev) +{ + return adreno_is_gen7_2_0(adreno_dev) || adreno_is_gen7_2_1(adreno_dev) || + adreno_is_gen7_9_x(adreno_dev) || adreno_is_gen7_11_0(adreno_dev); +} + +/* + * adreno_checkreg_off() - Checks the validity of a register enum + * @adreno_dev: Pointer to adreno device + * @offset_name: The register enum that is checked + */ +static inline bool adreno_checkreg_off(struct adreno_device *adreno_dev, + enum adreno_regs offset_name) +{ + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (offset_name >= ADRENO_REG_REGISTER_MAX || + gpudev->reg_offsets[offset_name] == ADRENO_REG_UNUSED) + return false; + + /* + * GPU register programming is kept common as much as possible + * across the cores, Use ADRENO_REG_SKIP when certain register + * programming needs to be skipped for certain GPU cores. + * Example: Certain registers on a5xx like IB1_BASE are 64 bit. + * Common programming programs 64bit register but upper 32 bits + * are skipped in a3xx using ADRENO_REG_SKIP. + */ + if (gpudev->reg_offsets[offset_name] == ADRENO_REG_SKIP) + return false; + + return true; +} + +/* + * adreno_readreg() - Read a register by getting its offset from the + * offset array defined in gpudev node + * @adreno_dev: Pointer to the adreno device + * @offset_name: The register enum that is to be read + * @val: Register value read is placed here + */ +static inline void adreno_readreg(struct adreno_device *adreno_dev, + enum adreno_regs offset_name, unsigned int *val) +{ + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (adreno_checkreg_off(adreno_dev, offset_name)) + kgsl_regread(KGSL_DEVICE(adreno_dev), + gpudev->reg_offsets[offset_name], val); + else + *val = 0; +} + +/* + * adreno_writereg() - Write a register by getting its offset from the + * offset array defined in gpudev node + * @adreno_dev: Pointer to the adreno device + * @offset_name: The register enum that is to be written + * @val: Value to write + */ +static inline void adreno_writereg(struct adreno_device *adreno_dev, + enum adreno_regs offset_name, unsigned int val) +{ + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (adreno_checkreg_off(adreno_dev, offset_name)) + kgsl_regwrite(KGSL_DEVICE(adreno_dev), + gpudev->reg_offsets[offset_name], val); +} + +/* + * adreno_getreg() - Returns the offset value of a register from the + * register offset array in the gpudev node + * @adreno_dev: Pointer to the adreno device + * @offset_name: The register enum whore offset is returned + */ +static inline unsigned int adreno_getreg(struct adreno_device *adreno_dev, + enum adreno_regs offset_name) +{ + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (!adreno_checkreg_off(adreno_dev, offset_name)) + return ADRENO_REG_REGISTER_MAX; + return gpudev->reg_offsets[offset_name]; +} + +/* + * adreno_write_gmureg() - Write a GMU register by getting its offset from the + * offset array defined in gpudev node + * @adreno_dev: Pointer to the adreno device + * @offset_name: The register enum that is to be written + * @val: Value to write + */ +static inline void adreno_write_gmureg(struct adreno_device *adreno_dev, + enum adreno_regs offset_name, unsigned int val) +{ + const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + + if (adreno_checkreg_off(adreno_dev, offset_name)) + gmu_core_regwrite(KGSL_DEVICE(adreno_dev), + gpudev->reg_offsets[offset_name], val); +} + +/** + * adreno_gpu_fault() - Return the current state of the GPU + * @adreno_dev: A pointer to the adreno_device to query + * + * Return 0 if there is no fault or positive with the last type of fault that + * occurred + */ +static inline unsigned int adreno_gpu_fault(struct adreno_device *adreno_dev) +{ + /* make sure we're reading the latest value */ + smp_rmb(); + return atomic_read(&adreno_dev->dispatcher.fault); +} + +/** + * adreno_set_gpu_fault() - Set the current fault status of the GPU + * @adreno_dev: A pointer to the adreno_device to set + * @state: fault state to set + * + */ +static inline void adreno_set_gpu_fault(struct adreno_device *adreno_dev, + int state) +{ + /* only set the fault bit w/o overwriting other bits */ + atomic_or(state, &adreno_dev->dispatcher.fault); + + /* make sure other CPUs see the update */ + smp_wmb(); +} + +/** + * adreno_clear_gpu_fault() - Clear the GPU fault register + * @adreno_dev: A pointer to an adreno_device structure + * + * Clear the GPU fault status for the adreno device + */ + +static inline void adreno_clear_gpu_fault(struct adreno_device *adreno_dev) +{ + atomic_set(&adreno_dev->dispatcher.fault, 0); + + /* make sure other CPUs see the update */ + smp_wmb(); +} + +/** + * adreno_gpu_halt() - Return the GPU halt refcount + * @adreno_dev: A pointer to the adreno_device + */ +static inline int adreno_gpu_halt(struct adreno_device *adreno_dev) +{ + /* make sure we're reading the latest value */ + smp_rmb(); + return atomic_read(&adreno_dev->halt); +} + + +/** + * adreno_clear_gpu_halt() - Clear the GPU halt refcount + * @adreno_dev: A pointer to the adreno_device + */ +static inline void adreno_clear_gpu_halt(struct adreno_device *adreno_dev) +{ + atomic_set(&adreno_dev->halt, 0); + + /* make sure other CPUs see the update */ + smp_wmb(); +} + +/** + * adreno_get_gpu_halt() - Increment GPU halt refcount + * @adreno_dev: A pointer to the adreno_device + */ +static inline void adreno_get_gpu_halt(struct adreno_device *adreno_dev) +{ + atomic_inc(&adreno_dev->halt); +} + +/** + * adreno_put_gpu_halt() - Decrement GPU halt refcount + * @adreno_dev: A pointer to the adreno_device + */ +static inline void adreno_put_gpu_halt(struct adreno_device *adreno_dev) +{ + /* Make sure the refcount is good */ + int ret = atomic_dec_if_positive(&adreno_dev->halt); + + WARN(ret < 0, "GPU halt refcount unbalanced\n"); +} + + +#ifdef CONFIG_DEBUG_FS +void adreno_debugfs_init(struct adreno_device *adreno_dev); +void adreno_context_debugfs_init(struct adreno_device *adreno_dev, + struct adreno_context *ctx); +#else +static inline void adreno_debugfs_init(struct adreno_device *adreno_dev) { } +static inline void adreno_context_debugfs_init(struct adreno_device *device, + struct adreno_context *context) +{ + context->debug_root = NULL; +} +#endif + +/** + * adreno_compare_pm4_version() - Compare the PM4 microcode version + * @adreno_dev: Pointer to the adreno_device struct + * @version: Version number to compare again + * + * Compare the current version against the specified version and return -1 if + * the current code is older, 0 if equal or 1 if newer. + */ +static inline int adreno_compare_pm4_version(struct adreno_device *adreno_dev, + unsigned int version) +{ + if (adreno_dev->fw[ADRENO_FW_PM4].version == version) + return 0; + + return (adreno_dev->fw[ADRENO_FW_PM4].version > version) ? 1 : -1; +} + +/** + * adreno_compare_pfp_version() - Compare the PFP microcode version + * @adreno_dev: Pointer to the adreno_device struct + * @version: Version number to compare against + * + * Compare the current version against the specified version and return -1 if + * the current code is older, 0 if equal or 1 if newer. + */ +static inline int adreno_compare_pfp_version(struct adreno_device *adreno_dev, + unsigned int version) +{ + if (adreno_dev->fw[ADRENO_FW_PFP].version == version) + return 0; + + return (adreno_dev->fw[ADRENO_FW_PFP].version > version) ? 1 : -1; +} + +/** + * adreno_in_preempt_state() - Check if preemption state is equal to given state + * @adreno_dev: Device whose preemption state is checked + * @state: State to compare against + */ +static inline bool adreno_in_preempt_state(struct adreno_device *adreno_dev, + enum adreno_preempt_states state) +{ + return atomic_read(&adreno_dev->preempt.state) == state; +} +/** + * adreno_set_preempt_state() - Set the specified preemption state + * @adreno_dev: Device to change preemption state + * @state: State to set + */ +static inline void adreno_set_preempt_state(struct adreno_device *adreno_dev, + enum adreno_preempt_states state) +{ + /* + * atomic_set doesn't use barriers, so we need to do it ourselves. One + * before... + */ + smp_wmb(); + atomic_set(&adreno_dev->preempt.state, state); + + /* ... and one after */ + smp_wmb(); +} + +static inline bool adreno_is_preemption_enabled( + struct adreno_device *adreno_dev) +{ + return test_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv); +} + + +/** + * adreno_preemption_feature_set() - Check whether adreno preemption feature is statically enabled + * either via adreno feature bit, or via the cmdline override + * @adreno_dev: Device whose preemption state is checked + */ +static inline bool adreno_preemption_feature_set(struct adreno_device *adreno_dev) +{ + return ADRENO_FEATURE(adreno_dev, ADRENO_PREEMPTION) || adreno_dev->preempt_override; +} + +/* + * adreno_compare_prio_level() - Compares 2 priority levels based on enum values + * @p1: First priority level + * @p2: Second priority level + * + * Returns greater than 0 if p1 is higher priority, 0 if levels are equal else + * less than 0 + */ +static inline int adreno_compare_prio_level(int p1, int p2) +{ + return p2 - p1; +} + +void adreno_readreg64(struct adreno_device *adreno_dev, + enum adreno_regs lo, enum adreno_regs hi, uint64_t *val); + +void adreno_writereg64(struct adreno_device *adreno_dev, + enum adreno_regs lo, enum adreno_regs hi, uint64_t val); + +unsigned int adreno_get_rptr(struct adreno_ringbuffer *rb); + +void adreno_touch_wake(struct kgsl_device *device); + +static inline bool adreno_rb_empty(struct adreno_ringbuffer *rb) +{ + return (adreno_get_rptr(rb) == rb->wptr); +} + +static inline bool adreno_soft_fault_detect(struct adreno_device *adreno_dev) +{ + return adreno_dev->fast_hang_detect && + !test_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv); +} + +static inline bool adreno_long_ib_detect(struct adreno_device *adreno_dev) +{ + return adreno_dev->long_ib_detect && + !test_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv); +} + +/** + * adreno_support_64bit - Return true if the GPU supports 64 bit addressing + * @adreno_dev: An Adreno GPU device handle + * + * Return: True if the device supports 64 bit addressing + */ +static inline bool adreno_support_64bit(struct adreno_device *adreno_dev) +{ + /* + * The IOMMU API takes a unsigned long for the iova so we can't support + * 64 bit addresses when the kernel is in 32 bit mode even if we wanted + * so we need to check that we are using a5xx or newer and that the + * unsigned long is big enough for our purposes. + */ + return (BITS_PER_LONG > 32 && ADRENO_GPUREV(adreno_dev) >= 500); +} + +static inline void adreno_ringbuffer_set_pagetable(struct kgsl_device *device, + struct adreno_ringbuffer *rb, struct kgsl_pagetable *pt) +{ + unsigned long flags; + + spin_lock_irqsave(&rb->preempt_lock, flags); + + kgsl_sharedmem_writel(device->scratch, + SCRATCH_RB_OFFSET(rb->id, current_rb_ptname), pt->name); + + kgsl_sharedmem_writeq(device->scratch, + SCRATCH_RB_OFFSET(rb->id, ttbr0), + kgsl_mmu_pagetable_get_ttbr0(pt)); + + kgsl_sharedmem_writel(device->scratch, + SCRATCH_RB_OFFSET(rb->id, contextidr), 0); + + spin_unlock_irqrestore(&rb->preempt_lock, flags); +} + +static inline u32 counter_delta(struct kgsl_device *device, + unsigned int reg, unsigned int *counter) +{ + u32 val, ret = 0; + + if (!reg) + return 0; + + kgsl_regread(device, reg, &val); + + if (*counter) { + if (val >= *counter) + ret = val - *counter; + else + ret = (UINT_MAX - *counter) + val; + } + + *counter = val; + return ret; +} + +static inline int adreno_perfcntr_active_oob_get( + struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret = adreno_active_count_get(adreno_dev); + + if (!ret) { + ret = gmu_core_dev_oob_set(device, oob_perfcntr); + if (ret) + adreno_active_count_put(adreno_dev); + } + + return ret; +} + +static inline void adreno_perfcntr_active_oob_put( + struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + gmu_core_dev_oob_clear(device, oob_perfcntr); + adreno_active_count_put(adreno_dev); +} + +/** + * adreno_wait_for_halt_ack - wait for acknowlegement for a bus halt request + * @ack_reg: register offset to wait for acknowledge + * @mask: A mask value to wait for + * + * Return: 0 on success or -ETIMEDOUT if the request timed out + */ +static inline int adreno_wait_for_halt_ack(struct kgsl_device *device, + int ack_reg, unsigned int mask) +{ + u32 val; + int ret = kgsl_regmap_read_poll_timeout(&device->regmap, ack_reg, + val, (val & mask) == mask, 100, 100 * 1000); + + if (ret) + dev_err(device->dev, + "GBIF/VBIF Halt ack timeout: reg=%08x mask=%08x status=%08x\n", + ack_reg, mask, val); + + return ret; +} + +/** + * adreno_move_preempt_state - Update the preemption state + * @adreno_dev: An Adreno GPU device handle + * @old: The current state of the preemption + * @new: The new state of the preemption + * + * Return: True if the state was updated or false if not + */ +static inline bool adreno_move_preempt_state(struct adreno_device *adreno_dev, + enum adreno_preempt_states old, enum adreno_preempt_states new) +{ + return (atomic_cmpxchg(&adreno_dev->preempt.state, old, new) == old); +} + +/** + * adreno_reg_offset_init - Helper function to initialize reg_offsets + * @reg_offsets: Pointer to an array of register offsets + * + * Helper function to setup register_offsets for a target. Go through + * and set ADRENO_REG_UNUSED for all unused entries in the list. + */ +static inline void adreno_reg_offset_init(u32 *reg_offsets) +{ + int i; + + /* + * Initialize uninitialzed gpu registers, only needs to be done once. + * Make all offsets that are not initialized to ADRENO_REG_UNUSED + */ + for (i = 0; i < ADRENO_REG_REGISTER_MAX; i++) { + if (!reg_offsets[i]) + reg_offsets[i] = ADRENO_REG_UNUSED; + } +} + +static inline u32 adreno_get_level(struct kgsl_context *context) +{ + u32 level; + + if (kgsl_context_is_lpac(context)) + return KGSL_LPAC_RB_ID; + + level = context->priority / KGSL_PRIORITY_MAX_RB_LEVELS; + + return min_t(u32, level, KGSL_PRIORITY_MAX_RB_LEVELS - 1); +} + + +/** + * adreno_get_firwmare - Load firmware into a adreno_firmware struct + * @adreno_dev: An Adreno GPU device handle + * @fwfile: Firmware file to load + * @firmware: A &struct adreno_firmware container for the firmware. + * + * Load the specified firmware file into the memdesc in &struct adreno_firmware + * and get the size and version from the data. + * + * Return: 0 on success or negative on failure + */ +int adreno_get_firmware(struct adreno_device *adreno_dev, + const char *fwfile, struct adreno_firmware *firmware); +/** + * adreno_zap_shader_load - Helper function for loading the zap shader + * adreno_dev: A handle to an Adreno GPU device + * name: Name of the zap shader to load + * + * A target indepedent helper function for loading the zap shader. + * + * Return: 0 on success or negative on failure. + */ +int adreno_zap_shader_load(struct adreno_device *adreno_dev, + const char *name); + +/** + * adreno_irq_callbacks - Helper function to handle IRQ callbacks + * @adreno_dev: Adreno GPU device handle + * @funcs: List of callback functions + * @status: Interrupt status + * + * Walk the bits in the interrupt status and call any applicable callbacks. + * Return: IRQ_HANDLED if one or more interrupt callbacks were called. + */ +irqreturn_t adreno_irq_callbacks(struct adreno_device *adreno_dev, + const struct adreno_irq_funcs *funcs, u32 status); + + +/** + * adreno_device_probe - Generic adreno device probe function + * @pdev: Pointer to the platform device + * @adreno_dev: Adreno GPU device handle + * + * Do the generic setup for the Adreno device. Called from the target specific + * probe functions. + * + * Return: 0 on success or negative on failure + */ +int adreno_device_probe(struct platform_device *pdev, + struct adreno_device *adreno_dev); + +/** + * adreno_power_cycle - Suspend and resume the device + * @adreno_dev: Pointer to the adreno device + * @callback: Function that needs to be executed + * @priv: Argument to be passed to the callback + * + * Certain properties that can be set via sysfs need to power + * cycle the device to take effect. This function suspends + * the device, executes the callback, and resumes the device. + * + * Return: 0 on success or negative on failure + */ +int adreno_power_cycle(struct adreno_device *adreno_dev, + void (*callback)(struct adreno_device *adreno_dev, void *priv), + void *priv); + +/** + * adreno_power_cycle_bool - Power cycle the device to change device setting + * @adreno_dev: Pointer to the adreno device + * @flag: Flag that needs to be set + * @val: The value flag should be set to + * + * Certain properties that can be set via sysfs need to power cycle the device + * to take effect. This function suspends the device, sets the flag, and + * resumes the device. + * + * Return: 0 on success or negative on failure + */ +int adreno_power_cycle_bool(struct adreno_device *adreno_dev, + bool *flag, bool val); + +/** + * adreno_power_cycle_u32 - Power cycle the device to change device setting + * @adreno_dev: Pointer to the adreno device + * @flag: Flag that needs to be set + * @val: The value flag should be set to + * + * Certain properties that can be set via sysfs need to power cycle the device + * to take effect. This function suspends the device, sets the flag, and + * resumes the device. + * + * Return: 0 on success or negative on failure + */ +int adreno_power_cycle_u32(struct adreno_device *adreno_dev, + u32 *flag, u32 val); + +/** + * adreno_set_active_ctxs_null - Give up active context refcount + * @adreno_dev: Adreno GPU device handle + * + * This puts back the reference for that last active context on + * each ringbuffer when going in and out of slumber. + */ +void adreno_set_active_ctxs_null(struct adreno_device *adreno_dev); + +/** + * adreno_get_bus_counters - Allocate the bus dcvs counters + * @adreno_dev: Adreno GPU device handle + * + * This function allocates the various gpu counters to measure + * gpu bus usage for bus dcvs + */ +void adreno_get_bus_counters(struct adreno_device *adreno_dev); + +/** + * adreno_suspend_context - Make sure device is idle + * @device: Pointer to the kgsl device + * + * This function processes the profiling results and checks if the + * device is idle so that it can be turned off safely + * + * Return: 0 on success or negative error on failure + */ +int adreno_suspend_context(struct kgsl_device *device); + +/* + * adreno_profile_submit_time - Populate profiling buffer with timestamps + * @time: Container for the statistics + * + * Populate the draw object user profiling buffer with the timestamps + * recored in the adreno_submit_time structure at the time of draw object + * submission. + */ +void adreno_profile_submit_time(struct adreno_submit_time *time); + +void adreno_preemption_timer(struct timer_list *t); + +/** + * adreno_create_profile_buffer - Create a buffer to store profiling data + * @adreno_dev: Adreno GPU device handle + */ +void adreno_create_profile_buffer(struct adreno_device *adreno_dev); + +/** + * adreno_isidle - return true if the hardware is idle + * @adreno_dev: Adreno GPU device handle + * + * Return: True if the hardware is idle + */ +bool adreno_isidle(struct adreno_device *adreno_dev); + +/** + * adreno_allocate_global - Helper function to allocate a global GPU object + * @device: A GPU device handle + * @memdesc: Pointer to a &struct kgsl_memdesc pointer + * @size: Size of the allocation in bytes + * @padding: Amount of extra adding to add to the VA allocation + * @flags: Control flags for the allocation + * @priv: Internal flags for the allocation + * @name: Name of the allocation (for the debugfs file) + * + * Allocate a global object if it hasn't already been alllocated and put it in + * the pointer pointed to by @memdesc. + * Return: 0 on success or negative on error + */ +static inline int adreno_allocate_global(struct kgsl_device *device, + struct kgsl_memdesc **memdesc, u64 size, u32 padding, u64 flags, + u32 priv, const char *name) +{ + if (!IS_ERR_OR_NULL(*memdesc)) + return 0; + + *memdesc = kgsl_allocate_global(device, size, padding, flags, priv, name); + return PTR_ERR_OR_ZERO(*memdesc); +} + +static inline void adreno_set_dispatch_ops(struct adreno_device *adreno_dev, + const struct adreno_dispatch_ops *ops) +{ + adreno_dev->dispatch_ops = ops; +} + +#ifdef CONFIG_QCOM_KGSL_FENCE_TRACE +/** + * adreno_fence_trace_array_init - Initialize an always on trace array + * @device: A GPU device handle + * + * Register an always-on trace array to for fence timeout debugging + */ +void adreno_fence_trace_array_init(struct kgsl_device *device); +#else +static inline void adreno_fence_trace_array_init(struct kgsl_device *device) {} +#endif + +/* + * adreno_drawobj_set_constraint - Set a power constraint + * @device: Pointer to a KGSL device structure + * @drawobj: Draw object for which constraint is to be set + * + * Set the power constraint if requested by this context + */ +void adreno_drawobj_set_constraint(struct kgsl_device *device, + struct kgsl_drawobj *drawobj); + +/** + * adreno_get_gpu_model - Gets gpu model name from device tree (or) chipid + * @device: A GPU device handle + * + * Return: GPU model name string + */ +const char *adreno_get_gpu_model(struct kgsl_device *device); + +int adreno_verify_cmdobj(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, struct kgsl_drawobj *drawobj[], + uint32_t count); + +/** + * adreno_mark_for_coldboot - Set a flag to coldboot gpu in the slumber exit + * @adreno_dev: Adreno device handle + * + */ +void adreno_mark_for_coldboot(struct adreno_device *adreno_dev); + +/** + * adreno_smmu_is_stalled() - Check whether smmu is stalled or not + * @device: Pointer to adreno device + * + * Return - True if smmu is stalled or false otherwise + */ +bool adreno_smmu_is_stalled(struct adreno_device *adreno_dev); + +/** + * adreno_get_ahb_timeout_val() - Get the ahb_timeout value + * @adreno_dev: Adreno device handle + * @noc_timeout_us: GPU config NOC timeout value in usec + * + * Return - AHB timeout value to be programmed in AHB CNTL registers + */ +u32 adreno_get_ahb_timeout_val(struct adreno_device *adreno_dev, u32 noc_timeout_us); + +/** + * adreno_llcc_slice_deactivate - Deactivate GPU and GPUHTW llcc slices + * @adreno_dev: Adreno device handle + */ +static inline void adreno_llcc_slice_deactivate(struct adreno_device *adreno_dev) +{ + if (adreno_dev->gpu_llc_slice_enable && !IS_ERR_OR_NULL(adreno_dev->gpu_llc_slice)) + llcc_slice_deactivate(adreno_dev->gpu_llc_slice); + + if (adreno_dev->gpuhtw_llc_slice_enable && !IS_ERR_OR_NULL(adreno_dev->gpuhtw_llc_slice)) + llcc_slice_deactivate(adreno_dev->gpuhtw_llc_slice); +} + +/** + * adreno_irq_free - Free an interrupt allocated for GPU + * @adreno_dev: Adreno device handle + */ +static inline void adreno_irq_free(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!(adreno_dev->irq_mask || device->pwrctrl.interrupt_num)) + return; + + devm_free_irq(&device->pdev->dev, device->pwrctrl.interrupt_num, device); + adreno_dev->irq_mask = 0; + device->pwrctrl.interrupt_num = 0; +} +#endif /*__ADRENO_H */ diff --git a/qcom/opensource/graphics-kernel/adreno_a3xx.c b/qcom/opensource/graphics-kernel/adreno_a3xx.c new file mode 100644 index 0000000000..e7b9eab1fb --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a3xx.c @@ -0,0 +1,1514 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "adreno.h" +#include "adreno_cp_parser.h" +#include "adreno_a3xx.h" +#include "adreno_pm4types.h" +#include "adreno_snapshot.h" +#include "adreno_trace.h" + +/* + * Define registers for a3xx that contain addresses used by the + * cp parser logic + */ +const unsigned int a3xx_cp_addr_regs[ADRENO_CP_ADDR_MAX] = { + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_0, + A3XX_VSC_PIPE_DATA_ADDRESS_0), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_0, + A3XX_VSC_PIPE_DATA_LENGTH_0), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_1, + A3XX_VSC_PIPE_DATA_ADDRESS_1), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_1, + A3XX_VSC_PIPE_DATA_LENGTH_1), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_2, + A3XX_VSC_PIPE_DATA_ADDRESS_2), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_2, + A3XX_VSC_PIPE_DATA_LENGTH_2), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_3, + A3XX_VSC_PIPE_DATA_ADDRESS_3), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_3, + A3XX_VSC_PIPE_DATA_LENGTH_3), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_4, + A3XX_VSC_PIPE_DATA_ADDRESS_4), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_4, + A3XX_VSC_PIPE_DATA_LENGTH_4), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_5, + A3XX_VSC_PIPE_DATA_ADDRESS_5), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_5, + A3XX_VSC_PIPE_DATA_LENGTH_5), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_6, + A3XX_VSC_PIPE_DATA_ADDRESS_6), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_6, + A3XX_VSC_PIPE_DATA_LENGTH_6), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_7, + A3XX_VSC_PIPE_DATA_ADDRESS_7), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_7, + A3XX_VSC_PIPE_DATA_LENGTH_7), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_0, + A3XX_VFD_FETCH_INSTR_1_0), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_1, + A3XX_VFD_FETCH_INSTR_1_1), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_2, + A3XX_VFD_FETCH_INSTR_1_2), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_3, + A3XX_VFD_FETCH_INSTR_1_3), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_4, + A3XX_VFD_FETCH_INSTR_1_4), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_5, + A3XX_VFD_FETCH_INSTR_1_5), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_6, + A3XX_VFD_FETCH_INSTR_1_6), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_7, + A3XX_VFD_FETCH_INSTR_1_7), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_8, + A3XX_VFD_FETCH_INSTR_1_8), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_9, + A3XX_VFD_FETCH_INSTR_1_9), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_10, + A3XX_VFD_FETCH_INSTR_1_A), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_11, + A3XX_VFD_FETCH_INSTR_1_B), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_12, + A3XX_VFD_FETCH_INSTR_1_C), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_13, + A3XX_VFD_FETCH_INSTR_1_D), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_14, + A3XX_VFD_FETCH_INSTR_1_E), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_15, + A3XX_VFD_FETCH_INSTR_1_F), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_VSC_SIZE_ADDRESS, + A3XX_VSC_SIZE_ADDRESS), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_VS_PVT_MEM_ADDR, + A3XX_SP_VS_PVT_MEM_ADDR_REG), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_FS_PVT_MEM_ADDR, + A3XX_SP_FS_PVT_MEM_ADDR_REG), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_VS_OBJ_START_REG, + A3XX_SP_VS_OBJ_START_REG), + ADRENO_REG_DEFINE(ADRENO_CP_ADDR_SP_FS_OBJ_START_REG, + A3XX_SP_FS_OBJ_START_REG), +}; + +static const unsigned int _a3xx_pwron_fixup_fs_instructions[] = { + 0x00000000, 0x302CC300, 0x00000000, 0x302CC304, + 0x00000000, 0x302CC308, 0x00000000, 0x302CC30C, + 0x00000000, 0x302CC310, 0x00000000, 0x302CC314, + 0x00000000, 0x302CC318, 0x00000000, 0x302CC31C, + 0x00000000, 0x302CC320, 0x00000000, 0x302CC324, + 0x00000000, 0x302CC328, 0x00000000, 0x302CC32C, + 0x00000000, 0x302CC330, 0x00000000, 0x302CC334, + 0x00000000, 0x302CC338, 0x00000000, 0x302CC33C, + 0x00000000, 0x00000400, 0x00020000, 0x63808003, + 0x00060004, 0x63828007, 0x000A0008, 0x6384800B, + 0x000E000C, 0x6386800F, 0x00120010, 0x63888013, + 0x00160014, 0x638A8017, 0x001A0018, 0x638C801B, + 0x001E001C, 0x638E801F, 0x00220020, 0x63908023, + 0x00260024, 0x63928027, 0x002A0028, 0x6394802B, + 0x002E002C, 0x6396802F, 0x00320030, 0x63988033, + 0x00360034, 0x639A8037, 0x003A0038, 0x639C803B, + 0x003E003C, 0x639E803F, 0x00000000, 0x00000400, + 0x00000003, 0x80D60003, 0x00000007, 0x80D60007, + 0x0000000B, 0x80D6000B, 0x0000000F, 0x80D6000F, + 0x00000013, 0x80D60013, 0x00000017, 0x80D60017, + 0x0000001B, 0x80D6001B, 0x0000001F, 0x80D6001F, + 0x00000023, 0x80D60023, 0x00000027, 0x80D60027, + 0x0000002B, 0x80D6002B, 0x0000002F, 0x80D6002F, + 0x00000033, 0x80D60033, 0x00000037, 0x80D60037, + 0x0000003B, 0x80D6003B, 0x0000003F, 0x80D6003F, + 0x00000000, 0x03000000, 0x00000000, 0x00000000, +}; + +#define A3XX_INT_MASK \ + ((1 << A3XX_INT_RBBM_AHB_ERROR) | \ + (1 << A3XX_INT_RBBM_ATB_BUS_OVERFLOW) | \ + (1 << A3XX_INT_CP_T0_PACKET_IN_IB) | \ + (1 << A3XX_INT_CP_OPCODE_ERROR) | \ + (1 << A3XX_INT_CP_RESERVED_BIT_ERROR) | \ + (1 << A3XX_INT_CP_HW_FAULT) | \ + (1 << A3XX_INT_CP_IB1_INT) | \ + (1 << A3XX_INT_CP_IB2_INT) | \ + (1 << A3XX_INT_CP_RB_INT) | \ + (1 << A3XX_INT_CACHE_FLUSH_TS) | \ + (1 << A3XX_INT_CP_REG_PROTECT_FAULT) | \ + (1 << A3XX_INT_CP_AHB_ERROR_HALT) | \ + (1 << A3XX_INT_UCHE_OOB_ACCESS)) + +/** + * _a3xx_pwron_fixup() - Initialize a special command buffer to run a + * post-power collapse shader workaround + * @adreno_dev: Pointer to a adreno_device struct + * + * Some targets require a special workaround shader to be executed after + * power-collapse. Construct the IB once at init time and keep it + * handy + * + * Returns: 0 on success or negative on error + */ +static int _a3xx_pwron_fixup(struct adreno_device *adreno_dev) +{ + unsigned int *cmds; + int count = ARRAY_SIZE(_a3xx_pwron_fixup_fs_instructions); + + /* Return if the fixup is already in place */ + if (test_bit(ADRENO_DEVICE_PWRON_FIXUP, &adreno_dev->priv)) + return 0; + + adreno_dev->pwron_fixup = kgsl_allocate_global(KGSL_DEVICE(adreno_dev), + PAGE_SIZE, 0, KGSL_MEMFLAGS_GPUREADONLY, 0, "pwron_fixup"); + + if (IS_ERR(adreno_dev->pwron_fixup)) + return PTR_ERR(adreno_dev->pwron_fixup); + + cmds = adreno_dev->pwron_fixup->hostptr; + + *cmds++ = cp_type0_packet(A3XX_UCHE_CACHE_INVALIDATE0_REG, 2); + *cmds++ = 0x00000000; + *cmds++ = 0x90000000; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_REG_RMW, 3); + *cmds++ = A3XX_RBBM_CLOCK_CTL; + *cmds++ = 0xFFFCFFFF; + *cmds++ = 0x00010000; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_0_REG, 1); + *cmds++ = 0x1E000150; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_SET_CONSTANT, 2); + *cmds++ = CP_REG(A3XX_HLSQ_CONTROL_0_REG); + *cmds++ = 0x1E000150; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_0_REG, 1); + *cmds++ = 0x1E000150; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_1_REG, 1); + *cmds++ = 0x00000040; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_2_REG, 1); + *cmds++ = 0x80000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_3_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_VS_CONTROL_REG, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_HLSQ_FS_CONTROL_REG, 1); + *cmds++ = 0x0D001002; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONST_VSPRESV_RANGE_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONST_FSPRESV_RANGE_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_0_REG, 1); + *cmds++ = 0x00401101; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_1_REG, 1); + *cmds++ = 0x00000400; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_2_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_3_REG, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_4_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_5_REG, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_NDRANGE_6_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_CONTROL_0_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_CONTROL_1_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_KERNEL_CONST_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_KERNEL_GROUP_X_REG, 1); + *cmds++ = 0x00000010; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_KERNEL_GROUP_Y_REG, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_KERNEL_GROUP_Z_REG, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_WG_OFFSET_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_SP_CTRL_REG, 1); + *cmds++ = 0x00040000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_CTRL_REG0, 1); + *cmds++ = 0x0000000A; + *cmds++ = cp_type0_packet(A3XX_SP_VS_CTRL_REG1, 1); + *cmds++ = 0x00000001; + *cmds++ = cp_type0_packet(A3XX_SP_VS_PARAM_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_4, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_5, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_6, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OUT_REG_7, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_VPC_DST_REG_0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_VPC_DST_REG_1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_VPC_DST_REG_2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_VPC_DST_REG_3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OBJ_OFFSET_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_OBJ_START_REG, 1); + *cmds++ = 0x00000004; + *cmds++ = cp_type0_packet(A3XX_SP_VS_PVT_MEM_PARAM_REG, 1); + *cmds++ = 0x04008001; + *cmds++ = cp_type0_packet(A3XX_SP_VS_PVT_MEM_ADDR_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_PVT_MEM_SIZE_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_VS_LENGTH_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_CTRL_REG0, 1); + *cmds++ = 0x0DB0400A; + *cmds++ = cp_type0_packet(A3XX_SP_FS_CTRL_REG1, 1); + *cmds++ = 0x00300402; + *cmds++ = cp_type0_packet(A3XX_SP_FS_OBJ_OFFSET_REG, 1); + *cmds++ = 0x00010000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_OBJ_START_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_PVT_MEM_PARAM_REG, 1); + *cmds++ = 0x04008001; + *cmds++ = cp_type0_packet(A3XX_SP_FS_PVT_MEM_ADDR_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_PVT_MEM_SIZE_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_FLAT_SHAD_MODE_REG_0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_FLAT_SHAD_MODE_REG_1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_OUTPUT_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_MRT_REG_0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_MRT_REG_1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_MRT_REG_2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_MRT_REG_3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_IMAGE_OUTPUT_REG_0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_IMAGE_OUTPUT_REG_1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_IMAGE_OUTPUT_REG_2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_IMAGE_OUTPUT_REG_3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_SP_FS_LENGTH_REG, 1); + *cmds++ = 0x0000000D; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_CLIP_CNTL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_GB_CLIP_ADJ, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_XOFFSET, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_XSCALE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_YOFFSET, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_YSCALE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_ZOFFSET, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_VPORT_ZSCALE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X4, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y4, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z4, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W4, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_X5, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Y5, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_Z5, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_CL_USER_PLANE_W5, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SU_POINT_MINMAX, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SU_POINT_SIZE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SU_POLY_OFFSET_OFFSET, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SU_POLY_OFFSET_SCALE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SU_MODE_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SC_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SC_SCREEN_SCISSOR_TL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SC_SCREEN_SCISSOR_BR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SC_WINDOW_SCISSOR_BR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_SC_WINDOW_SCISSOR_TL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_TSE_DEBUG_ECO, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_PERFCOUNTER0_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_PERFCOUNTER1_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_PERFCOUNTER2_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_GRAS_PERFCOUNTER3_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MODE_CONTROL, 1); + *cmds++ = 0x00008000; + *cmds++ = cp_type0_packet(A3XX_RB_RENDER_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MSAA_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_ALPHA_REFERENCE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_CONTROL0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_CONTROL1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_CONTROL2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_CONTROL3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_INFO0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_INFO1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_INFO2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_INFO3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_BASE0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_BASE1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_BASE2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BUF_BASE3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BLEND_CONTROL0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BLEND_CONTROL1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BLEND_CONTROL2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_MRT_BLEND_CONTROL3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_BLEND_RED, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_BLEND_GREEN, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_BLEND_BLUE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_BLEND_ALPHA, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_CLEAR_COLOR_DW0, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_CLEAR_COLOR_DW1, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_CLEAR_COLOR_DW2, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_CLEAR_COLOR_DW3, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_COPY_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_COPY_DEST_BASE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_COPY_DEST_PITCH, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_COPY_DEST_INFO, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_DEPTH_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_DEPTH_CLEAR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_DEPTH_BUF_INFO, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_DEPTH_BUF_PITCH, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_CLEAR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_BUF_INFO, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_BUF_PITCH, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_REF_MASK, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_STENCIL_REF_MASK_BF, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_LRZ_VSC_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_WINDOW_OFFSET, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_SAMPLE_COUNT_CONTROL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_SAMPLE_COUNT_ADDR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_Z_CLAMP_MIN, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_Z_CLAMP_MAX, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_GMEM_BASE_ADDR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_DEBUG_ECO_CONTROLS_ADDR, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_PERFCOUNTER0_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_PERFCOUNTER1_SELECT, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_RB_FRAME_BUFFER_DIMENSION, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_LOAD_STATE, 4); + *cmds++ = (1 << CP_LOADSTATE_DSTOFFSET_SHIFT) | + (0 << CP_LOADSTATE_STATESRC_SHIFT) | + (6 << CP_LOADSTATE_STATEBLOCKID_SHIFT) | + (1 << CP_LOADSTATE_NUMOFUNITS_SHIFT); + *cmds++ = (1 << CP_LOADSTATE_STATETYPE_SHIFT) | + (0 << CP_LOADSTATE_EXTSRCADDR_SHIFT); + *cmds++ = 0x00400000; + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_LOAD_STATE, 4); + *cmds++ = (2 << CP_LOADSTATE_DSTOFFSET_SHIFT) | + (6 << CP_LOADSTATE_STATEBLOCKID_SHIFT) | + (1 << CP_LOADSTATE_NUMOFUNITS_SHIFT); + *cmds++ = (1 << CP_LOADSTATE_STATETYPE_SHIFT); + *cmds++ = 0x00400220; + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_LOAD_STATE, 4); + *cmds++ = (6 << CP_LOADSTATE_STATEBLOCKID_SHIFT) | + (1 << CP_LOADSTATE_NUMOFUNITS_SHIFT); + *cmds++ = (1 << CP_LOADSTATE_STATETYPE_SHIFT); + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_LOAD_STATE, 2 + count); + *cmds++ = (6 << CP_LOADSTATE_STATEBLOCKID_SHIFT) | + (13 << CP_LOADSTATE_NUMOFUNITS_SHIFT); + *cmds++ = 0x00000000; + + memcpy(cmds, _a3xx_pwron_fixup_fs_instructions, count << 2); + + cmds += count; + + *cmds++ = cp_type3_packet(CP_EXEC_CL, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CL_CONTROL_0_REG, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type0_packet(A3XX_HLSQ_CONTROL_0_REG, 1); + *cmds++ = 0x1E000150; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_SET_CONSTANT, 2); + *cmds++ = CP_REG(A3XX_HLSQ_CONTROL_0_REG); + *cmds++ = 0x1E000050; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_REG_RMW, 3); + *cmds++ = A3XX_RBBM_CLOCK_CTL; + *cmds++ = 0xFFFCFFFF; + *cmds++ = 0x00000000; + *cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + *cmds++ = 0x00000000; + + /* + * Remember the number of dwords in the command buffer for when we + * program the indirect buffer call in the ringbuffer + */ + adreno_dev->pwron_fixup_dwords = + (cmds - (unsigned int *) adreno_dev->pwron_fixup->hostptr); + + /* Mark the flag in ->priv to show that we have the fix */ + set_bit(ADRENO_DEVICE_PWRON_FIXUP, &adreno_dev->priv); + return 0; +} + +static int a3xx_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore) +{ + struct adreno_device *adreno_dev; + struct kgsl_device *device; + int ret; + + adreno_dev = (struct adreno_device *) + of_device_get_match_data(&pdev->dev); + + memset(adreno_dev, 0, sizeof(*adreno_dev)); + + adreno_dev->gpucore = gpucore; + adreno_dev->chipid = chipid; + + adreno_reg_offset_init(gpucore->gpudev->reg_offsets); + + + device = KGSL_DEVICE(adreno_dev); + + timer_setup(&device->idle_timer, kgsl_timer, 0); + + INIT_WORK(&device->idle_check_ws, kgsl_idle_check); + + adreno_dev->irq_mask = A3XX_INT_MASK; + + ret = adreno_device_probe(pdev, adreno_dev); + if (ret) + return ret; + + a3xx_coresight_init(adreno_dev); + + return adreno_dispatcher_init(adreno_dev); +} + +static int a3xx_send_me_init(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int *cmds; + int ret; + + cmds = adreno_ringbuffer_allocspace(rb, 18); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + *cmds++ = cp_type3_packet(CP_ME_INIT, 17); + + *cmds++ = 0x000003f7; + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + *cmds++ = 0x00000080; + *cmds++ = 0x00000100; + *cmds++ = 0x00000180; + *cmds++ = 0x00006600; + *cmds++ = 0x00000150; + *cmds++ = 0x0000014e; + *cmds++ = 0x00000154; + *cmds++ = 0x00000001; + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + + /* Enable protected mode registers for A3XX */ + *cmds++ = 0x20000000; + + *cmds++ = 0x00000000; + *cmds++ = 0x00000000; + + /* Submit the command to the ringbuffer */ + kgsl_pwrscale_busy(device); + kgsl_regwrite(device, A3XX_CP_RB_WPTR, rb->_wptr); + rb->wptr = rb->_wptr; + + ret = adreno_spin_idle(adreno_dev, 2000); + if (ret) { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + dev_err(device->dev, "CP initialization failed to idle\n"); + kgsl_device_snapshot(device, NULL, NULL, false); + } + + return ret; +} + +static void a3xx_microcode_load(struct adreno_device *adreno_dev); + +static int a3xx_rb_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_ringbuffer *rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev); + + memset(rb->buffer_desc->hostptr, 0xaa, KGSL_RB_SIZE); + rb->wptr = 0; + rb->_wptr = 0; + rb->wptr_preempt_end = ~0; + + /* + * The size of the ringbuffer in the hardware is the log2 + * representation of the size in quadwords (sizedwords / 2). + * Also disable the host RPTR shadow register as it might be unreliable + * in certain circumstances. + */ + + kgsl_regwrite(device, A3XX_CP_RB_CNTL, + (ilog2(KGSL_RB_DWORDS >> 1) & 0x3F) | + (1 << 27)); + + kgsl_regwrite(device, A3XX_CP_RB_BASE, rb->buffer_desc->gpuaddr); + + a3xx_microcode_load(adreno_dev); + + /* clear ME_HALT to start micro engine */ + kgsl_regwrite(device, A3XX_CP_ME_CNTL, 0); + + return a3xx_send_me_init(adreno_dev, rb); +} + +/* + * a3xx soft fault detection + * + * a3xx targets do not have hardware fault detection so we need to do it the old + * fashioned way by periodically reading a set of registers and counters and + * checking that they are advancing. There are 6 registers and four 64 bit + * counters that we keep an eye on. + */ + +#define A3XX_SOFT_FAULT_DETECT_REGS 6 +#define A3XX_SOFT_FAULT_DETECT_COUNTERS 4 +#define A3XX_SOFT_FAULT_DETECT_COUNT \ + (A3XX_SOFT_FAULT_DETECT_REGS + (A3XX_SOFT_FAULT_DETECT_COUNTERS * 2)) + +static bool a3xx_soft_fault_detect_isidle(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 reg; + + if (kgsl_state_is_awake(device)) { + if (!adreno_rb_empty(adreno_dev->cur_rb)) + return false; + + /* only check rbbm status to determine if GPU is idle */ + kgsl_regread(device, A3XX_RBBM_STATUS, ®); + + if (reg & 0x7ffffffe) + return false; + } + + memset(adreno_dev->soft_ft_vals, 0, A3XX_SOFT_FAULT_DETECT_COUNT << 2); + return true; +} + +/* Read the fault detect registers and compare them to the stored version */ +static int a3xx_soft_fault_detect_read_compare(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_ringbuffer *rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev); + int i, ret = 0; + unsigned int ts; + + if (!test_bit(ADRENO_DEVICE_SOFT_FAULT_DETECT, &adreno_dev->priv)) + return 1; + + /* Check to see if the device is idle - if so report no hang */ + if (a3xx_soft_fault_detect_isidle(adreno_dev)) + ret = 1; + + for (i = 0; i < A3XX_SOFT_FAULT_DETECT_COUNT; i++) { + unsigned int val; + + if (!adreno_dev->soft_ft_regs[i]) + continue; + + kgsl_regread(device, adreno_dev->soft_ft_regs[i], &val); + if (val != adreno_dev->soft_ft_vals[i]) + ret = 1; + adreno_dev->soft_ft_vals[i] = val; + } + + if (!adreno_rb_readtimestamp(adreno_dev, adreno_dev->cur_rb, + KGSL_TIMESTAMP_RETIRED, &ts)) { + if (ts != rb->fault_detect_ts) + ret = 1; + + rb->fault_detect_ts = ts; + } + + return ret; +} + +/* + * This is called on a regular basis while cmdobjs are inflight. Fault + * detection registers are read and compared to the existing values - if they + * changed then the GPU is still running. If they are the same between + * subsequent calls then the GPU may have faulted + */ +static void a3xx_soft_fault_timer(struct timer_list *t) +{ + struct adreno_dispatcher *dispatcher = from_timer(dispatcher, + t, fault_timer); + struct adreno_device *adreno_dev = container_of(dispatcher, + struct adreno_device, dispatcher); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* Leave if the user decided to turn off fast hang detection */ + if (!adreno_soft_fault_detect(adreno_dev)) + return; + + if (adreno_gpu_fault(adreno_dev)) { + adreno_dispatcher_schedule(device); + return; + } + + /* + * Read the fault registers - if it returns 0 then they haven't changed + * so mark the dispatcher as faulted and schedule the work loop. + */ + + if (!a3xx_soft_fault_detect_read_compare(adreno_dev)) + adreno_dispatcher_fault(adreno_dev, ADRENO_SOFT_FAULT); + else if (dispatcher->inflight > 0) + adreno_dispatcher_start_fault_timer(adreno_dev); +} + +/* + * Start fault detection. The counters are only assigned while fault detection + * is running so that they can be used for other purposes if fault detection is + * disabled + */ +static void a3xx_soft_fault_detect_start(struct adreno_device *adreno_dev) +{ + u32 *regs = &adreno_dev->soft_ft_regs[A3XX_SOFT_FAULT_DETECT_COUNTERS]; + int ret = 0; + + if (!test_bit(ADRENO_DEVICE_SOFT_FAULT_DETECT, &adreno_dev->priv)) + return; + + if (adreno_dev->fast_hang_detect == 1) + return; + + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_SP, SP_ALU_ACTIVE_CYCLES, + ®s[0], ®s[1]); + + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_SP, SP0_ICL1_MISSES, + ®s[2], ®s[3]); + + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_SP, SP_FS_CFLOW_INSTRUCTIONS, + ®s[4], ®s[5]); + + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_TSE, TSE_INPUT_PRIM_NUM, + ®s[6], ®s[7]); + + WARN(ret, "Unable to allocate one or more fault detect counters\n"); + adreno_dev->fast_hang_detect = 1; +} + +/* Helper function to put back a counter */ +static void put_counter(struct adreno_device *adreno_dev, + int group, int countable, u32 *lo, u32 *hi) +{ + adreno_perfcounter_put(adreno_dev, group, countable, + PERFCOUNTER_FLAG_KERNEL); + + *lo = 0; + *hi = 0; +} + +/* Stop fault detection and return the counters */ +static void a3xx_soft_fault_detect_stop(struct adreno_device *adreno_dev) +{ + u32 *regs = &adreno_dev->soft_ft_regs[A3XX_SOFT_FAULT_DETECT_COUNTERS]; + + if (!test_bit(ADRENO_DEVICE_SOFT_FAULT_DETECT, &adreno_dev->priv)) + return; + + if (!adreno_dev->fast_hang_detect) + return; + + put_counter(adreno_dev, KGSL_PERFCOUNTER_GROUP_SP, SP_ALU_ACTIVE_CYCLES, + ®s[0], ®s[1]); + + put_counter(adreno_dev, KGSL_PERFCOUNTER_GROUP_SP, SP0_ICL1_MISSES, + ®s[2], ®s[3]); + + put_counter(adreno_dev, KGSL_PERFCOUNTER_GROUP_SP, + SP_FS_CFLOW_INSTRUCTIONS, ®s[4], ®s[5]); + + put_counter(adreno_dev, KGSL_PERFCOUNTER_GROUP_TSE, TSE_INPUT_PRIM_NUM, + ®s[6], ®s[7]); + + adreno_dev->fast_hang_detect = 0; +} + +/* Initialize the registers and set up the data structures */ +static void a3xx_soft_fault_detect_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_SOFT_FAULT_DETECT)) + return; + + /* Disable the fast hang detect bit until we know its a go */ + adreno_dev->fast_hang_detect = 0; + + adreno_dev->soft_ft_regs = devm_kcalloc(&device->pdev->dev, + A3XX_SOFT_FAULT_DETECT_COUNT, sizeof(u32), GFP_KERNEL); + + adreno_dev->soft_ft_vals = devm_kcalloc(&device->pdev->dev, + A3XX_SOFT_FAULT_DETECT_COUNT, sizeof(u32), GFP_KERNEL); + + if (!adreno_dev->soft_ft_regs || !adreno_dev->soft_ft_vals) + return; + + adreno_dev->soft_ft_count = A3XX_SOFT_FAULT_DETECT_COUNT; + + adreno_dev->soft_ft_regs[0] = A3XX_RBBM_STATUS; + adreno_dev->soft_ft_regs[1] = A3XX_CP_RB_RPTR; + adreno_dev->soft_ft_regs[2] = A3XX_CP_IB1_BASE; + adreno_dev->soft_ft_regs[3] = A3XX_CP_IB1_BUFSZ; + adreno_dev->soft_ft_regs[4] = A3XX_CP_IB2_BASE; + adreno_dev->soft_ft_regs[5] = A3XX_CP_IB2_BUFSZ; + + set_bit(ADRENO_DEVICE_SOFT_FAULT_DETECT, &adreno_dev->priv); + + a3xx_soft_fault_detect_start(adreno_dev); +} + +static void a3xx_remove(struct adreno_device *adreno_dev) +{ + a3xx_soft_fault_detect_stop(adreno_dev); +} + +static int a3xx_microcode_read(struct adreno_device *adreno_dev); + +/* + * a3xx_init() - Initialize gpu specific data + * @adreno_dev: Pointer to adreno device + */ +static int a3xx_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_iommu *iommu = KGSL_IOMMU(device); + struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; + int ret; + + /* + * Set up the a3xx only soft fault timer before heading into the generic + * dispatcher setup + */ + if (ADRENO_FEATURE(adreno_dev, ADRENO_SOFT_FAULT_DETECT)) + timer_setup(&dispatcher->fault_timer, a3xx_soft_fault_timer, 0); + + ret = a3xx_ringbuffer_init(adreno_dev); + if (ret) + return ret; + + ret = a3xx_microcode_read(adreno_dev); + if (ret) + return ret; + + _a3xx_pwron_fixup(adreno_dev); + + ret = adreno_allocate_global(device, &iommu->setstate, PAGE_SIZE, + 0, KGSL_MEMFLAGS_GPUREADONLY, 0, "setstate"); + + if (!ret) + kgsl_sharedmem_writel(iommu->setstate, + KGSL_IOMMU_SETSTATE_NOP_OFFSET, + cp_type3_packet(CP_NOP, 1)); + + kgsl_mmu_set_feature(device, KGSL_MMU_NEED_GUARD_PAGE); + + /* Put the hardware in a responsive state to set up fault detection*/ + ret = kgsl_pwrctrl_change_state(device, KGSL_STATE_AWARE); + if (ret) + return ret; + + a3xx_soft_fault_detect_init(adreno_dev); + + kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); + return 0; +} + +/* + * a3xx_err_callback() - Call back for a3xx error interrupts + * @adreno_dev: Pointer to device + * @bit: Interrupt bit + */ +static void a3xx_err_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg; + + switch (bit) { + case A3XX_INT_RBBM_AHB_ERROR: { + kgsl_regread(device, A3XX_RBBM_AHB_ERROR_STATUS, ®); + + /* + * Return the word address of the erroring register so that it + * matches the register specification + */ + dev_crit_ratelimited(device->dev, + "RBBM | AHB bus error | %s | addr=%x | ports=%x:%x\n", + reg & (1 << 28) ? "WRITE" : "READ", + (reg & 0xFFFFF) >> 2, + (reg >> 20) & 0x3, + (reg >> 24) & 0xF); + + /* Clear the error */ + kgsl_regwrite(device, A3XX_RBBM_AHB_CMD, (1 << 3)); + break; + } + case A3XX_INT_RBBM_ATB_BUS_OVERFLOW: + dev_crit_ratelimited(device->dev, + "RBBM: ATB bus oveflow\n"); + break; + case A3XX_INT_CP_T0_PACKET_IN_IB: + dev_crit_ratelimited(device->dev, + "ringbuffer TO packet in IB interrupt\n"); + break; + case A3XX_INT_CP_OPCODE_ERROR: + dev_crit_ratelimited(device->dev, + "ringbuffer opcode error interrupt\n"); + break; + case A3XX_INT_CP_RESERVED_BIT_ERROR: + dev_crit_ratelimited(device->dev, + "ringbuffer reserved bit error interrupt\n"); + break; + case A3XX_INT_CP_HW_FAULT: + kgsl_regread(device, A3XX_CP_HW_FAULT, ®); + dev_crit_ratelimited(device->dev, + "CP | Ringbuffer HW fault | status=%x\n", + reg); + break; + case A3XX_INT_CP_REG_PROTECT_FAULT: + kgsl_regread(device, A3XX_CP_PROTECT_STATUS, ®); + dev_crit_ratelimited(device->dev, + "CP | Protected mode error| %s | addr=%x\n", + reg & (1 << 24) ? "WRITE" : "READ", + (reg & 0xFFFFF) >> 2); + break; + case A3XX_INT_CP_AHB_ERROR_HALT: + dev_crit_ratelimited(device->dev, + "ringbuffer AHB error interrupt\n"); + break; + case A3XX_INT_UCHE_OOB_ACCESS: + dev_crit_ratelimited(device->dev, + "UCHE: Out of bounds access\n"); + break; + default: + dev_crit_ratelimited(device->dev, "Unknown interrupt\n"); + } +} + +static const struct adreno_irq_funcs a3xx_irq_funcs[32] = { + ADRENO_IRQ_CALLBACK(NULL), /* 0 - RBBM_GPU_IDLE */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 1 - RBBM_AHB_ERROR */ + ADRENO_IRQ_CALLBACK(NULL), /* 2 - RBBM_REG_TIMEOUT */ + ADRENO_IRQ_CALLBACK(NULL), /* 3 - RBBM_ME_MS_TIMEOUT */ + ADRENO_IRQ_CALLBACK(NULL), /* 4 - RBBM_PFP_MS_TIMEOUT */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 5 - RBBM_ATB_BUS_OVERFLOW */ + ADRENO_IRQ_CALLBACK(NULL), /* 6 - RBBM_VFD_ERROR */ + ADRENO_IRQ_CALLBACK(NULL), /* 7 - CP_SW */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 8 - CP_T0_PACKET_IN_IB */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 9 - CP_OPCODE_ERROR */ + /* 10 - CP_RESERVED_BIT_ERROR */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 11 - CP_HW_FAULT */ + ADRENO_IRQ_CALLBACK(NULL), /* 12 - CP_DMA */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 13 - CP_IB2_INT */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 14 - CP_IB1_INT */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 15 - CP_RB_INT */ + /* 16 - CP_REG_PROTECT_FAULT */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), + ADRENO_IRQ_CALLBACK(NULL), /* 17 - CP_RB_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 18 - CP_VS_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 19 - CP_PS_DONE_TS */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 20 - CP_CACHE_FLUSH_TS */ + /* 21 - CP_AHB_ERROR_FAULT */ + ADRENO_IRQ_CALLBACK(a3xx_err_callback), + ADRENO_IRQ_CALLBACK(NULL), /* 22 - Unused */ + ADRENO_IRQ_CALLBACK(NULL), /* 23 - Unused */ + /* 24 - MISC_HANG_DETECT */ + ADRENO_IRQ_CALLBACK(adreno_hang_int_callback), + ADRENO_IRQ_CALLBACK(a3xx_err_callback), /* 25 - UCHE_OOB_ACCESS */ +}; + +static struct { + u32 reg; + u32 base; + u32 count; +} a3xx_protected_blocks[] = { + /* RBBM */ + { A3XX_CP_PROTECT_REG_0, 0x0018, 0 }, + { A3XX_CP_PROTECT_REG_0 + 1, 0x0020, 2 }, + { A3XX_CP_PROTECT_REG_0 + 2, 0x0033, 0 }, + { A3XX_CP_PROTECT_REG_0 + 3, 0x0042, 0 }, + { A3XX_CP_PROTECT_REG_0 + 4, 0x0050, 4 }, + { A3XX_CP_PROTECT_REG_0 + 5, 0x0063, 0 }, + { A3XX_CP_PROTECT_REG_0 + 6, 0x0100, 4 }, + /* CP */ + { A3XX_CP_PROTECT_REG_0 + 7, 0x01c0, 5 }, + { A3XX_CP_PROTECT_REG_0 + 8, 0x01ec, 1 }, + { A3XX_CP_PROTECT_REG_0 + 9, 0x01f6, 1 }, + { A3XX_CP_PROTECT_REG_0 + 10, 0x01f8, 2 }, + { A3XX_CP_PROTECT_REG_0 + 11, 0x045e, 2 }, + { A3XX_CP_PROTECT_REG_0 + 12, 0x0460, 4 }, + /* RB */ + { A3XX_CP_PROTECT_REG_0 + 13, 0x0cc0, 0 }, + /* VBIF */ + { A3XX_CP_PROTECT_REG_0 + 14, 0x3000, 6 }, + /* + * SMMU + * For A3xx, base offset for smmu region is 0xa000 and length is + * 0x1000 bytes. Offset must be in dword and length of the block + * must be ilog2(dword length). + * 0xa000 >> 2 = 0x2800, ilog2(0x1000 >> 2) = 10. + */ + { A3XX_CP_PROTECT_REG_0 + 15, 0x2800, 10 }, + /* There are no remaining protected mode registers for a3xx */ +}; + +static void a3xx_protect_init(struct kgsl_device *device) +{ + int i; + + kgsl_regwrite(device, A3XX_CP_PROTECT_CTRL, 0x00000007); + + for (i = 0; i < ARRAY_SIZE(a3xx_protected_blocks); i++) { + u32 val = 0x60000000 | + (a3xx_protected_blocks[i].count << 24) | + (a3xx_protected_blocks[i].base << 2); + + kgsl_regwrite(device, a3xx_protected_blocks[i].reg, val); + } +} + +bool a3xx_gx_is_on(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + bool gdsc_on, clk_on; + + clk_on = __clk_is_enabled(pwr->grp_clks[0]); + + gdsc_on = regulator_is_enabled(pwr->gx_gdsc); + + return (gdsc_on & clk_on); +} + +static int a3xx_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_a3xx_core *a3xx_core = to_a3xx_core(adreno_dev); + int ret; + + ret = kgsl_mmu_start(device); + if (ret) + return ret; + + adreno_get_bus_counters(adreno_dev); + adreno_perfcounter_restore(adreno_dev); + + if (adreno_dev->soft_ft_regs) + memset(adreno_dev->soft_ft_regs, 0, + adreno_dev->soft_ft_count << 2); + + /* Set up VBIF registers from the GPU core definition */ + kgsl_regmap_multi_write(&device->regmap, a3xx_core->vbif, + a3xx_core->vbif_count); + + /* Make all blocks contribute to the GPU BUSY perf counter */ + kgsl_regwrite(device, A3XX_RBBM_GPU_BUSY_MASKED, 0xFFFFFFFF); + + /* Tune the hystersis counters for SP and CP idle detection */ + kgsl_regwrite(device, A3XX_RBBM_SP_HYST_CNT, 0x10); + kgsl_regwrite(device, A3XX_RBBM_WAIT_IDLE_CLOCKS_CTL, 0x10); + + /* + * Enable the RBBM error reporting bits. This lets us get + * useful information on failure + */ + + kgsl_regwrite(device, A3XX_RBBM_AHB_CTL0, 0x00000001); + + /* Enable AHB error reporting */ + kgsl_regwrite(device, A3XX_RBBM_AHB_CTL1, 0xA6FFFFFF); + + /* Turn on the power counters */ + kgsl_regwrite(device, A3XX_RBBM_RBBM_CTL, 0x00030000); + + /* + * Turn on hang detection - this spews a lot of useful information + * into the RBBM registers on a hang + */ + kgsl_regwrite(device, A3XX_RBBM_INTERFACE_HANG_INT_CTL, + (1 << 16) | 0xFFF); + + /* Enable 64-byte cacheline size. HW Default is 32-byte (0x000000E0). */ + kgsl_regwrite(device, A3XX_UCHE_CACHE_MODE_CONTROL_REG, 0x00000001); + + /* Enable VFD to access most of the UCHE (7 ways out of 8) */ + kgsl_regwrite(device, A3XX_UCHE_CACHE_WAYS_VFD, 0x07); + + /* Enable Clock gating */ + kgsl_regwrite(device, A3XX_RBBM_CLOCK_CTL, A3XX_RBBM_CLOCK_CTL_DEFAULT); + + /* Turn on protection */ + a3xx_protect_init(device); + + /* Turn on performance counters */ + kgsl_regwrite(device, A3XX_RBBM_PERFCTR_CTL, 0x01); + + kgsl_regwrite(device, A3XX_CP_DEBUG, A3XX_CP_DEBUG_DEFAULT); + + /* CP ROQ queue sizes (bytes) - RB:16, ST:16, IB1:32, IB2:64 */ + kgsl_regwrite(device, A3XX_CP_QUEUE_THRESHOLDS, 0x000E0602); + return 0; +} + +/* Register offset defines for A3XX */ +static unsigned int a3xx_register_offsets[ADRENO_REG_REGISTER_MAX] = { + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE, A3XX_CP_RB_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE_HI, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR, A3XX_CP_RB_RPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_WPTR, A3XX_CP_RB_WPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_CNTL, A3XX_CP_ME_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_CNTL, A3XX_CP_RB_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE, A3XX_CP_IB1_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE_HI, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BUFSZ, A3XX_CP_IB1_BUFSZ), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE, A3XX_CP_IB2_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE_HI, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BUFSZ, A3XX_CP_IB2_BUFSZ), + ADRENO_REG_DEFINE(ADRENO_REG_CP_TIMESTAMP, A3XX_CP_SCRATCH_REG0), + ADRENO_REG_DEFINE(ADRENO_REG_CP_SCRATCH_REG6, A3XX_CP_SCRATCH_REG6), + ADRENO_REG_DEFINE(ADRENO_REG_CP_SCRATCH_REG7, A3XX_CP_SCRATCH_REG7), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PROTECT_REG_0, A3XX_CP_PROTECT_REG_0), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS, A3XX_RBBM_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PERFCTR_PWR_1_LO, + A3XX_RBBM_PERFCTR_PWR_1_LO), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_MASK, A3XX_RBBM_INT_0_MASK), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_CLOCK_CTL, A3XX_RBBM_CLOCK_CTL), + ADRENO_REG_DEFINE(ADRENO_REG_PA_SC_AA_CONFIG, A3XX_PA_SC_AA_CONFIG), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_PM_OVERRIDE2, A3XX_RBBM_PM_OVERRIDE2), + ADRENO_REG_DEFINE(ADRENO_REG_SQ_GPR_MANAGEMENT, A3XX_SQ_GPR_MANAGEMENT), + ADRENO_REG_DEFINE(ADRENO_REG_SQ_INST_STORE_MANAGEMENT, + A3XX_SQ_INST_STORE_MANAGEMENT), + ADRENO_REG_DEFINE(ADRENO_REG_TP0_CHICKEN, A3XX_TP0_CHICKEN), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SW_RESET_CMD, A3XX_RBBM_SW_RESET_CMD), +}; + +static int _load_firmware(struct kgsl_device *device, const char *fwfile, + void **buf, int *len) +{ + const struct firmware *fw = NULL; + int ret; + + ret = request_firmware(&fw, fwfile, &device->pdev->dev); + + if (ret) { + dev_err(&device->pdev->dev, "request_firmware(%s) failed: %d\n", + fwfile, ret); + return ret; + } + + if (!fw) + return -EINVAL; + + *buf = devm_kmemdup(&device->pdev->dev, fw->data, fw->size, GFP_KERNEL); + *len = fw->size; + + release_firmware(fw); + return (*buf) ? 0 : -ENOMEM; +} + +static int a3xx_microcode_read(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_firmware *pm4_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4); + struct adreno_firmware *pfp_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); + const struct adreno_a3xx_core *a3xx_core = to_a3xx_core(adreno_dev); + + if (pm4_fw->fwvirt == NULL) { + int len; + void *ptr; + + int ret = _load_firmware(device, + a3xx_core->pm4fw_name, &ptr, &len); + + if (ret) { + dev_err(device->dev, "Failed to read pm4 ucode %s\n", + a3xx_core->pm4fw_name); + return ret; + } + + /* PM4 size is 3 dword aligned plus 1 dword of version */ + if (len % ((sizeof(uint32_t) * 3)) != sizeof(uint32_t)) { + dev_err(device->dev, + "Bad pm4 microcode size: %d\n", + len); + kfree(ptr); + return -ENOMEM; + } + + pm4_fw->size = len / sizeof(uint32_t); + pm4_fw->fwvirt = ptr; + pm4_fw->version = pm4_fw->fwvirt[1]; + } + + if (pfp_fw->fwvirt == NULL) { + int len; + void *ptr; + + int ret = _load_firmware(device, + a3xx_core->pfpfw_name, &ptr, &len); + if (ret) { + dev_err(device->dev, "Failed to read pfp ucode %s\n", + a3xx_core->pfpfw_name); + return ret; + } + + /* PFP size shold be dword aligned */ + if (len % sizeof(uint32_t) != 0) { + dev_err(device->dev, + "Bad PFP microcode size: %d\n", + len); + kfree(ptr); + return -ENOMEM; + } + + pfp_fw->size = len / sizeof(uint32_t); + pfp_fw->fwvirt = ptr; + pfp_fw->version = pfp_fw->fwvirt[1]; + } + + return 0; +} + +static void a3xx_microcode_load(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + size_t pm4_size = adreno_dev->fw[ADRENO_FW_PM4].size; + size_t pfp_size = adreno_dev->fw[ADRENO_FW_PFP].size; + int i; + + /* load the CP ucode using AHB writes */ + kgsl_regwrite(device, A3XX_CP_ME_RAM_WADDR, 0); + + for (i = 1; i < pm4_size; i++) + kgsl_regwrite(device, A3XX_CP_ME_RAM_DATA, + adreno_dev->fw[ADRENO_FW_PM4].fwvirt[i]); + + kgsl_regwrite(device, A3XX_CP_PFP_UCODE_ADDR, 0); + + for (i = 1; i < pfp_size; i++) + kgsl_regwrite(device, A3XX_CP_PFP_UCODE_DATA, + adreno_dev->fw[ADRENO_FW_PFP].fwvirt[i]); +} + +static u64 a3xx_read_alwayson(struct adreno_device *adreno_dev) +{ + /* A3XX does not have a always on timer */ + return 0; +} + +static irqreturn_t a3xx_irq_handler(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + irqreturn_t ret; + u32 status; + + /* Get the current interrupt status */ + kgsl_regread(device, A3XX_RBBM_INT_0_STATUS, &status); + + /* + * Clear all the interrupt bits except A3XX_INT_RBBM_AHB_ERROR. + * The interrupt will stay asserted until it is cleared by the handler + * so don't touch it yet to avoid a storm + */ + + kgsl_regwrite(device, A3XX_RBBM_INT_CLEAR_CMD, + status & ~A3XX_INT_RBBM_AHB_ERROR); + + /* Call the helper to execute the callbacks */ + ret = adreno_irq_callbacks(adreno_dev, a3xx_irq_funcs, status); + + trace_kgsl_a3xx_irq_status(adreno_dev, status); + + /* Now clear AHB_ERROR if it was set */ + if (status & A3XX_INT_RBBM_AHB_ERROR) + kgsl_regwrite(device, A3XX_RBBM_INT_CLEAR_CMD, + A3XX_INT_RBBM_AHB_ERROR); + + return ret; +} + +static bool a3xx_hw_isidle(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 status; + + kgsl_regread(device, A3XX_RBBM_STATUS, &status); + + if (status & 0x7ffffffe) + return false; + + kgsl_regread(device, A3XX_RBBM_INT_0_STATUS, &status); + + /* Return busy if a interrupt is pending */ + return !((status & adreno_dev->irq_mask) || + atomic_read(&adreno_dev->pending_irq_refcnt)); +} + +static int a3xx_clear_pending_transactions(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 mask = A30X_VBIF_XIN_HALT_CTRL0_MASK; + int ret; + + kgsl_regwrite(device, A3XX_VBIF_XIN_HALT_CTRL0, mask); + ret = adreno_wait_for_halt_ack(device, A3XX_VBIF_XIN_HALT_CTRL1, mask); + kgsl_regwrite(device, A3XX_VBIF_XIN_HALT_CTRL0, 0); + + return ret; +} + +static bool a3xx_is_hw_collapsible(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* + * Skip power collapse for A304, if power ctrl flag is set to + * non zero. As A304 soft_reset will not work, power collapse + * needs to disable to avoid soft_reset. + */ + if (adreno_is_a304(adreno_dev) && device->pwrctrl.ctrl_flags) + return false; + + return adreno_isidle(adreno_dev); +} + +static void a3xx_power_stats(struct adreno_device *adreno_dev, + struct kgsl_power_stats *stats) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_busy_data *busy = &adreno_dev->busy_data; + s64 freq = kgsl_pwrctrl_active_freq(&device->pwrctrl) / 1000000; + u64 gpu_busy; + + /* Set the GPU busy counter for frequency scaling */ + gpu_busy = counter_delta(device, A3XX_RBBM_PERFCTR_PWR_1_LO, + &busy->gpu_busy); + + stats->busy_time = gpu_busy / freq; + + if (!device->pwrctrl.bus_control) + return; + + stats->ram_time = counter_delta(device, adreno_dev->ram_cycles_lo, + &busy->bif_ram_cycles); + + stats->ram_wait = counter_delta(device, adreno_dev->starved_ram_lo, + &busy->bif_starved_ram); +} + +static int a3xx_setproperty(struct kgsl_device_private *dev_priv, + u32 type, void __user *value, u32 sizebytes) +{ + struct kgsl_device *device = dev_priv->device; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + u32 enable; + + if (type != KGSL_PROP_PWRCTRL) + return -ENODEV; + + if (sizebytes != sizeof(enable)) + return -EINVAL; + + if (copy_from_user(&enable, value, sizeof(enable))) + return -EFAULT; + + mutex_lock(&device->mutex); + if (enable) { + device->pwrctrl.ctrl_flags = 0; + + if (!adreno_active_count_get(adreno_dev)) { + a3xx_soft_fault_detect_start(adreno_dev); + adreno_active_count_put(adreno_dev); + } + + kgsl_pwrscale_enable(device); + } else { + kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); + device->pwrctrl.ctrl_flags = KGSL_PWR_ON; + + a3xx_soft_fault_detect_stop(adreno_dev); + kgsl_pwrscale_disable(device, true); + } + mutex_unlock(&device->mutex); + + return 0; +} + +const struct adreno_gpudev adreno_a3xx_gpudev = { + .reg_offsets = a3xx_register_offsets, + .irq_handler = a3xx_irq_handler, + .probe = a3xx_probe, + .rb_start = a3xx_rb_start, + .init = a3xx_init, + .start = a3xx_start, + .snapshot = a3xx_snapshot, + .read_alwayson = a3xx_read_alwayson, + .hw_isidle = a3xx_hw_isidle, + .power_ops = &adreno_power_operations, + .clear_pending_transactions = a3xx_clear_pending_transactions, + .ringbuffer_submitcmd = a3xx_ringbuffer_submitcmd, + .is_hw_collapsible = a3xx_is_hw_collapsible, + .power_stats = a3xx_power_stats, + .setproperty = a3xx_setproperty, + .remove = a3xx_remove, + .gx_is_on = a3xx_gx_is_on, +}; diff --git a/qcom/opensource/graphics-kernel/adreno_a3xx.h b/qcom/opensource/graphics-kernel/adreno_a3xx.h new file mode 100644 index 0000000000..5a3a902bd3 --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a3xx.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2013-2016, 2019-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ +#ifndef __A3XX_H +#define __A3XX_H + +#include "a3xx_reg.h" +/** + * struct adreno_a3xx_core - a3xx specific GPU core definitions + */ +struct adreno_a3xx_core { + /** @base: Container for the generic &struct adreno_gpu_core */ + struct adreno_gpu_core base; + /** pm4fw_name: Name of the PM4 microcode file */ + const char *pm4fw_name; + /** pfpfw_name: Name of the PFP microcode file */ + const char *pfpfw_name; + /** @vbif: List of registers and values to write for VBIF */ + const struct kgsl_regmap_list *vbif; + /** @vbif_count: Number of registers in @vbif */ + u32 vbif_count; +}; + +struct adreno_device; + +/** + * to_a3xx_core - return the a3xx specific GPU core struct + * @adreno_dev: An Adreno GPU device handle + * + * Returns: + * A pointer to the a3xx specific GPU core struct + */ +static inline const struct adreno_a3xx_core * +to_a3xx_core(struct adreno_device *adreno_dev) +{ + const struct adreno_gpu_core *core = adreno_dev->gpucore; + + return container_of(core, struct adreno_a3xx_core, base); +} + +void a3xx_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot); + +extern const struct adreno_perfcounters adreno_a3xx_perfcounters; + +/** + * a3xx_ringbuffer_init - Initialize the ringbuffer + * @adreno_dev: An Adreno GPU handle + * + * Initialize the ringbuffer for a3xx. + * Return: 0 on success or negative on failure + */ +int a3xx_ringbuffer_init(struct adreno_device *adreno_dev); + +/** + * a3xx_ringbuffer_submitcmd - Submit a user command to the ringbuffer + * @adreno_dev: An Adreno GPU handle + * @cmdobj: Pointer to a user command object + * @flags: Internal submit flags + * @time: Optional pointer to a adreno_submit_time container + * + * Return: 0 on success or negative on failure + */ +int a3xx_ringbuffer_submitcmd(struct adreno_device *adreno_dev, + struct kgsl_drawobj_cmd *cmdobj, u32 flags, + struct adreno_submit_time *time); + +#ifdef CONFIG_QCOM_KGSL_CORESIGHT +void a3xx_coresight_init(struct adreno_device *device); +#else +static inline void a3xx_coresight_init(struct adreno_device *device) { } +#endif + +#endif /*__A3XX_H */ diff --git a/qcom/opensource/graphics-kernel/adreno_a3xx_coresight.c b/qcom/opensource/graphics-kernel/adreno_a3xx_coresight.c new file mode 100644 index 0000000000..862a5a4ec9 --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a3xx_coresight.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022, 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include "adreno.h" +#include "adreno_a3xx.h" +#include "adreno_coresight.h" + +static struct adreno_coresight_register a3xx_coresight_registers[] = { + { A3XX_RBBM_DEBUG_BUS_CTL, 0x0001093F }, + { A3XX_RBBM_EXT_TRACE_STOP_CNT, 0x00017fff }, + { A3XX_RBBM_EXT_TRACE_START_CNT, 0x0001000f }, + { A3XX_RBBM_EXT_TRACE_PERIOD_CNT, 0x0001ffff }, + { A3XX_RBBM_EXT_TRACE_CMD, 0x00000001 }, + { A3XX_RBBM_EXT_TRACE_BUS_CTL, 0x89100010 }, + { A3XX_RBBM_DEBUG_BUS_STB_CTL0, 0x00000000 }, + { A3XX_RBBM_DEBUG_BUS_STB_CTL1, 0xFFFFFFFE }, + { A3XX_RBBM_INT_TRACE_BUS_CTL, 0x00201111 }, +}; + +static ADRENO_CORESIGHT_ATTR(config_debug_bus, + &a3xx_coresight_registers[0]); +static ADRENO_CORESIGHT_ATTR(config_trace_stop_cnt, + &a3xx_coresight_registers[1]); +static ADRENO_CORESIGHT_ATTR(config_trace_start_cnt, + &a3xx_coresight_registers[2]); +static ADRENO_CORESIGHT_ATTR(config_trace_period_cnt, + &a3xx_coresight_registers[3]); +static ADRENO_CORESIGHT_ATTR(config_trace_cmd, + &a3xx_coresight_registers[4]); +static ADRENO_CORESIGHT_ATTR(config_trace_bus_ctl, + &a3xx_coresight_registers[5]); + +static struct attribute *a3xx_coresight_attrs[] = { + &coresight_attr_config_debug_bus.attr.attr, + &coresight_attr_config_trace_start_cnt.attr.attr, + &coresight_attr_config_trace_stop_cnt.attr.attr, + &coresight_attr_config_trace_period_cnt.attr.attr, + &coresight_attr_config_trace_cmd.attr.attr, + &coresight_attr_config_trace_bus_ctl.attr.attr, + NULL, +}; + +static const struct attribute_group a3xx_coresight_group = { + .attrs = a3xx_coresight_attrs, +}; + +static const struct attribute_group *a3xx_coresight_groups[] = { + &a3xx_coresight_group, + NULL, +}; + +static const struct adreno_coresight a3xx_coresight = { + .registers = a3xx_coresight_registers, + .count = ARRAY_SIZE(a3xx_coresight_registers), + .groups = a3xx_coresight_groups, +}; + +void a3xx_coresight_init(struct adreno_device *adreno_dev) +{ + adreno_coresight_add_device(adreno_dev, "coresight-gfx", + &a3xx_coresight, &adreno_dev->gx_coresight); +} diff --git a/qcom/opensource/graphics-kernel/adreno_a3xx_perfcounter.c b/qcom/opensource/graphics-kernel/adreno_a3xx_perfcounter.c new file mode 100644 index 0000000000..a525fef97b --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a3xx_perfcounter.c @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#include "adreno.h" +#include "adreno_a3xx.h" +#include "adreno_perfcounter.h" +#include "kgsl_device.h" + +/* Bit flag for RBMM_PERFCTR_CTL */ +#define RBBM_PERFCTR_CTL_ENABLE 0x00000001 +#define VBIF2_PERF_CNT_SEL_MASK 0x7F +/* offset of clear register from select register */ +#define VBIF2_PERF_CLR_REG_SEL_OFF 8 +/* offset of enable register from select register */ +#define VBIF2_PERF_EN_REG_SEL_OFF 16 +/* offset of clear register from the enable register */ +#define VBIF2_PERF_PWR_CLR_REG_EN_OFF 8 + +static void a3xx_counter_load(struct adreno_device *adreno_dev, + struct adreno_perfcount_register *reg) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int index = reg->load_bit / 32; + u32 enable = BIT(reg->load_bit & 31); + + kgsl_regwrite(device, A3XX_RBBM_PERFCTR_LOAD_VALUE_LO, + lower_32_bits(reg->value)); + + kgsl_regwrite(device, A3XX_RBBM_PERFCTR_LOAD_VALUE_HI, + upper_32_bits(reg->value)); + + if (index == 0) + kgsl_regwrite(device, A3XX_RBBM_PERFCTR_LOAD_CMD0, enable); + else + kgsl_regwrite(device, A3XX_RBBM_PERFCTR_LOAD_CMD1, enable); +} + +static int a3xx_counter_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + + kgsl_regwrite(device, reg->select, countable); + reg->value = 0; + + return 0; +} + +static u64 a3xx_counter_read(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + u32 val, hi, lo; + + kgsl_regread(device, A3XX_RBBM_PERFCTR_CTL, &val); + kgsl_regwrite(device, A3XX_RBBM_PERFCTR_CTL, + val & ~RBBM_PERFCTR_CTL_ENABLE); + + kgsl_regread(device, reg->offset, &lo); + kgsl_regread(device, reg->offset_hi, &hi); + + kgsl_regwrite(device, A3XX_RBBM_PERFCTR_CTL, val); + + return (((u64) hi) << 32) | lo; +} + +static int a3xx_counter_pwr_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + return 0; +} + +static u64 a3xx_counter_pwr_read(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + u32 val, hi, lo; + + kgsl_regread(device, A3XX_RBBM_RBBM_CTL, &val); + + /* Freeze the counter so we can read it */ + if (!counter) + kgsl_regwrite(device, A3XX_RBBM_RBBM_CTL, val & ~0x10000); + else + kgsl_regwrite(device, A3XX_RBBM_RBBM_CTL, val & ~0x20000); + + kgsl_regread(device, reg->offset, &lo); + kgsl_regread(device, reg->offset_hi, &hi); + + kgsl_regwrite(device, A3XX_RBBM_RBBM_CTL, val); + + return ((((u64) hi) << 32) | lo) + reg->value; +} + +static int a3xx_counter_vbif_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + + if (countable > VBIF2_PERF_CNT_SEL_MASK) + return -EINVAL; + + /* + * Write 1, followed by 0 to CLR register for + * clearing the counter + */ + kgsl_regwrite(device, + reg->select - VBIF2_PERF_CLR_REG_SEL_OFF, 1); + kgsl_regwrite(device, + reg->select - VBIF2_PERF_CLR_REG_SEL_OFF, 0); + kgsl_regwrite(device, + reg->select, countable & VBIF2_PERF_CNT_SEL_MASK); + /* enable reg is 8 DWORDS before select reg */ + kgsl_regwrite(device, + reg->select - VBIF2_PERF_EN_REG_SEL_OFF, 1); + + kgsl_regwrite(device, reg->select, countable); + + reg->value = 0; + return 0; +} + +static u64 a3xx_counter_vbif_read(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + u32 hi, lo; + + /* freeze counter */ + kgsl_regwrite(device, reg->select - VBIF2_PERF_EN_REG_SEL_OFF, 0); + + kgsl_regread(device, reg->offset, &lo); + kgsl_regread(device, reg->offset_hi, &hi); + + /* un-freeze counter */ + kgsl_regwrite(device, reg->select - VBIF2_PERF_EN_REG_SEL_OFF, 1); + + return ((((u64) hi) << 32) | lo) + reg->value; +} + +static int a3xx_counter_vbif_pwr_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + + /* + * Write 1, followed by 0 to CLR register for + * clearing the counter + */ + kgsl_regwrite(device, reg->select + + VBIF2_PERF_PWR_CLR_REG_EN_OFF, 1); + kgsl_regwrite(device, reg->select + + VBIF2_PERF_PWR_CLR_REG_EN_OFF, 0); + kgsl_regwrite(device, reg->select, 1); + + reg->value = 0; + return 0; +} + +static u64 a3xx_counter_vbif_pwr_read(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + u32 hi, lo; + + /* freeze counter */ + kgsl_regwrite(device, reg->select, 0); + + kgsl_regread(device, reg->offset, &lo); + kgsl_regread(device, reg->offset_hi, &hi); + + /* un-freeze counter */ + kgsl_regwrite(device, reg->select, 1); + + return ((((u64) hi) << 32) | lo) + reg->value; +} + +/* + * Define the available perfcounter groups - these get used by + * adreno_perfcounter_get and adreno_perfcounter_put + */ + +static struct adreno_perfcount_register a3xx_perfcounters_cp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_CP_0_LO, + A3XX_RBBM_PERFCTR_CP_0_HI, 0, A3XX_CP_PERFCOUNTER_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_rbbm[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RBBM_0_LO, + A3XX_RBBM_PERFCTR_RBBM_0_HI, 1, A3XX_RBBM_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RBBM_1_LO, + A3XX_RBBM_PERFCTR_RBBM_1_HI, 2, A3XX_RBBM_PERFCOUNTER1_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_pc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_PC_0_LO, + A3XX_RBBM_PERFCTR_PC_0_HI, 3, A3XX_PC_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_PC_1_LO, + A3XX_RBBM_PERFCTR_PC_1_HI, 4, A3XX_PC_PERFCOUNTER1_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_PC_2_LO, + A3XX_RBBM_PERFCTR_PC_2_HI, 5, A3XX_PC_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_PC_3_LO, + A3XX_RBBM_PERFCTR_PC_3_HI, 6, A3XX_PC_PERFCOUNTER3_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_vfd[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_VFD_0_LO, + A3XX_RBBM_PERFCTR_VFD_0_HI, 7, A3XX_VFD_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_VFD_1_LO, + A3XX_RBBM_PERFCTR_VFD_1_HI, 8, A3XX_VFD_PERFCOUNTER1_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_hlsq[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_0_LO, + A3XX_RBBM_PERFCTR_HLSQ_0_HI, 9, + A3XX_HLSQ_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_1_LO, + A3XX_RBBM_PERFCTR_HLSQ_1_HI, 10, + A3XX_HLSQ_PERFCOUNTER1_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_2_LO, + A3XX_RBBM_PERFCTR_HLSQ_2_HI, 11, + A3XX_HLSQ_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_3_LO, + A3XX_RBBM_PERFCTR_HLSQ_3_HI, 12, + A3XX_HLSQ_PERFCOUNTER3_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_4_LO, + A3XX_RBBM_PERFCTR_HLSQ_4_HI, 13, + A3XX_HLSQ_PERFCOUNTER4_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_HLSQ_5_LO, + A3XX_RBBM_PERFCTR_HLSQ_5_HI, 14, + A3XX_HLSQ_PERFCOUNTER5_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_vpc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_VPC_0_LO, + A3XX_RBBM_PERFCTR_VPC_0_HI, 15, A3XX_VPC_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_VPC_1_LO, + A3XX_RBBM_PERFCTR_VPC_1_HI, 16, A3XX_VPC_PERFCOUNTER1_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_tse[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TSE_0_LO, + A3XX_RBBM_PERFCTR_TSE_0_HI, 17, A3XX_GRAS_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TSE_1_LO, + A3XX_RBBM_PERFCTR_TSE_1_HI, 18, A3XX_GRAS_PERFCOUNTER1_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_ras[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RAS_0_LO, + A3XX_RBBM_PERFCTR_RAS_0_HI, 19, A3XX_GRAS_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RAS_1_LO, + A3XX_RBBM_PERFCTR_RAS_1_HI, 20, A3XX_GRAS_PERFCOUNTER3_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_uche[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_0_LO, + A3XX_RBBM_PERFCTR_UCHE_0_HI, 21, + A3XX_UCHE_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_1_LO, + A3XX_RBBM_PERFCTR_UCHE_1_HI, 22, + A3XX_UCHE_PERFCOUNTER1_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_2_LO, + A3XX_RBBM_PERFCTR_UCHE_2_HI, 23, + A3XX_UCHE_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_3_LO, + A3XX_RBBM_PERFCTR_UCHE_3_HI, 24, + A3XX_UCHE_PERFCOUNTER3_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_4_LO, + A3XX_RBBM_PERFCTR_UCHE_4_HI, 25, + A3XX_UCHE_PERFCOUNTER4_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_UCHE_5_LO, + A3XX_RBBM_PERFCTR_UCHE_5_HI, 26, + A3XX_UCHE_PERFCOUNTER5_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_tp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_0_LO, + A3XX_RBBM_PERFCTR_TP_0_HI, 27, A3XX_TP_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_1_LO, + A3XX_RBBM_PERFCTR_TP_1_HI, 28, A3XX_TP_PERFCOUNTER1_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_2_LO, + A3XX_RBBM_PERFCTR_TP_2_HI, 29, A3XX_TP_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_3_LO, + A3XX_RBBM_PERFCTR_TP_3_HI, 30, A3XX_TP_PERFCOUNTER3_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_4_LO, + A3XX_RBBM_PERFCTR_TP_4_HI, 31, A3XX_TP_PERFCOUNTER4_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_TP_5_LO, + A3XX_RBBM_PERFCTR_TP_5_HI, 32, A3XX_TP_PERFCOUNTER5_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_sp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_0_LO, + A3XX_RBBM_PERFCTR_SP_0_HI, 33, A3XX_SP_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_1_LO, + A3XX_RBBM_PERFCTR_SP_1_HI, 34, A3XX_SP_PERFCOUNTER1_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_2_LO, + A3XX_RBBM_PERFCTR_SP_2_HI, 35, A3XX_SP_PERFCOUNTER2_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_3_LO, + A3XX_RBBM_PERFCTR_SP_3_HI, 36, A3XX_SP_PERFCOUNTER3_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_4_LO, + A3XX_RBBM_PERFCTR_SP_4_HI, 37, A3XX_SP_PERFCOUNTER4_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_5_LO, + A3XX_RBBM_PERFCTR_SP_5_HI, 38, A3XX_SP_PERFCOUNTER5_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_6_LO, + A3XX_RBBM_PERFCTR_SP_6_HI, 39, A3XX_SP_PERFCOUNTER6_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_SP_7_LO, + A3XX_RBBM_PERFCTR_SP_7_HI, 40, A3XX_SP_PERFCOUNTER7_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_rb[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RB_0_LO, + A3XX_RBBM_PERFCTR_RB_0_HI, 41, A3XX_RB_PERFCOUNTER0_SELECT }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_RB_1_LO, + A3XX_RBBM_PERFCTR_RB_1_HI, 42, A3XX_RB_PERFCOUNTER1_SELECT }, +}; + +static struct adreno_perfcount_register a3xx_perfcounters_pwr[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_RBBM_PERFCTR_PWR_0_LO, + A3XX_RBBM_PERFCTR_PWR_0_HI, -1, 0 }, + /* + * A3XX_RBBM_PERFCTR_PWR_1_LO is used for frequency scaling and removed + * from the pool of available counters + */ +}; + +static struct adreno_perfcount_register a3xx_perfcounters_vbif2[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF2_PERF_CNT_LOW0, + A3XX_VBIF2_PERF_CNT_HIGH0, -1, A3XX_VBIF2_PERF_CNT_SEL0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF2_PERF_CNT_LOW1, + A3XX_VBIF2_PERF_CNT_HIGH1, -1, A3XX_VBIF2_PERF_CNT_SEL1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF2_PERF_CNT_LOW2, + A3XX_VBIF2_PERF_CNT_HIGH2, -1, A3XX_VBIF2_PERF_CNT_SEL2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A3XX_VBIF2_PERF_CNT_LOW3, + A3XX_VBIF2_PERF_CNT_HIGH3, -1, A3XX_VBIF2_PERF_CNT_SEL3 }, +}; +/* + * Placing EN register in select field since vbif perf counters + * don't have select register to program + */ +static struct adreno_perfcount_register a3xx_perfcounters_vbif2_pwr[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, + 0, A3XX_VBIF2_PERF_PWR_CNT_LOW0, + A3XX_VBIF2_PERF_PWR_CNT_HIGH0, -1, + A3XX_VBIF2_PERF_PWR_CNT_EN0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, + 0, A3XX_VBIF2_PERF_PWR_CNT_LOW1, + A3XX_VBIF2_PERF_PWR_CNT_HIGH1, -1, + A3XX_VBIF2_PERF_PWR_CNT_EN1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, + 0, A3XX_VBIF2_PERF_PWR_CNT_LOW2, + A3XX_VBIF2_PERF_PWR_CNT_HIGH2, -1, + A3XX_VBIF2_PERF_PWR_CNT_EN2 }, +}; + +#define A3XX_PERFCOUNTER_GROUP(offset, name, enable, read, load) \ + ADRENO_PERFCOUNTER_GROUP(a3xx, offset, name, enable, read, load) + +#define A3XX_PERFCOUNTER_GROUP_FLAGS(offset, name, flags, enable, read, load) \ + ADRENO_PERFCOUNTER_GROUP_FLAGS(a3xx, offset, name, flags, enable, read, load) + +#define A3XX_REGULAR_PERFCOUNTER_GROUP(offset, name) \ + A3XX_PERFCOUNTER_GROUP(offset, name, a3xx_counter_enable,\ + a3xx_counter_read, a3xx_counter_load) + +static const struct adreno_perfcount_group +a3xx_perfcounter_groups[KGSL_PERFCOUNTER_GROUP_MAX] = { + A3XX_REGULAR_PERFCOUNTER_GROUP(CP, cp), + A3XX_REGULAR_PERFCOUNTER_GROUP(RBBM, rbbm), + A3XX_REGULAR_PERFCOUNTER_GROUP(PC, pc), + A3XX_REGULAR_PERFCOUNTER_GROUP(VFD, vfd), + A3XX_REGULAR_PERFCOUNTER_GROUP(HLSQ, hlsq), + A3XX_REGULAR_PERFCOUNTER_GROUP(VPC, vpc), + A3XX_REGULAR_PERFCOUNTER_GROUP(TSE, tse), + A3XX_REGULAR_PERFCOUNTER_GROUP(RAS, ras), + A3XX_REGULAR_PERFCOUNTER_GROUP(UCHE, uche), + A3XX_REGULAR_PERFCOUNTER_GROUP(TP, tp), + A3XX_REGULAR_PERFCOUNTER_GROUP(SP, sp), + A3XX_REGULAR_PERFCOUNTER_GROUP(RB, rb), + A3XX_PERFCOUNTER_GROUP_FLAGS(PWR, pwr, + ADRENO_PERFCOUNTER_GROUP_FIXED, + a3xx_counter_pwr_enable, a3xx_counter_pwr_read, NULL), + A3XX_PERFCOUNTER_GROUP(VBIF, vbif2, + a3xx_counter_vbif_enable, a3xx_counter_vbif_read, NULL), + A3XX_PERFCOUNTER_GROUP_FLAGS(VBIF_PWR, vbif2_pwr, + ADRENO_PERFCOUNTER_GROUP_FIXED, + a3xx_counter_vbif_pwr_enable, a3xx_counter_vbif_pwr_read, + NULL), + +}; + +const struct adreno_perfcounters adreno_a3xx_perfcounters = { + a3xx_perfcounter_groups, + ARRAY_SIZE(a3xx_perfcounter_groups), +}; diff --git a/qcom/opensource/graphics-kernel/adreno_a3xx_ringbuffer.c b/qcom/opensource/graphics-kernel/adreno_a3xx_ringbuffer.c new file mode 100644 index 0000000000..3fbc91b8b5 --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a3xx_ringbuffer.c @@ -0,0 +1,458 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include "adreno.h" +#include "adreno_a3xx.h" +#include "adreno_pm4types.h" +#include "adreno_ringbuffer.h" +#include "adreno_trace.h" +#include "kgsl_trace.h" + +static int a3xx_wait_reg(unsigned int *cmds, unsigned int addr, + unsigned int val, unsigned int mask, + unsigned int interval) +{ + cmds[0] = cp_type3_packet(CP_WAIT_REG_EQ, 4); + cmds[1] = addr; + cmds[2] = val; + cmds[3] = mask; + cmds[4] = interval; + + return 5; +} + +static int a3xx_vbif_lock(unsigned int *cmds) +{ + int count; + + /* + * glue commands together until next + * WAIT_FOR_ME + */ + count = a3xx_wait_reg(cmds, A3XX_CP_WFI_PEND_CTR, + 1, 0xFFFFFFFF, 0xF); + + /* MMU-500 VBIF stall */ + cmds[count++] = cp_type3_packet(CP_REG_RMW, 3); + cmds[count++] = A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL0; + /* AND to unmask the HALT bit */ + cmds[count++] = ~(VBIF_RECOVERABLE_HALT_CTRL); + /* OR to set the HALT bit */ + cmds[count++] = 0x1; + + /* Wait for acknowledgment */ + count += a3xx_wait_reg(&cmds[count], + A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL1, + 1, 0xFFFFFFFF, 0xF); + + return count; +} + +static int a3xx_vbif_unlock(unsigned int *cmds) +{ + /* MMU-500 VBIF unstall */ + cmds[0] = cp_type3_packet(CP_REG_RMW, 3); + cmds[1] = A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL0; + /* AND to unmask the HALT bit */ + cmds[2] = ~(VBIF_RECOVERABLE_HALT_CTRL); + /* OR to reset the HALT bit */ + cmds[3] = 0; + + /* release all commands since _vbif_lock() with wait_for_me */ + cmds[4] = cp_type3_packet(CP_WAIT_FOR_ME, 1); + cmds[5] = 0; + + return 6; +} + +#define A3XX_GPU_OFFSET 0xa000 + +static int a3xx_cp_smmu_reg(unsigned int *cmds, + u32 reg, + unsigned int num) +{ + cmds[0] = cp_type3_packet(CP_REG_WR_NO_CTXT, num + 1); + cmds[1] = (A3XX_GPU_OFFSET + reg) >> 2; + + return 2; +} + +/* This function is only needed for A3xx targets */ +static int a3xx_tlbiall(unsigned int *cmds) +{ + unsigned int tlbstatus = (A3XX_GPU_OFFSET + + KGSL_IOMMU_CTX_TLBSTATUS) >> 2; + int count; + + count = a3xx_cp_smmu_reg(cmds, KGSL_IOMMU_CTX_TLBIALL, 1); + cmds[count++] = 1; + + count += a3xx_cp_smmu_reg(&cmds[count], KGSL_IOMMU_CTX_TLBSYNC, 1); + cmds[count++] = 0; + + count += a3xx_wait_reg(&cmds[count], tlbstatus, 0, + KGSL_IOMMU_CTX_TLBSTATUS_SACTIVE, 0xF); + + return count; +} + +/* offset at which a nop command is placed in setstate */ +#define KGSL_IOMMU_SETSTATE_NOP_OFFSET 1024 + +static int a3xx_rb_pagetable_switch(struct adreno_device *adreno_dev, + struct kgsl_pagetable *pagetable, u32 *cmds) +{ + u64 ttbr0 = kgsl_mmu_pagetable_get_ttbr0(pagetable); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_iommu *iommu = KGSL_IOMMU(device); + int count = 0; + + /* Skip pagetable switch if current context is using default PT. */ + if (pagetable == device->mmu.defaultpagetable) + return 0; + /* + * Adding an indirect buffer ensures that the prefetch stalls until + * the commands in indirect buffer have completed. We need to stall + * prefetch with a nop indirect buffer when updating pagetables + * because it provides stabler synchronization. + */ + cmds[count++] = cp_type3_packet(CP_WAIT_FOR_ME, 1); + cmds[count++] = 0; + + cmds[count++] = cp_type3_packet(CP_INDIRECT_BUFFER_PFE, 2); + cmds[count++] = lower_32_bits(iommu->setstate->gpuaddr); + cmds[count++] = 2; + + cmds[count++] = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + cmds[count++] = 0; + + cmds[count++] = cp_type3_packet(CP_WAIT_FOR_ME, 1); + cmds[count++] = 0; + + count += a3xx_vbif_lock(&cmds[count]); + + count += a3xx_cp_smmu_reg(&cmds[count], KGSL_IOMMU_CTX_TTBR0, 2); + cmds[count++] = lower_32_bits(ttbr0); + cmds[count++] = upper_32_bits(ttbr0); + + count += a3xx_vbif_unlock(&cmds[count]); + + count += a3xx_tlbiall(&cmds[count]); + + /* wait for me to finish the TLBI */ + cmds[count++] = cp_type3_packet(CP_WAIT_FOR_ME, 1); + cmds[count++] = 0; + cmds[count++] = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + cmds[count++] = 0; + + /* Invalidate the state */ + cmds[count++] = cp_type3_packet(CP_INVALIDATE_STATE, 1); + cmds[count++] = 0x7ffff; + + return count; +} + +#define RB_SOPTIMESTAMP(device, rb) \ + MEMSTORE_RB_GPU_ADDR(device, rb, soptimestamp) +#define CTXT_SOPTIMESTAMP(device, drawctxt) \ + MEMSTORE_ID_GPU_ADDR(device, (drawctxt)->base.id, soptimestamp) + +#define RB_EOPTIMESTAMP(device, rb) \ + MEMSTORE_RB_GPU_ADDR(device, rb, eoptimestamp) +#define CTXT_EOPTIMESTAMP(device, drawctxt) \ + MEMSTORE_ID_GPU_ADDR(device, (drawctxt)->base.id, eoptimestamp) + +int a3xx_ringbuffer_init(struct adreno_device *adreno_dev) +{ + adreno_dev->num_ringbuffers = 1; + + adreno_dev->cur_rb = &(adreno_dev->ringbuffers[0]); + + return adreno_ringbuffer_setup(adreno_dev, + &adreno_dev->ringbuffers[0], 0); +} + +#define A3XX_SUBMIT_MAX 55 + +static int a3xx_ringbuffer_addcmds(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, struct adreno_context *drawctxt, + u32 flags, u32 *in, u32 dwords, u32 timestamp, + struct adreno_submit_time *time) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 size = A3XX_SUBMIT_MAX + dwords; + u32 *cmds, index = 0; + u64 profile_gpuaddr; + u32 profile_dwords; + + if (adreno_drawctxt_detached(drawctxt)) + return -ENOENT; + + if (adreno_gpu_fault(adreno_dev) != 0) + return -EPROTO; + + rb->timestamp++; + + if (drawctxt) + drawctxt->internal_timestamp = rb->timestamp; + + cmds = adreno_ringbuffer_allocspace(rb, size); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + /* Identify the start of a command */ + cmds[index++] = cp_type3_packet(CP_NOP, 1); + cmds[index++] = drawctxt ? CMD_IDENTIFIER : CMD_INTERNAL_IDENTIFIER; + + if (IS_PWRON_FIXUP(flags)) { + cmds[index++] = cp_type3_packet(CP_SET_PROTECTED_MODE, 1); + cmds[index++] = 0; + + cmds[index++] = cp_type3_packet(CP_NOP, 1); + cmds[index++] = PWRON_FIXUP_IDENTIFIER; + + cmds[index++] = cp_type3_packet(CP_INDIRECT_BUFFER_PFE, 2); + cmds[index++] = lower_32_bits(adreno_dev->pwron_fixup->gpuaddr); + cmds[index++] = adreno_dev->pwron_fixup_dwords; + + cmds[index++] = cp_type3_packet(CP_SET_PROTECTED_MODE, 1); + cmds[index++] = 0; + } + + profile_gpuaddr = adreno_profile_preib_processing(adreno_dev, + drawctxt, &profile_dwords); + + if (profile_gpuaddr) { + cmds[index++] = cp_type3_packet(CP_INDIRECT_BUFFER_PFE, 2); + cmds[index++] = lower_32_bits(profile_gpuaddr); + cmds[index++] = profile_dwords; + } + + if (drawctxt) { + cmds[index++] = cp_type3_packet(CP_MEM_WRITE, 2); + cmds[index++] = lower_32_bits(CTXT_SOPTIMESTAMP(device, + drawctxt)); + cmds[index++] = timestamp; + } + + cmds[index++] = cp_type3_packet(CP_MEM_WRITE, 2); + cmds[index++] = lower_32_bits(RB_SOPTIMESTAMP(device, rb)); + cmds[index++] = rb->timestamp; + + if (IS_NOTPROTECTED(flags)) { + cmds[index++] = cp_type3_packet(CP_SET_PROTECTED_MODE, 1); + cmds[index++] = 0; + } + + memcpy(&cmds[index], in, dwords << 2); + index += dwords; + + if (IS_NOTPROTECTED(flags)) { + cmds[index++] = cp_type3_packet(CP_SET_PROTECTED_MODE, 1); + cmds[index++] = 1; + } + + /* + * Flush HLSQ lazy updates to make sure there are no resourses pending + * for indirect loads after the timestamp + */ + + cmds[index++] = cp_type3_packet(CP_EVENT_WRITE, 1); + cmds[index++] = 0x07; /* HLSQ FLUSH */ + cmds[index++] = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + cmds[index++] = 0; + + profile_gpuaddr = adreno_profile_postib_processing(adreno_dev, + drawctxt, &profile_dwords); + + if (profile_gpuaddr) { + cmds[index++] = cp_type3_packet(CP_INDIRECT_BUFFER_PFE, 2); + cmds[index++] = lower_32_bits(profile_gpuaddr); + cmds[index++] = profile_dwords; + } + + /* + * If this is an internal command, just write the ringbuffer timestamp, + * otherwise, write both + */ + if (!drawctxt) { + cmds[index++] = cp_type3_packet(CP_EVENT_WRITE, 3); + cmds[index++] = CACHE_FLUSH_TS | (1 << 31); + cmds[index++] = lower_32_bits(RB_EOPTIMESTAMP(device, rb)); + cmds[index++] = rb->timestamp; + } else { + cmds[index++] = cp_type3_packet(CP_EVENT_WRITE, 3); + cmds[index++] = CACHE_FLUSH_TS | (1 << 31); + cmds[index++] = lower_32_bits(CTXT_EOPTIMESTAMP(device, + drawctxt)); + cmds[index++] = timestamp; + + cmds[index++] = cp_type3_packet(CP_EVENT_WRITE, 3); + cmds[index++] = CACHE_FLUSH_TS; + cmds[index++] = lower_32_bits(RB_EOPTIMESTAMP(device, rb)); + cmds[index++] = rb->timestamp; + } + + /* Trigger a context rollover */ + cmds[index++] = cp_type3_packet(CP_SET_CONSTANT, 2); + cmds[index++] = (4 << 16) | (A3XX_HLSQ_CL_KERNEL_GROUP_X_REG - 0x2000); + cmds[index++] = 0; + + if (IS_WFI(flags)) { + cmds[index++] = cp_type3_packet(CP_WAIT_FOR_IDLE, 1); + cmds[index++] = 0; + } + + /* Adjust the thing for the number of bytes we actually wrote */ + rb->_wptr -= (size - index); + + kgsl_pwrscale_busy(device); + kgsl_regwrite(device, A3XX_CP_RB_WPTR, rb->_wptr); + rb->wptr = rb->_wptr; + + return 0; +} + +static int a3xx_rb_context_switch(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, + struct adreno_context *drawctxt) +{ + struct kgsl_pagetable *pagetable = + adreno_drawctxt_get_pagetable(drawctxt); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int count = 0; + u32 cmds[64]; + + if (adreno_drawctxt_get_pagetable(rb->drawctxt_active) != pagetable) + count += a3xx_rb_pagetable_switch(adreno_dev, pagetable, cmds); + + cmds[count++] = cp_type3_packet(CP_NOP, 1); + cmds[count++] = CONTEXT_TO_MEM_IDENTIFIER; + + cmds[count++] = cp_type3_packet(CP_MEM_WRITE, 2); + cmds[count++] = lower_32_bits(MEMSTORE_RB_GPU_ADDR(device, rb, + current_context)); + cmds[count++] = drawctxt->base.id; + + cmds[count++] = cp_type3_packet(CP_MEM_WRITE, 2); + cmds[count++] = lower_32_bits(MEMSTORE_ID_GPU_ADDR(device, + KGSL_MEMSTORE_GLOBAL, current_context)); + cmds[count++] = drawctxt->base.id; + + cmds[count++] = cp_type0_packet(A3XX_UCHE_CACHE_INVALIDATE0_REG, 2); + cmds[count++] = 0; + cmds[count++] = 0x90000000; + + return a3xx_ringbuffer_addcmds(adreno_dev, rb, NULL, F_NOTPROTECTED, + cmds, count, 0, NULL); +} + +static int a3xx_drawctxt_switch(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, + struct adreno_context *drawctxt) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (rb->drawctxt_active == drawctxt) + return 0; + + if (kgsl_context_detached(&drawctxt->base)) + return -ENOENT; + + if (!_kgsl_context_get(&drawctxt->base)) + return -ENOENT; + + trace_adreno_drawctxt_switch(rb, drawctxt); + + a3xx_rb_context_switch(adreno_dev, rb, drawctxt); + + /* Release the current drawctxt as soon as the new one is switched */ + adreno_put_drawctxt_on_timestamp(device, rb->drawctxt_active, + rb, rb->timestamp); + + rb->drawctxt_active = drawctxt; + return 0; +} + +#define A3XX_COMMAND_DWORDS 4 + +int a3xx_ringbuffer_submitcmd(struct adreno_device *adreno_dev, + struct kgsl_drawobj_cmd *cmdobj, u32 flags, + struct adreno_submit_time *time) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); + struct adreno_context *drawctxt = ADRENO_CONTEXT(drawobj->context); + struct adreno_ringbuffer *rb = drawctxt->rb; + int ret = 0, numibs = 0, index = 0; + u32 *cmds; + + /* Count the number of IBs (if we are not skipping) */ + if (!IS_SKIP(flags)) { + struct list_head *tmp; + + list_for_each(tmp, &cmdobj->cmdlist) + numibs++; + } + + cmds = kmalloc((A3XX_COMMAND_DWORDS + (numibs * 4)) << 2, GFP_KERNEL); + if (!cmds) { + ret = -ENOMEM; + goto done; + } + + cmds[index++] = cp_type3_packet(CP_NOP, 1); + cmds[index++] = START_IB_IDENTIFIER; + + if (numibs) { + struct kgsl_memobj_node *ib; + + list_for_each_entry(ib, &cmdobj->cmdlist, node) { + if (ib->priv & MEMOBJ_SKIP || + (ib->flags & KGSL_CMDLIST_CTXTSWITCH_PREAMBLE + && !IS_PREAMBLE(flags))) + cmds[index++] = cp_type3_packet(CP_NOP, 3); + + cmds[index++] = + cp_type3_packet(CP_INDIRECT_BUFFER_PFE, 2); + cmds[index++] = lower_32_bits(ib->gpuaddr); + cmds[index++] = ib->size >> 2; + } + } + + cmds[index++] = cp_type3_packet(CP_NOP, 1); + cmds[index++] = END_IB_IDENTIFIER; + + ret = a3xx_drawctxt_switch(adreno_dev, rb, drawctxt); + + /* + * In the unlikely event of an error in the drawctxt switch, + * treat it like a hang + */ + if (ret) { + /* + * It is "normal" to get a -ENOSPC or a -ENOENT. Don't log it, + * the upper layers know how to handle it + */ + if (ret != -ENOSPC && ret != -ENOENT) + dev_err(device->dev, + "Unable to switch draw context: %d\n", + ret); + goto done; + } + + adreno_drawobj_set_constraint(device, drawobj); + + ret = a3xx_ringbuffer_addcmds(adreno_dev, drawctxt->rb, drawctxt, + flags, cmds, index, drawobj->timestamp, NULL); + +done: + trace_kgsl_issueibcmds(device, drawctxt->base.id, numibs, + drawobj->timestamp, drawobj->flags, ret, drawctxt->type); + + kfree(cmds); + return ret; +} diff --git a/qcom/opensource/graphics-kernel/adreno_a3xx_snapshot.c b/qcom/opensource/graphics-kernel/adreno_a3xx_snapshot.c new file mode 100644 index 0000000000..4b4d4184d6 --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a3xx_snapshot.c @@ -0,0 +1,449 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012-2017,2019-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include + +#include "adreno.h" +#include "adreno_a3xx.h" +#include "adreno_snapshot.h" +#include "kgsl_device.h" + +/* + * Set of registers to dump for A3XX on snapshot. + * Registers in pairs - first value is the start offset, second + * is the stop offset (inclusive) + */ + +static const unsigned int a3xx_registers[] = { + 0x0000, 0x0002, 0x0010, 0x0012, 0x0018, 0x0018, 0x0020, 0x0027, + 0x0029, 0x002b, 0x002e, 0x0033, 0x0040, 0x0042, 0x0050, 0x005c, + 0x0060, 0x006c, 0x0080, 0x0082, 0x0084, 0x0088, 0x0090, 0x00e5, + 0x00ea, 0x00ed, 0x0100, 0x0100, 0x0110, 0x0123, 0x01c0, 0x01c1, + 0x01c3, 0x01c5, 0x01c7, 0x01c7, 0x01d5, 0x01d9, 0x01dc, 0x01dd, + 0x01ea, 0x01ea, 0x01ee, 0x01f1, 0x01f5, 0x01f6, 0x01f8, 0x01f9, + 0x01fc, 0x01ff, + 0x0440, 0x0440, 0x0443, 0x0443, 0x0445, 0x0445, 0x044d, 0x044f, + 0x0452, 0x0452, 0x0454, 0x046f, 0x047c, 0x047c, 0x047f, 0x047f, + 0x0578, 0x057f, 0x0600, 0x0602, 0x0605, 0x0607, 0x060a, 0x060e, + 0x0612, 0x0614, 0x0c01, 0x0c02, 0x0c06, 0x0c1d, 0x0c3d, 0x0c3f, + 0x0c48, 0x0c4b, 0x0c80, 0x0c80, 0x0c88, 0x0c8b, 0x0ca0, 0x0cb7, + 0x0cc0, 0x0cc1, 0x0cc6, 0x0cc7, 0x0ce4, 0x0ce5, + 0x0e41, 0x0e45, 0x0e64, 0x0e65, + 0x0e80, 0x0e82, 0x0e84, 0x0e89, 0x0ea0, 0x0ea1, 0x0ea4, 0x0ea7, + 0x0ec4, 0x0ecb, 0x0ee0, 0x0ee0, 0x0f00, 0x0f01, 0x0f03, 0x0f09, + 0x2040, 0x2040, 0x2044, 0x2044, 0x2048, 0x204d, 0x2068, 0x2069, + 0x206c, 0x206d, 0x2070, 0x2070, 0x2072, 0x2072, 0x2074, 0x2075, + 0x2079, 0x207a, 0x20c0, 0x20d3, 0x20e4, 0x20ef, 0x2100, 0x2109, + 0x210c, 0x210c, 0x210e, 0x210e, 0x2110, 0x2111, 0x2114, 0x2115, + 0x21e4, 0x21e4, 0x21ea, 0x21ea, 0x21ec, 0x21ed, 0x21f0, 0x21f0, + 0x2240, 0x227e, + 0x2280, 0x228b, 0x22c0, 0x22c0, 0x22c4, 0x22ce, 0x22d0, 0x22d8, + 0x22df, 0x22e6, 0x22e8, 0x22e9, 0x22ec, 0x22ec, 0x22f0, 0x22f7, + 0x22ff, 0x22ff, 0x2340, 0x2343, + 0x2440, 0x2440, 0x2444, 0x2444, 0x2448, 0x244d, + 0x2468, 0x2469, 0x246c, 0x246d, 0x2470, 0x2470, 0x2472, 0x2472, + 0x2474, 0x2475, 0x2479, 0x247a, 0x24c0, 0x24d3, 0x24e4, 0x24ef, + 0x2500, 0x2509, 0x250c, 0x250c, 0x250e, 0x250e, 0x2510, 0x2511, + 0x2514, 0x2515, 0x25e4, 0x25e4, 0x25ea, 0x25ea, 0x25ec, 0x25ed, + 0x25f0, 0x25f0, + 0x2640, 0x267e, 0x2680, 0x268b, 0x26c0, 0x26c0, 0x26c4, 0x26ce, + 0x26d0, 0x26d8, 0x26df, 0x26e6, 0x26e8, 0x26e9, 0x26ec, 0x26ec, + 0x26f0, 0x26f7, 0x26ff, 0x26ff, 0x2740, 0x2743, + 0x300C, 0x300E, 0x301C, 0x301D, + 0x302A, 0x302A, 0x302C, 0x302D, 0x3030, 0x3031, 0x3034, 0x3036, + 0x303C, 0x303C, 0x305E, 0x305F, +}; + +/* Removed the following HLSQ register ranges from being read during + * fault tolerance since reading the registers may cause the device to hang: + */ +static const unsigned int a3xx_hlsq_registers[] = { + 0x0e00, 0x0e05, 0x0e0c, 0x0e0c, 0x0e22, 0x0e23, + 0x2200, 0x2212, 0x2214, 0x2217, 0x221a, 0x221a, + 0x2600, 0x2612, 0x2614, 0x2617, 0x261a, 0x261a, +}; + +/* Shader memory size in words */ +#define SHADER_MEMORY_SIZE 0x4000 + +/** + * _rbbm_debug_bus_read - Helper function to read data from the RBBM + * debug bus. + * @device - GPU device to read/write registers + * @block_id - Debug bus block to read from + * @index - Index in the debug bus block to read + * @ret - Value of the register read + */ +static void _rbbm_debug_bus_read(struct kgsl_device *device, + unsigned int block_id, unsigned int index, unsigned int *val) +{ + unsigned int block = (block_id << 8) | 1 << 16; + + kgsl_regwrite(device, A3XX_RBBM_DEBUG_BUS_CTL, block | index); + kgsl_regread(device, A3XX_RBBM_DEBUG_BUS_DATA_STATUS, val); +} + +/** + * a3xx_snapshot_shader_memory - Helper function to dump the GPU shader + * memory to the snapshot buffer. + * @device: GPU device whose shader memory is to be dumped + * @buf: Pointer to binary snapshot data blob being made + * @remain: Number of remaining bytes in the snapshot blob + * @priv: Unused parameter + * + */ +static size_t a3xx_snapshot_shader_memory(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + void *data = buf + sizeof(*header); + unsigned int shader_read_len = SHADER_MEMORY_SIZE; + + if (remain < DEBUG_SECTION_SZ(shader_read_len)) { + SNAPSHOT_ERR_NOMEM(device, "SHADER MEMORY"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_SHADER_MEMORY; + header->size = shader_read_len; + + /* Map shader memory to kernel, for dumping */ + if (IS_ERR_OR_NULL(device->shader_mem_virt)) { + struct resource *res; + + res = platform_get_resource_byname(device->pdev, + IORESOURCE_MEM, "kgsl_3d0_shader_memory"); + + if (res) + device->shader_mem_virt = + devm_ioremap_resource(&device->pdev->dev, res); + } + + if (IS_ERR_OR_NULL(device->shader_mem_virt)) { + dev_err(device->dev, "Unable to map the shader memory\n"); + return 0; + } + + memcpy_fromio(data, device->shader_mem_virt, shader_read_len << 2); + + return DEBUG_SECTION_SZ(shader_read_len); +} + +static size_t a3xx_snapshot_debugbus_block(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_debugbus *header + = (struct kgsl_snapshot_debugbus *)buf; + struct adreno_debugbus_block *block = priv; + int i; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + size_t size; + + size = (0x40 * sizeof(unsigned int)) + sizeof(*header); + + if (remain < size) { + SNAPSHOT_ERR_NOMEM(device, "DEBUGBUS"); + return 0; + } + + header->id = block->block_id; + header->count = 0x40; + + for (i = 0; i < 0x40; i++) + _rbbm_debug_bus_read(device, block->block_id, i, &data[i]); + + return size; +} + +static struct adreno_debugbus_block debugbus_blocks[] = { + { RBBM_BLOCK_ID_CP, 0x52, }, + { RBBM_BLOCK_ID_RBBM, 0x40, }, + { RBBM_BLOCK_ID_VBIF, 0x40, }, + { RBBM_BLOCK_ID_HLSQ, 0x40, }, + { RBBM_BLOCK_ID_UCHE, 0x40, }, + { RBBM_BLOCK_ID_PC, 0x40, }, + { RBBM_BLOCK_ID_VFD, 0x40, }, + { RBBM_BLOCK_ID_VPC, 0x40, }, + { RBBM_BLOCK_ID_TSE, 0x40, }, + { RBBM_BLOCK_ID_RAS, 0x40, }, + { RBBM_BLOCK_ID_VSC, 0x40, }, + { RBBM_BLOCK_ID_SP_0, 0x40, }, + { RBBM_BLOCK_ID_SP_1, 0x40, }, + { RBBM_BLOCK_ID_SP_2, 0x40, }, + { RBBM_BLOCK_ID_SP_3, 0x40, }, + { RBBM_BLOCK_ID_TPL1_0, 0x40, }, + { RBBM_BLOCK_ID_TPL1_1, 0x40, }, + { RBBM_BLOCK_ID_TPL1_2, 0x40, }, + { RBBM_BLOCK_ID_TPL1_3, 0x40, }, + { RBBM_BLOCK_ID_RB_0, 0x40, }, + { RBBM_BLOCK_ID_RB_1, 0x40, }, + { RBBM_BLOCK_ID_RB_2, 0x40, }, + { RBBM_BLOCK_ID_RB_3, 0x40, }, + { RBBM_BLOCK_ID_MARB_0, 0x40, }, + { RBBM_BLOCK_ID_MARB_1, 0x40, }, + { RBBM_BLOCK_ID_MARB_2, 0x40, }, + { RBBM_BLOCK_ID_MARB_3, 0x40, }, +}; + +static void a3xx_snapshot_debugbus(struct kgsl_device *device, + struct kgsl_snapshot *snapshot) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(debugbus_blocks); i++) { + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_DEBUGBUS, snapshot, + a3xx_snapshot_debugbus_block, + (void *) &debugbus_blocks[i]); + } +} + +static void _snapshot_hlsq_regs(struct kgsl_device *device, + struct kgsl_snapshot *snapshot) +{ + unsigned int next_pif = 0; + + /* + * Trying to read HLSQ registers when the HLSQ block is busy + * will cause the device to hang. The RBBM_DEBUG_BUS has information + * that will tell us if the HLSQ block is busy or not. Read values + * from the debug bus to ensure the HLSQ block is not busy (this + * is hardware dependent). If the HLSQ block is busy do not + * dump the registers, otherwise dump the HLSQ registers. + */ + + /* + * tpif status bits: RBBM_BLOCK_ID_HLSQ index 4 [4:0] + * spif status bits: RBBM_BLOCK_ID_HLSQ index 7 [5:0] + * + * if ((tpif == 0, 1, 28) && (spif == 0, 1, 10)) + * then dump HLSQ registers + */ + + /* check tpif */ + _rbbm_debug_bus_read(device, RBBM_BLOCK_ID_HLSQ, 4, &next_pif); + next_pif &= 0x1f; + if (next_pif != 0 && next_pif != 1 && next_pif != 28) + return; + + /* check spif */ + _rbbm_debug_bus_read(device, RBBM_BLOCK_ID_HLSQ, 7, &next_pif); + next_pif &= 0x3f; + if (next_pif != 0 && next_pif != 1 && next_pif != 10) + return; + + SNAPSHOT_REGISTERS(device, snapshot, a3xx_hlsq_registers); +} + +#define VPC_MEM_SIZE 512 + +static size_t a3xx_snapshot_vpc_memory(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + size_t size = 4 * VPC_MEM_SIZE; + int bank, addr, i = 0; + + if (remain < DEBUG_SECTION_SZ(size)) { + SNAPSHOT_ERR_NOMEM(device, "VPC MEMORY"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_VPC_MEMORY; + header->size = size; + + for (bank = 0; bank < 4; bank++) { + for (addr = 0; addr < VPC_MEM_SIZE; addr++) { + unsigned int val = bank | (addr << 4); + + kgsl_regwrite(device, A3XX_VPC_VPC_DEBUG_RAM_SEL, val); + kgsl_regread(device, A3XX_VPC_VPC_DEBUG_RAM_READ, + &data[i++]); + } + } + + return DEBUG_SECTION_SZ(size); +} + +static size_t a3xx_snapshot_cp_pm4_ram(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4); + size_t size = fw->size - 1; + + if (remain < DEBUG_SECTION_SZ(size)) { + SNAPSHOT_ERR_NOMEM(device, "CP PM4 RAM DEBUG"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_CP_PM4_RAM; + header->size = size; + + /* + * Read the firmware from the GPU rather than use our cache in order to + * try to catch mis-programming or corruption in the hardware. We do + * use the cached version of the size, however, instead of trying to + * maintain always changing hardcoded constants + */ + kgsl_regmap_read_indexed(&device->regmap, A3XX_CP_ME_RAM_RADDR, + A3XX_CP_ME_RAM_DATA, data, size); + + return DEBUG_SECTION_SZ(size); +} + +static size_t a3xx_snapshot_cp_pfp_ram(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); + int size = fw->size - 1; + + if (remain < DEBUG_SECTION_SZ(size)) { + SNAPSHOT_ERR_NOMEM(device, "CP PFP RAM DEBUG"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_CP_PFP_RAM; + header->size = size; + + /* + * Read the firmware from the GPU rather than use our cache in order to + * try to catch mis-programming or corruption in the hardware. We do + * use the cached version of the size, however, instead of trying to + * maintain always changing hardcoded constants + */ + kgsl_regmap_read_indexed(&device->regmap, A3XX_CP_PFP_UCODE_ADDR, + A3XX_CP_PFP_UCODE_DATA, data, size); + + return DEBUG_SECTION_SZ(size); +} + +static size_t a3xx_snapshot_cp_roq(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *) buf; + u32 *data = (u32 *) (buf + sizeof(*header)); + + if (remain < DEBUG_SECTION_SZ(128)) { + SNAPSHOT_ERR_NOMEM(device, "CP ROQ DEBUG"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_CP_ROQ; + header->size = 128; + + kgsl_regmap_read_indexed(&device->regmap, A3XX_CP_ROQ_ADDR, + A3XX_CP_ROQ_DATA, data, 128); + + return DEBUG_SECTION_SZ(128); +} + +static size_t a3xx_snapshot_cp_meq(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *) buf; + u32 *data = (u32 *) (buf + sizeof(*header)); + + if (remain < DEBUG_SECTION_SZ(16)) { + SNAPSHOT_ERR_NOMEM(device, "CP MEQ DEBUG"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_CP_MEQ; + header->size = 16; + + kgsl_regmap_read_indexed(&device->regmap, A3XX_CP_MEQ_ADDR, + A3XX_CP_MEQ_DATA, data, 16); + + return DEBUG_SECTION_SZ(16); +} + +/* + * a3xx_snapshot() - A3XX GPU snapshot function + * @adreno_dev: Device being snapshotted + * @snapshot: Snapshot metadata + * @remain: Amount of space left in snapshot memory + * + * This is where all of the A3XX specific bits and pieces are grabbed + * into the snapshot memory + */ +void a3xx_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg; + + /* Disable Clock gating temporarily for the debug bus to work */ + kgsl_regwrite(device, A3XX_RBBM_CLOCK_CTL, 0x0); + + /* Save some CP information that the generic snapshot uses */ + kgsl_regread(device, A3XX_CP_IB1_BASE, ®); + snapshot->ib1base = (u64) reg; + + kgsl_regread(device, A3XX_CP_IB2_BASE, ®); + snapshot->ib2base = (u64) reg; + + kgsl_regread(device, A3XX_CP_IB1_BUFSZ, &snapshot->ib1size); + kgsl_regread(device, A3XX_CP_IB2_BUFSZ, &snapshot->ib2size); + + SNAPSHOT_REGISTERS(device, snapshot, a3xx_registers); + + _snapshot_hlsq_regs(device, snapshot); + + kgsl_snapshot_indexed_registers(device, snapshot, + A3XX_CP_STATE_DEBUG_INDEX, A3XX_CP_STATE_DEBUG_DATA, 0, 0x14); + + /* CP_ME indexed registers */ + kgsl_snapshot_indexed_registers(device, snapshot, + A3XX_CP_ME_CNTL, A3XX_CP_ME_STATUS, 64, 44); + + /* VPC memory */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a3xx_snapshot_vpc_memory, NULL); + + /* CP MEQ */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, snapshot, + a3xx_snapshot_cp_meq, NULL); + + /* Shader working/shadow memory */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a3xx_snapshot_shader_memory, NULL); + + + /* CP PFP and PM4 */ + + /* + * Reading the microcode while the CP is running will + * basically move the CP instruction pointer to + * whatever address we read. Big badaboom ensues. Stop the CP + * (if it isn't already stopped) to ensure that we are safe. + * We do this here and not earlier to avoid corrupting the RBBM + * status and CP registers - by the time we get here we don't + * care about the contents of the CP anymore. + */ + + kgsl_regread(device, A3XX_CP_ME_CNTL, ®); + reg |= (1 << 27) | (1 << 28); + kgsl_regwrite(device, A3XX_CP_ME_CNTL, reg); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a3xx_snapshot_cp_pfp_ram, NULL); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a3xx_snapshot_cp_pm4_ram, NULL); + + /* CP ROQ */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a3xx_snapshot_cp_roq, NULL); + + a3xx_snapshot_debugbus(device, snapshot); +} diff --git a/qcom/opensource/graphics-kernel/adreno_a5xx.c b/qcom/opensource/graphics-kernel/adreno_a5xx.c new file mode 100644 index 0000000000..07bf27b505 --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a5xx.c @@ -0,0 +1,2500 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2014-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "adreno.h" +#include "adreno_a5xx.h" +#include "adreno_a5xx_packets.h" +#include "adreno_pm4types.h" +#include "adreno_trace.h" +#include "kgsl_trace.h" + +static int critical_packet_constructed; +static unsigned int crit_pkts_dwords; + +static void a5xx_irq_storm_worker(struct work_struct *work); +static int _read_fw2_block_header(struct kgsl_device *device, + uint32_t *header, uint32_t remain, + uint32_t id, uint32_t major, uint32_t minor); +static void a5xx_gpmu_reset(struct work_struct *work); +static int a5xx_gpmu_init(struct adreno_device *adreno_dev); + +/** + * Number of times to check if the regulator enabled before + * giving up and returning failure. + */ +#define PWR_RETRY 100 + +/** + * Number of times to check if the GPMU firmware is initialized before + * giving up and returning failure. + */ +#define GPMU_FW_INIT_RETRY 5000 + +#define A530_QFPROM_RAW_PTE_ROW0_MSB 0x134 +#define A530_QFPROM_RAW_PTE_ROW2_MSB 0x144 + +#define A5XX_INT_MASK \ + ((1 << A5XX_INT_RBBM_AHB_ERROR) | \ + (1 << A5XX_INT_RBBM_TRANSFER_TIMEOUT) | \ + (1 << A5XX_INT_RBBM_ME_MS_TIMEOUT) | \ + (1 << A5XX_INT_RBBM_PFP_MS_TIMEOUT) | \ + (1 << A5XX_INT_RBBM_ETS_MS_TIMEOUT) | \ + (1 << A5XX_INT_RBBM_ATB_ASYNC_OVERFLOW) | \ + (1 << A5XX_INT_RBBM_GPC_ERROR) | \ + (1 << A5XX_INT_CP_HW_ERROR) | \ + (1 << A5XX_INT_CP_CACHE_FLUSH_TS) | \ + (1 << A5XX_INT_RBBM_ATB_BUS_OVERFLOW) | \ + (1 << A5XX_INT_MISC_HANG_DETECT) | \ + (1 << A5XX_INT_UCHE_OOB_ACCESS) | \ + (1 << A5XX_INT_UCHE_TRAP_INTR) | \ + (1 << A5XX_INT_CP_SW) | \ + (1 << A5XX_INT_GPMU_FIRMWARE) | \ + (1 << A5XX_INT_GPMU_VOLTAGE_DROOP)) + +static int a5xx_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore) +{ + struct adreno_device *adreno_dev; + struct kgsl_device *device; + int ret; + + adreno_dev = (struct adreno_device *) + of_device_get_match_data(&pdev->dev); + + memset(adreno_dev, 0, sizeof(*adreno_dev)); + + adreno_dev->gpucore = gpucore; + adreno_dev->chipid = chipid; + + adreno_reg_offset_init(gpucore->gpudev->reg_offsets); + + adreno_dev->sptp_pc_enabled = + ADRENO_FEATURE(adreno_dev, ADRENO_SPTP_PC); + + if (adreno_is_a540(adreno_dev)) + adreno_dev->throttling_enabled = true; + + adreno_dev->hwcg_enabled = true; + adreno_dev->lm_enabled = + ADRENO_FEATURE(adreno_dev, ADRENO_LM); + + /* Setup defaults that might get changed by the fuse bits */ + adreno_dev->lm_leakage = 0x4e001a; + + device = KGSL_DEVICE(adreno_dev); + + timer_setup(&device->idle_timer, kgsl_timer, 0); + + INIT_WORK(&device->idle_check_ws, kgsl_idle_check); + + adreno_dev->irq_mask = A5XX_INT_MASK; + + ret = adreno_device_probe(pdev, adreno_dev); + if (ret) + return ret; + + a5xx_coresight_init(adreno_dev); + + return adreno_dispatcher_init(adreno_dev); +} + +static void _do_fixup(const struct adreno_critical_fixup *fixups, int count, + uint64_t *gpuaddrs, unsigned int *buffer) +{ + int i; + + for (i = 0; i < count; i++) { + buffer[fixups[i].lo_offset] = + lower_32_bits(gpuaddrs[fixups[i].buffer]) | + fixups[i].mem_offset; + + buffer[fixups[i].hi_offset] = + upper_32_bits(gpuaddrs[fixups[i].buffer]); + } +} + +static int a5xx_critical_packet_construct(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int *cmds; + uint64_t gpuaddrs[4]; + + adreno_dev->critpkts = kgsl_allocate_global(device, + PAGE_SIZE * 4, 0, 0, 0, "crit_pkts"); + if (IS_ERR(adreno_dev->critpkts)) + return PTR_ERR(adreno_dev->critpkts); + + adreno_dev->critpkts_secure = kgsl_allocate_global(device, + PAGE_SIZE, 0, KGSL_MEMFLAGS_SECURE, 0, "crit_pkts_secure"); + if (IS_ERR(adreno_dev->critpkts_secure)) + return PTR_ERR(adreno_dev->critpkts_secure); + + cmds = adreno_dev->critpkts->hostptr; + + gpuaddrs[0] = adreno_dev->critpkts_secure->gpuaddr; + gpuaddrs[1] = adreno_dev->critpkts->gpuaddr + PAGE_SIZE; + gpuaddrs[2] = adreno_dev->critpkts->gpuaddr + (PAGE_SIZE * 2); + gpuaddrs[3] = adreno_dev->critpkts->gpuaddr + (PAGE_SIZE * 3); + + crit_pkts_dwords = ARRAY_SIZE(_a5xx_critical_pkts); + + memcpy(cmds, _a5xx_critical_pkts, crit_pkts_dwords << 2); + + _do_fixup(critical_pkt_fixups, ARRAY_SIZE(critical_pkt_fixups), + gpuaddrs, cmds); + + cmds = adreno_dev->critpkts->hostptr + PAGE_SIZE; + memcpy(cmds, _a5xx_critical_pkts_mem01, + ARRAY_SIZE(_a5xx_critical_pkts_mem01) << 2); + + cmds = adreno_dev->critpkts->hostptr + (PAGE_SIZE * 2); + memcpy(cmds, _a5xx_critical_pkts_mem02, + ARRAY_SIZE(_a5xx_critical_pkts_mem02) << 2); + + cmds = adreno_dev->critpkts->hostptr + (PAGE_SIZE * 3); + memcpy(cmds, _a5xx_critical_pkts_mem03, + ARRAY_SIZE(_a5xx_critical_pkts_mem03) << 2); + + _do_fixup(critical_pkt_mem03_fixups, + ARRAY_SIZE(critical_pkt_mem03_fixups), gpuaddrs, cmds); + + critical_packet_constructed = 1; + + return 0; +} + +static int a5xx_microcode_read(struct adreno_device *adreno_dev); + +static int a5xx_init(struct adreno_device *adreno_dev) +{ + const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev); + int ret; + + ret = a5xx_ringbuffer_init(adreno_dev); + if (ret) + return ret; + + ret = a5xx_microcode_read(adreno_dev); + if (ret) + return ret; + + if (a5xx_has_gpmu(adreno_dev)) + INIT_WORK(&adreno_dev->gpmu_work, a5xx_gpmu_reset); + + adreno_dev->highest_bank_bit = a5xx_core->highest_bank_bit; + + INIT_WORK(&adreno_dev->irq_storm_work, a5xx_irq_storm_worker); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_CRITICAL_PACKETS)) + a5xx_critical_packet_construct(adreno_dev); + + adreno_create_profile_buffer(adreno_dev); + a5xx_crashdump_init(adreno_dev); + + return 0; +} + +static const struct { + u32 reg; + u32 base; + u32 count; +} a5xx_protected_blocks[] = { + /* RBBM */ + { A5XX_CP_PROTECT_REG_0, 0x004, 2 }, + { A5XX_CP_PROTECT_REG_0 + 1, 0x008, 3 }, + { A5XX_CP_PROTECT_REG_0 + 2, 0x010, 4 }, + { A5XX_CP_PROTECT_REG_0 + 3, 0x020, 5 }, + { A5XX_CP_PROTECT_REG_0 + 4, 0x040, 6 }, + { A5XX_CP_PROTECT_REG_0 + 5, 0x080, 6 }, + /* Content protection */ + { A5XX_CP_PROTECT_REG_0 + 6, A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 4 }, + { A5XX_CP_PROTECT_REG_0 + 7, A5XX_RBBM_SECVID_TRUST_CNTL, 1 }, + /* CP */ + { A5XX_CP_PROTECT_REG_0 + 8, 0x800, 6 }, + { A5XX_CP_PROTECT_REG_0 + 9, 0x840, 3 }, + { A5XX_CP_PROTECT_REG_0 + 10, 0x880, 5 }, + { A5XX_CP_PROTECT_REG_0 + 11, 0xaa0, 0 }, + /* RB */ + { A5XX_CP_PROTECT_REG_0 + 12, 0xcc0, 0 }, + { A5XX_CP_PROTECT_REG_0 + 13, 0xcf0, 1 }, + /* VPC */ + { A5XX_CP_PROTECT_REG_0 + 14, 0xe68, 3 }, + { A5XX_CP_PROTECT_REG_0 + 15, 0xe70, 4 }, + /* UCHE */ + { A5XX_CP_PROTECT_REG_0 + 16, 0xe80, 4 }, + /* A5XX_CP_PROTECT_REG_17 will be used for SMMU */ + /* A5XX_CP_PROTECT_REG_18 - A5XX_CP_PROTECT_REG_31 are available */ +}; + +static void _setprotectreg(struct kgsl_device *device, u32 offset, + u32 base, u32 count) +{ + kgsl_regwrite(device, offset, 0x60000000 | (count << 24) | (base << 2)); +} + +static void a5xx_protect_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 reg; + int i; + + /* enable access protection to privileged registers */ + kgsl_regwrite(device, A5XX_CP_PROTECT_CNTL, 0x00000007); + + for (i = 0; i < ARRAY_SIZE(a5xx_protected_blocks); i++) { + reg = a5xx_protected_blocks[i].reg; + + _setprotectreg(device, reg, a5xx_protected_blocks[i].base, + a5xx_protected_blocks[i].count); + } + + /* + * For a530 and a540 the SMMU region is 0x20000 bytes long and 0x10000 + * bytes on all other targets. The base offset for both is 0x40000. + * Write it to the next available slot + */ + if (adreno_is_a530(adreno_dev) || adreno_is_a540(adreno_dev)) + _setprotectreg(device, reg + 1, 0x40000, ilog2(0x20000)); + else + _setprotectreg(device, reg + 1, 0x40000, ilog2(0x10000)); +} + +/* + * _poll_gdsc_status() - Poll the GDSC status register + * @adreno_dev: The adreno device pointer + * @status_reg: Offset of the status register + * @status_value: The expected bit value + * + * Poll the status register till the power-on bit is equal to the + * expected value or the max retries are exceeded. + */ +static int _poll_gdsc_status(struct adreno_device *adreno_dev, + unsigned int status_reg, + unsigned int status_value) +{ + unsigned int reg, retry = PWR_RETRY; + + /* Bit 20 is the power on bit of SPTP and RAC GDSC status register */ + do { + udelay(1); + kgsl_regread(KGSL_DEVICE(adreno_dev), status_reg, ®); + } while (((reg & BIT(20)) != (status_value << 20)) && retry--); + if ((reg & BIT(20)) != (status_value << 20)) + return -ETIMEDOUT; + return 0; +} + +static void a5xx_restore_isense_regs(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg, i, ramp = GPMU_ISENSE_SAVE; + static unsigned int isense_regs[6] = {0xFFFF}, isense_reg_addr[] = { + A5XX_GPU_CS_DECIMAL_ALIGN, + A5XX_GPU_CS_SENSOR_PARAM_CORE_1, + A5XX_GPU_CS_SENSOR_PARAM_CORE_2, + A5XX_GPU_CS_SW_OV_FUSE_EN, + A5XX_GPU_CS_ENDPOINT_CALIBRATION_DONE, + A5XX_GPMU_TEMP_SENSOR_CONFIG}; + + if (!adreno_is_a540(adreno_dev)) + return; + + /* read signature */ + kgsl_regread(device, ramp++, ®); + + if (reg == 0xBABEFACE) { + /* store memory locations in buffer */ + for (i = 0; i < ARRAY_SIZE(isense_regs); i++) + kgsl_regread(device, ramp + i, isense_regs + i); + + /* clear signature */ + kgsl_regwrite(device, GPMU_ISENSE_SAVE, 0x0); + } + + /* if we never stored memory locations - do nothing */ + if (isense_regs[0] == 0xFFFF) + return; + + /* restore registers from memory */ + for (i = 0; i < ARRAY_SIZE(isense_reg_addr); i++) + kgsl_regwrite(device, isense_reg_addr[i], isense_regs[i]); + +} + +/* + * a5xx_regulator_enable() - Enable any necessary HW regulators + * @adreno_dev: The adreno device pointer + * + * Some HW blocks may need their regulators explicitly enabled + * on a restart. Clocks must be on during this call. + */ +static int a5xx_regulator_enable(struct adreno_device *adreno_dev) +{ + unsigned int ret; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (test_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, + &adreno_dev->priv)) + return 0; + + if (!(adreno_is_a530(adreno_dev) || adreno_is_a540(adreno_dev))) { + /* Halt the sp_input_clk at HM level */ + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL, 0x00000055); + a5xx_hwcg_set(adreno_dev, true); + /* Turn on sp_input_clk at HM level */ + kgsl_regrmw(device, A5XX_RBBM_CLOCK_CNTL, 0xFF, 0); + + set_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, + &adreno_dev->priv); + return 0; + } + + /* + * Turn on smaller power domain first to reduce voltage droop. + * Set the default register values; set SW_COLLAPSE to 0. + */ + kgsl_regwrite(device, A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000); + /* Insert a delay between RAC and SPTP GDSC to reduce voltage droop */ + udelay(3); + ret = _poll_gdsc_status(adreno_dev, A5XX_GPMU_RBCCU_PWR_CLK_STATUS, 1); + if (ret) { + dev_err(device->dev, "RBCCU GDSC enable failed\n"); + return ret; + } + + kgsl_regwrite(device, A5XX_GPMU_SP_POWER_CNTL, 0x778000); + ret = _poll_gdsc_status(adreno_dev, A5XX_GPMU_SP_PWR_CLK_STATUS, 1); + if (ret) { + dev_err(device->dev, "SPTP GDSC enable failed\n"); + return ret; + } + + /* Disable SP clock */ + kgsl_regrmw(device, A5XX_GPMU_GPMU_SP_CLOCK_CONTROL, + CNTL_IP_CLK_ENABLE, 0); + /* Enable hardware clockgating */ + a5xx_hwcg_set(adreno_dev, true); + /* Enable SP clock */ + kgsl_regrmw(device, A5XX_GPMU_GPMU_SP_CLOCK_CONTROL, + CNTL_IP_CLK_ENABLE, 1); + + a5xx_restore_isense_regs(adreno_dev); + + set_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, &adreno_dev->priv); + return 0; +} + +/* + * a5xx_regulator_disable() - Disable any necessary HW regulators + * @adreno_dev: The adreno device pointer + * + * Some HW blocks may need their regulators explicitly disabled + * on a power down to prevent current spikes. Clocks must be on + * during this call. + */ +static void a5xx_regulator_disable(struct adreno_device *adreno_dev) +{ + unsigned int reg; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (adreno_is_a512(adreno_dev) || adreno_is_a508(adreno_dev)) + return; + + if (!test_and_clear_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, + &adreno_dev->priv)) + return; + + /* If feature is not supported or not enabled */ + if (!adreno_dev->sptp_pc_enabled) { + /* Set the default register values; set SW_COLLAPSE to 1 */ + kgsl_regwrite(device, A5XX_GPMU_SP_POWER_CNTL, 0x778001); + /* + * Insert a delay between SPTP and RAC GDSC to reduce voltage + * droop. + */ + udelay(3); + if (_poll_gdsc_status(adreno_dev, + A5XX_GPMU_SP_PWR_CLK_STATUS, 0)) + dev_warn(device->dev, "SPTP GDSC disable failed\n"); + + kgsl_regwrite(device, A5XX_GPMU_RBCCU_POWER_CNTL, 0x778001); + if (_poll_gdsc_status(adreno_dev, + A5XX_GPMU_RBCCU_PWR_CLK_STATUS, 0)) + dev_warn(device->dev, "RBCCU GDSC disable failed\n"); + } else if (test_bit(ADRENO_DEVICE_GPMU_INITIALIZED, + &adreno_dev->priv)) { + /* GPMU firmware is supposed to turn off SPTP & RAC GDSCs. */ + kgsl_regread(device, A5XX_GPMU_SP_PWR_CLK_STATUS, ®); + if (reg & BIT(20)) + dev_warn(device->dev, "SPTP GDSC is not disabled\n"); + kgsl_regread(device, A5XX_GPMU_RBCCU_PWR_CLK_STATUS, ®); + if (reg & BIT(20)) + dev_warn(device->dev, "RBCCU GDSC is not disabled\n"); + /* + * GPMU firmware is supposed to set GMEM to non-retention. + * Bit 14 is the memory core force on bit. + */ + kgsl_regread(device, A5XX_GPMU_RBCCU_CLOCK_CNTL, ®); + if (reg & BIT(14)) + dev_warn(device->dev, "GMEM is forced on\n"); + } + + if (adreno_is_a530(adreno_dev)) { + /* Reset VBIF before PC to avoid popping bogus FIFO entries */ + kgsl_regwrite(device, A5XX_RBBM_BLOCK_SW_RESET_CMD, + 0x003C0000); + kgsl_regwrite(device, A5XX_RBBM_BLOCK_SW_RESET_CMD, 0); + } +} + +/* + * a5xx_enable_pc() - Enable the GPMU based power collapse of the SPTP and RAC + * blocks + * @adreno_dev: The adreno device pointer + */ +static void a5xx_enable_pc(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!adreno_dev->sptp_pc_enabled) + return; + + kgsl_regwrite(device, A5XX_GPMU_PWR_COL_INTER_FRAME_CTRL, 0x0000007F); + kgsl_regwrite(device, A5XX_GPMU_PWR_COL_BINNING_CTRL, 0); + kgsl_regwrite(device, A5XX_GPMU_PWR_COL_INTER_FRAME_HYST, 0x000A0080); + kgsl_regwrite(device, A5XX_GPMU_PWR_COL_STAGGER_DELAY, 0x00600040); + + trace_adreno_sp_tp((unsigned long) __builtin_return_address(0)); +}; + +/* + * The maximum payload of a type4 packet is the max size minus one for the + * opcode + */ +#define TYPE4_MAX_PAYLOAD (PM4_TYPE4_PKT_SIZE_MAX - 1) + +static int _gpmu_create_load_cmds(struct adreno_device *adreno_dev, + uint32_t *ucode, uint32_t size) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + uint32_t *start, *cmds; + uint32_t offset = 0; + uint32_t cmds_size = size; + + /* Add a dword for each PM4 packet */ + cmds_size += (size / TYPE4_MAX_PAYLOAD) + 1; + + /* Add 4 dwords for the protected mode */ + cmds_size += 4; + + if (adreno_dev->gpmu_cmds != NULL) + return 0; + + adreno_dev->gpmu_cmds = devm_kmalloc(&device->pdev->dev, + cmds_size << 2, GFP_KERNEL); + if (adreno_dev->gpmu_cmds == NULL) + return -ENOMEM; + + cmds = adreno_dev->gpmu_cmds; + start = cmds; + + /* Turn CP protection OFF */ + cmds += cp_protected_mode(adreno_dev, cmds, 0); + + /* + * Prebuild the cmd stream to send to the GPU to load + * the GPMU firmware + */ + while (size > 0) { + int tmp_size = size; + + if (size >= TYPE4_MAX_PAYLOAD) + tmp_size = TYPE4_MAX_PAYLOAD; + + *cmds++ = cp_type4_packet( + A5XX_GPMU_INST_RAM_BASE + offset, + tmp_size); + + memcpy(cmds, &ucode[offset], tmp_size << 2); + + cmds += tmp_size; + offset += tmp_size; + size -= tmp_size; + } + + /* Turn CP protection ON */ + cmds += cp_protected_mode(adreno_dev, cmds, 1); + + adreno_dev->gpmu_cmds_size = (size_t) (cmds - start); + + return 0; +} + + +/* + * _load_gpmu_firmware() - Load the ucode into the GPMU RAM + * @adreno_dev: Pointer to adreno device + */ +static int _load_gpmu_firmware(struct adreno_device *adreno_dev) +{ + uint32_t *data; + const struct firmware *fw = NULL; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev); + uint32_t *cmds, cmd_size; + int ret = -EINVAL; + u32 gmu_major = 1; + + if (!a5xx_has_gpmu(adreno_dev)) + return 0; + + /* a530 used GMU major 1 and A540 used GMU major 3 */ + if (adreno_is_a540(adreno_dev)) + gmu_major = 3; + + /* gpmu fw already saved and verified so do nothing new */ + if (adreno_dev->gpmu_cmds_size != 0) + return 0; + + if (a5xx_core->gpmufw_name == NULL) + return 0; + + ret = request_firmware(&fw, a5xx_core->gpmufw_name, &device->pdev->dev); + if (ret || fw == NULL) { + dev_err(&device->pdev->dev, + "request_firmware (%s) failed: %d\n", + a5xx_core->gpmufw_name, ret); + return ret; + } + + data = (uint32_t *)fw->data; + + if (data[0] >= (fw->size / sizeof(uint32_t)) || data[0] < 2) + goto err; + + if (data[1] != GPMU_FIRMWARE_ID) + goto err; + ret = _read_fw2_block_header(device, &data[2], + data[0] - 2, GPMU_FIRMWARE_ID, gmu_major, 0); + if (ret) + goto err; + + /* Integer overflow check for cmd_size */ + if (data[2] > (data[0] - 2)) + goto err; + + cmds = data + data[2] + 3; + cmd_size = data[0] - data[2] - 2; + + if (cmd_size > GPMU_INST_RAM_SIZE) { + dev_err(device->dev, + "GPMU firmware block size is larger than RAM size\n"); + goto err; + } + + /* Everything is cool, so create some commands */ + ret = _gpmu_create_load_cmds(adreno_dev, cmds, cmd_size); +err: + if (fw) + release_firmware(fw); + + return ret; +} + +static void a5xx_spin_idle_debug(struct adreno_device *adreno_dev, + const char *str) +{ + struct kgsl_device *device = &adreno_dev->dev; + unsigned int rptr, wptr; + unsigned int status, status3, intstatus; + unsigned int hwfault; + + dev_err(device->dev, str); + + kgsl_regread(device, A5XX_CP_RB_RPTR, &rptr); + kgsl_regread(device, A5XX_CP_RB_WPTR, &wptr); + + kgsl_regread(device, A5XX_RBBM_STATUS, &status); + kgsl_regread(device, A5XX_RBBM_STATUS3, &status3); + kgsl_regread(device, A5XX_RBBM_INT_0_STATUS, &intstatus); + kgsl_regread(device, A5XX_CP_HW_FAULT, &hwfault); + + + dev_err(device->dev, + "rb=%d pos=%X/%X rbbm_status=%8.8X/%8.8X int_0_status=%8.8X\n", + adreno_dev->cur_rb->id, rptr, wptr, status, status3, intstatus); + + dev_err(device->dev, " hwfault=%8.8X\n", hwfault); + + kgsl_device_snapshot(device, NULL, NULL, false); +} + +static int _gpmu_send_init_cmds(struct adreno_device *adreno_dev) +{ + struct adreno_ringbuffer *rb = adreno_dev->cur_rb; + uint32_t *cmds; + uint32_t size = adreno_dev->gpmu_cmds_size; + int ret; + + if (size == 0 || adreno_dev->gpmu_cmds == NULL) + return -EINVAL; + + cmds = adreno_ringbuffer_allocspace(rb, size); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + if (cmds == NULL) + return -ENOSPC; + + /* Copy to the RB the predefined fw sequence cmds */ + memcpy(cmds, adreno_dev->gpmu_cmds, size << 2); + + ret = a5xx_ringbuffer_submit(rb, NULL, true); + if (!ret) { + ret = adreno_spin_idle(adreno_dev, 2000); + if (ret) + a5xx_spin_idle_debug(adreno_dev, + "gpmu initialization failed to idle\n"); + } + return ret; +} + +/* + * a5xx_gpmu_start() - Initialize and start the GPMU + * @adreno_dev: Pointer to adreno device + * + * Load the GPMU microcode, set up any features such as hardware clock gating + * or IFPC, and take the GPMU out of reset. + */ +static int a5xx_gpmu_start(struct adreno_device *adreno_dev) +{ + int ret; + unsigned int reg, retry = GPMU_FW_INIT_RETRY; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!a5xx_has_gpmu(adreno_dev)) + return 0; + + ret = _gpmu_send_init_cmds(adreno_dev); + if (ret) + return ret; + + if (adreno_is_a530(adreno_dev)) { + /* GPMU clock gating setup */ + kgsl_regwrite(device, A5XX_GPMU_WFI_CONFIG, 0x00004014); + } + /* Kick off GPMU firmware */ + kgsl_regwrite(device, A5XX_GPMU_CM3_SYSRESET, 0); + /* + * The hardware team's estimation of GPMU firmware initialization + * latency is about 3000 cycles, that's about 5 to 24 usec. + */ + do { + udelay(1); + kgsl_regread(device, A5XX_GPMU_GENERAL_0, ®); + } while ((reg != 0xBABEFACE) && retry--); + + if (reg != 0xBABEFACE) { + dev_err(device->dev, + "GPMU firmware initialization timed out\n"); + return -ETIMEDOUT; + } + + if (!adreno_is_a530(adreno_dev)) { + kgsl_regread(device, A5XX_GPMU_GENERAL_1, ®); + + if (reg) { + dev_err(device->dev, + "GPMU firmware initialization failed: %d\n", + reg); + return -EIO; + } + } + set_bit(ADRENO_DEVICE_GPMU_INITIALIZED, &adreno_dev->priv); + /* + * We are in AWARE state and IRQ line from GPU to host is + * disabled. + * Read pending GPMU interrupts and clear GPMU_RBBM_INTR_INFO. + */ + kgsl_regread(device, A5XX_GPMU_RBBM_INTR_INFO, ®); + /* + * Clear RBBM interrupt mask if any of GPMU interrupts + * are pending. + */ + if (reg) + kgsl_regwrite(device, + A5XX_RBBM_INT_CLEAR_CMD, + 1 << A5XX_INT_GPMU_FIRMWARE); + return ret; +} + +void a5xx_hwcg_set(struct adreno_device *adreno_dev, bool on) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev); + int i; + + if (!adreno_dev->hwcg_enabled) + return; + + for (i = 0; i < a5xx_core->hwcg_count; i++) + kgsl_regwrite(device, a5xx_core->hwcg[i].offset, + on ? a5xx_core->hwcg[i].val : 0); + + /* enable top level HWCG */ + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL, on ? 0xAAA8AA00 : 0); + kgsl_regwrite(device, A5XX_RBBM_ISDB_CNT, on ? 0x00000182 : 0x00000180); +} + +static int _read_fw2_block_header(struct kgsl_device *device, + uint32_t *header, uint32_t remain, + uint32_t id, uint32_t major, uint32_t minor) +{ + uint32_t header_size; + int i = 1; + + if (header == NULL) + return -ENOMEM; + + header_size = header[0]; + /* Headers have limited size and always occur as pairs of words */ + if (header_size > MAX_HEADER_SIZE || header_size >= remain || + header_size % 2 || header_size == 0) + return -EINVAL; + /* Sequences must have an identifying id first thing in their header */ + if (id == GPMU_SEQUENCE_ID) { + if (header[i] != HEADER_SEQUENCE || + (header[i + 1] >= MAX_SEQUENCE_ID)) + return -EINVAL; + i += 2; + } + for (; i < header_size; i += 2) { + switch (header[i]) { + /* Major Version */ + case HEADER_MAJOR: + if ((major > header[i + 1]) && + header[i + 1]) { + dev_err(device->dev, + "GPMU major version mis-match %d, %d\n", + major, header[i + 1]); + return -EINVAL; + } + break; + case HEADER_MINOR: + if (minor > header[i + 1]) + dev_err(device->dev, + "GPMU minor version mis-match %d %d\n", + minor, header[i + 1]); + break; + case HEADER_DATE: + case HEADER_TIME: + break; + default: + dev_err(device->dev, "GPMU unknown header ID %d\n", + header[i]); + } + } + return 0; +} + +/* + * Read in the register sequence file and save pointers to the + * necessary sequences. + * + * GPU sequence file format (one dword per field unless noted): + * Block 1 length (length dword field not inclusive) + * Block 1 type = Sequence = 3 + * Block Header length (length dword field not inclusive) + * BH field ID = Sequence field ID + * BH field data = Sequence ID + * BH field ID + * BH field data + * ... + * Opcode 0 ID + * Opcode 0 data M words + * Opcode 1 ID + * Opcode 1 data N words + * ... + * Opcode X ID + * Opcode X data O words + * Block 2 length... + */ +static void _load_regfile(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev); + const struct firmware *fw; + uint64_t block_size = 0, block_total = 0; + uint32_t fw_size, *block; + int ret = -EINVAL; + u32 lm_major = 1; + + if (!a5xx_core->regfw_name) + return; + + ret = request_firmware(&fw, a5xx_core->regfw_name, &device->pdev->dev); + if (ret) { + dev_err(&device->pdev->dev, "request firmware failed %d, %s\n", + ret, a5xx_core->regfw_name); + return; + } + + /* a530v2 lm_major was 3. a530v3 lm_major was 1 */ + if (adreno_is_a530v2(adreno_dev)) + lm_major = 3; + + fw_size = fw->size / sizeof(uint32_t); + /* Min valid file of size 6, see file description */ + if (fw_size < 6) + goto err; + block = (uint32_t *)fw->data; + /* All offset numbers calculated from file description */ + while (block_total < fw_size) { + block_size = block[0]; + if (((block_total + block_size) >= fw_size) + || block_size < 5) + goto err; + if (block[1] != GPMU_SEQUENCE_ID) + goto err; + + /* For now ignore blocks other than the LM sequence */ + if (block[4] == LM_SEQUENCE_ID) { + ret = _read_fw2_block_header(device, &block[2], + block_size - 2, GPMU_SEQUENCE_ID, + lm_major, 0); + if (ret) + goto err; + + if (block[2] > (block_size - 2)) + goto err; + adreno_dev->lm_sequence = block + block[2] + 3; + adreno_dev->lm_size = block_size - block[2] - 2; + } + block_total += (block_size + 1); + block += (block_size + 1); + } + if (adreno_dev->lm_sequence) + return; + +err: + release_firmware(fw); + dev_err(device->dev, + "Register file failed to load sz=%d bsz=%llu header=%d\n", + fw_size, block_size, ret); +} + +static int _execute_reg_sequence(struct adreno_device *adreno_dev, + uint32_t *opcode, uint32_t length) +{ + uint32_t *cur = opcode; + uint64_t reg, val; + + /* todo double check the reg writes */ + while ((cur - opcode) < length) { + if (cur[0] == 1 && (length - (cur - opcode) >= 4)) { + /* Write a 32 bit value to a 64 bit reg */ + reg = cur[2]; + reg = (reg << 32) | cur[1]; + kgsl_regwrite(KGSL_DEVICE(adreno_dev), reg, cur[3]); + cur += 4; + } else if (cur[0] == 2 && (length - (cur - opcode) >= 5)) { + /* Write a 64 bit value to a 64 bit reg */ + reg = cur[2]; + reg = (reg << 32) | cur[1]; + val = cur[4]; + val = (val << 32) | cur[3]; + kgsl_regwrite(KGSL_DEVICE(adreno_dev), reg, val); + cur += 5; + } else if (cur[0] == 3 && (length - (cur - opcode) >= 2)) { + /* Delay for X usec */ + udelay(cur[1]); + cur += 2; + } else + return -EINVAL; + } + return 0; +} + +static uint32_t _write_voltage_table(struct adreno_device *adreno_dev, + unsigned int addr) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev); + int i; + struct dev_pm_opp *opp; + unsigned int mvolt = 0; + + kgsl_regwrite(device, addr++, a5xx_core->max_power); + kgsl_regwrite(device, addr++, pwr->num_pwrlevels); + + /* Write voltage in mV and frequency in MHz */ + for (i = 0; i < pwr->num_pwrlevels; i++) { + opp = dev_pm_opp_find_freq_exact(&device->pdev->dev, + pwr->pwrlevels[i].gpu_freq, true); + /* _opp_get returns uV, convert to mV */ + if (!IS_ERR(opp)) { + mvolt = dev_pm_opp_get_voltage(opp) / 1000; + dev_pm_opp_put(opp); + } + kgsl_regwrite(device, addr++, mvolt); + kgsl_regwrite(device, addr++, + pwr->pwrlevels[i].gpu_freq / 1000000); + } + return (pwr->num_pwrlevels * 2 + 2); +} + +static uint32_t lm_limit(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (adreno_dev->lm_limit) + return adreno_dev->lm_limit; + + if (of_property_read_u32(device->pdev->dev.of_node, "qcom,lm-limit", + &adreno_dev->lm_limit)) + adreno_dev->lm_limit = LM_DEFAULT_LIMIT; + + return adreno_dev->lm_limit; +} +/* + * a5xx_lm_init() - Initialize LM/DPM on the GPMU + * @adreno_dev: The adreno device pointer + */ +static void a530_lm_init(struct adreno_device *adreno_dev) +{ + uint32_t length; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev); + + if (!adreno_dev->lm_enabled) + return; + + /* If something was wrong with the sequence file, return */ + if (adreno_dev->lm_sequence == NULL) + return; + + /* Write LM registers including DPM ucode, coefficients, and config */ + if (_execute_reg_sequence(adreno_dev, adreno_dev->lm_sequence, + adreno_dev->lm_size)) { + /* If the sequence is invalid, it's not getting better */ + adreno_dev->lm_sequence = NULL; + dev_warn(device->dev, + "Invalid LM sequence\n"); + return; + } + + kgsl_regwrite(device, A5XX_GPMU_TEMP_SENSOR_ID, a5xx_core->gpmu_tsens); + kgsl_regwrite(device, A5XX_GPMU_DELTA_TEMP_THRESHOLD, 0x1); + kgsl_regwrite(device, A5XX_GPMU_TEMP_SENSOR_CONFIG, 0x1); + + kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE, + (0x80000000 | device->pwrctrl.active_pwrlevel)); + /* use the leakage to set this value at runtime */ + kgsl_regwrite(device, A5XX_GPMU_BASE_LEAKAGE, + adreno_dev->lm_leakage); + + /* Enable the power threshold and set it to 6000m */ + kgsl_regwrite(device, A5XX_GPMU_GPMU_PWR_THRESHOLD, + 0x80000000 | lm_limit(adreno_dev)); + + kgsl_regwrite(device, A5XX_GPMU_BEC_ENABLE, 0x10001FFF); + kgsl_regwrite(device, A5XX_GDPM_CONFIG1, 0x00201FF1); + + /* Send an initial message to the GPMU with the LM voltage table */ + kgsl_regwrite(device, AGC_MSG_STATE, 1); + kgsl_regwrite(device, AGC_MSG_COMMAND, AGC_POWER_CONFIG_PRODUCTION_ID); + length = _write_voltage_table(adreno_dev, AGC_MSG_PAYLOAD); + kgsl_regwrite(device, AGC_MSG_PAYLOAD_SIZE, length * sizeof(uint32_t)); + kgsl_regwrite(device, AGC_INIT_MSG_MAGIC, AGC_INIT_MSG_VALUE); +} + +/* + * a5xx_lm_enable() - Enable the LM/DPM feature on the GPMU + * @adreno_dev: The adreno device pointer + */ +static void a530_lm_enable(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!adreno_dev->lm_enabled) + return; + + /* If no sequence properly initialized, return */ + if (adreno_dev->lm_sequence == NULL) + return; + + kgsl_regwrite(device, A5XX_GDPM_INT_MASK, 0x00000000); + kgsl_regwrite(device, A5XX_GDPM_INT_EN, 0x0000000A); + kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK, 0x00000001); + kgsl_regwrite(device, A5XX_GPMU_TEMP_THRESHOLD_INTR_EN_MASK, + 0x00050000); + kgsl_regwrite(device, A5XX_GPMU_THROTTLE_UNMASK_FORCE_CTRL, + 0x00030000); + + if (adreno_is_a530(adreno_dev)) + /* Program throttle control, do not enable idle DCS on v3+ */ + kgsl_regwrite(device, A5XX_GPMU_CLOCK_THROTTLE_CTRL, + adreno_is_a530v2(adreno_dev) ? 0x00060011 : 0x00000011); +} + +static void a540_lm_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + uint32_t agc_lm_config = AGC_BCL_DISABLED | + ((ADRENO_CHIPID_PATCH(adreno_dev->chipid) & 0x3) + << AGC_GPU_VERSION_SHIFT); + unsigned int r; + + if (!adreno_dev->throttling_enabled) + agc_lm_config |= AGC_THROTTLE_DISABLE; + + if (adreno_dev->lm_enabled) { + agc_lm_config |= + AGC_LM_CONFIG_ENABLE_GPMU_ADAPTIVE | + AGC_LM_CONFIG_ISENSE_ENABLE; + + kgsl_regread(device, A5XX_GPMU_TEMP_SENSOR_CONFIG, &r); + + if ((r & GPMU_ISENSE_STATUS) == GPMU_ISENSE_END_POINT_CAL_ERR) { + dev_err(device->dev, + "GPMU: ISENSE end point calibration failure\n"); + agc_lm_config |= AGC_LM_CONFIG_ENABLE_ERROR; + } + } + + kgsl_regwrite(device, AGC_MSG_STATE, 0x80000001); + kgsl_regwrite(device, AGC_MSG_COMMAND, AGC_POWER_CONFIG_PRODUCTION_ID); + (void) _write_voltage_table(adreno_dev, AGC_MSG_PAYLOAD); + kgsl_regwrite(device, AGC_MSG_PAYLOAD + AGC_LM_CONFIG, agc_lm_config); + kgsl_regwrite(device, AGC_MSG_PAYLOAD + AGC_LEVEL_CONFIG, + (unsigned int) ~(GENMASK(LM_DCVS_LIMIT, 0) | + GENMASK(16+LM_DCVS_LIMIT, 16))); + + kgsl_regwrite(device, AGC_MSG_PAYLOAD_SIZE, + (AGC_LEVEL_CONFIG + 1) * sizeof(uint32_t)); + kgsl_regwrite(device, AGC_INIT_MSG_MAGIC, AGC_INIT_MSG_VALUE); + + kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE, + (0x80000000 | device->pwrctrl.active_pwrlevel)); + + kgsl_regwrite(device, A5XX_GPMU_GPMU_PWR_THRESHOLD, + PWR_THRESHOLD_VALID | lm_limit(adreno_dev)); + + kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK, + VOLTAGE_INTR_EN); +} + + +static void a5xx_lm_enable(struct adreno_device *adreno_dev) +{ + if (adreno_is_a530(adreno_dev)) + a530_lm_enable(adreno_dev); +} + +static void a5xx_lm_init(struct adreno_device *adreno_dev) +{ + if (adreno_is_a530(adreno_dev)) + a530_lm_init(adreno_dev); + else if (adreno_is_a540(adreno_dev)) + a540_lm_init(adreno_dev); +} + +static int gpmu_set_level(struct adreno_device *adreno_dev, unsigned int val) +{ + unsigned int reg; + int retry = 100; + + kgsl_regwrite(KGSL_DEVICE(adreno_dev), A5XX_GPMU_GPMU_VOLTAGE, val); + + do { + kgsl_regread(KGSL_DEVICE(adreno_dev), A5XX_GPMU_GPMU_VOLTAGE, + ®); + } while ((reg & 0x80000000) && retry--); + + return (reg & 0x80000000) ? -ETIMEDOUT : 0; +} + +/* + * a5xx_pwrlevel_change_settings() - Program the hardware during power level + * transitions + * @adreno_dev: The adreno device pointer + * @prelevel: The previous power level + * @postlevel: The new power level + * @post: True if called after the clock change has taken effect + */ +static void a5xx_pwrlevel_change_settings(struct adreno_device *adreno_dev, + unsigned int prelevel, unsigned int postlevel, + bool post) +{ + /* + * On pre A540 HW only call through if LMx is supported and enabled, and + * always call through for a540 + */ + if (!adreno_is_a540(adreno_dev) && !adreno_dev->lm_enabled) + return; + + if (!post) { + if (gpmu_set_level(adreno_dev, (0x80000010 | postlevel))) + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "GPMU pre powerlevel did not stabilize\n"); + } else { + if (gpmu_set_level(adreno_dev, (0x80000000 | postlevel))) + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "GPMU post powerlevel did not stabilize\n"); + } +} + +/* FW driven idle 10% throttle */ +#define IDLE_10PCT 0 +/* number of cycles when clock is throttled by 50% (CRC) */ +#define CRC_50PCT 1 +/* number of cycles when clock is throttled by more than 50% (CRC) */ +#define CRC_MORE50PCT 2 +/* number of cycles when clock is throttle by less than 50% (CRC) */ +#define CRC_LESS50PCT 3 + +static int64_t a5xx_read_throttling_counters(struct adreno_device *adreno_dev) +{ + int i; + int64_t adj; + uint32_t th[ADRENO_GPMU_THROTTLE_COUNTERS]; + struct adreno_busy_data *busy = &adreno_dev->busy_data; + + if (!adreno_dev->throttling_enabled) + return 0; + + for (i = 0; i < ADRENO_GPMU_THROTTLE_COUNTERS; i++) { + if (!adreno_dev->gpmu_throttle_counters[i]) + return 0; + + th[i] = counter_delta(KGSL_DEVICE(adreno_dev), + adreno_dev->gpmu_throttle_counters[i], + &busy->throttle_cycles[i]); + } + adj = th[CRC_MORE50PCT] - th[IDLE_10PCT]; + adj = th[CRC_50PCT] + th[CRC_LESS50PCT] / 3 + (adj < 0 ? 0 : adj) * 3; + + trace_kgsl_clock_throttling( + th[IDLE_10PCT], th[CRC_50PCT], + th[CRC_MORE50PCT], th[CRC_LESS50PCT], + adj); + return adj; +} + +/* + * a5xx_gpmu_reset() - Re-enable GPMU based power features and restart GPMU + * @work: Pointer to the work struct for gpmu reset + * + * Load the GPMU microcode, set up any features such as hardware clock gating + * or IFPC, and take the GPMU out of reset. + */ +static void a5xx_gpmu_reset(struct work_struct *work) +{ + struct adreno_device *adreno_dev = container_of(work, + struct adreno_device, gpmu_work); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (test_bit(ADRENO_DEVICE_GPMU_INITIALIZED, &adreno_dev->priv)) + return; + + /* + * If GPMU has already experienced a restart or is in the process of it + * after the watchdog timeout, then there is no need to reset GPMU + * again. + */ + if (device->state != KGSL_STATE_AWARE && device->state != KGSL_STATE_ACTIVE) + return; + + mutex_lock(&device->mutex); + + if (a5xx_regulator_enable(adreno_dev)) + goto out; + + /* Soft reset of the GPMU block */ + kgsl_regwrite(device, A5XX_RBBM_BLOCK_SW_RESET_CMD, BIT(16)); + + /* GPU comes up in secured mode, make it unsecured by default */ + if (!ADRENO_FEATURE(adreno_dev, ADRENO_CONTENT_PROTECTION)) + kgsl_regwrite(device, A5XX_RBBM_SECVID_TRUST_CNTL, 0x0); + + + a5xx_gpmu_init(adreno_dev); + +out: + mutex_unlock(&device->mutex); +} + +static void _setup_throttling_counters(struct adreno_device *adreno_dev) +{ + int i, ret = 0; + + if (!adreno_is_a540(adreno_dev)) + return; + + for (i = 0; i < ADRENO_GPMU_THROTTLE_COUNTERS; i++) { + /* reset throttled cycles ivalue */ + adreno_dev->busy_data.throttle_cycles[i] = 0; + + /* Throttle countables start at off set 43 */ + ret |= adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_GPMU_PWR, 43 + i, + &adreno_dev->gpmu_throttle_counters[i], NULL); + } + + WARN_ONCE(ret, "Unable to get one or more clock throttling registers\n"); +} + +/* + * a5xx_start() - Device start + * @adreno_dev: Pointer to adreno device + * + * a5xx device start + */ +static int a5xx_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev); + unsigned int bit; + int ret; + + ret = kgsl_mmu_start(device); + if (ret) + return ret; + + adreno_get_bus_counters(adreno_dev); + adreno_perfcounter_restore(adreno_dev); + + if (adreno_is_a530(adreno_dev) && + ADRENO_FEATURE(adreno_dev, ADRENO_LM)) + adreno_perfcounter_kernel_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_GPMU_PWR, 27, + &adreno_dev->lm_threshold_count, NULL); + + /* Enable 64 bit addressing */ + kgsl_regwrite(device, A5XX_CP_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_VSC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_GRAS_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_RB_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_PC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_HLSQ_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_VFD_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_VPC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_UCHE_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_SP_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_TPL1_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1); + + _setup_throttling_counters(adreno_dev); + + /* Set up VBIF registers from the GPU core definition */ + kgsl_regmap_multi_write(&device->regmap, a5xx_core->vbif, + a5xx_core->vbif_count); + + /* Make all blocks contribute to the GPU BUSY perf counter */ + kgsl_regwrite(device, A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF); + + /* Program RBBM counter 0 to report GPU busy for frequency scaling */ + kgsl_regwrite(device, A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6); + + /* + * Enable the RBBM error reporting bits. This lets us get + * useful information on failure + */ + kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL0, 0x00000001); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_FAULT_DETECT_MASK)) { + /* + * We have 4 RB units, and only RB0 activity signals are + * working correctly. Mask out RB1-3 activity signals + * from the HW hang detection logic as per + * recommendation of hardware team. + */ + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11, + 0xF0000000); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17, + 0xFFFFFFFF); + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18, + 0xFFFFFFFF); + } + + /* + * Set hang detection threshold to 4 million cycles + * (0x3FFFF*16) + */ + kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_INT_CNTL, + (1 << 30) | 0x3FFFF); + + /* Turn on performance counters */ + kgsl_regwrite(device, A5XX_RBBM_PERFCTR_CNTL, 0x01); + + /* + * This is to increase performance by restricting VFD's cache access, + * so that LRZ and other data get evicted less. + */ + kgsl_regwrite(device, A5XX_UCHE_CACHE_WAYS, 0x02); + + /* + * Set UCHE_WRITE_THRU_BASE to the UCHE_TRAP_BASE effectively + * disabling L2 bypass + */ + kgsl_regwrite(device, A5XX_UCHE_TRAP_BASE_LO, 0xffff0000); + kgsl_regwrite(device, A5XX_UCHE_TRAP_BASE_HI, 0x0001ffff); + kgsl_regwrite(device, A5XX_UCHE_WRITE_THRU_BASE_LO, 0xffff0000); + kgsl_regwrite(device, A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff); + + /* Program the GMEM VA range for the UCHE path */ + kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MIN_LO, + adreno_dev->uche_gmem_base); + kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x0); + kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MAX_LO, + adreno_dev->uche_gmem_base + + adreno_dev->gpucore->gmem_size - 1); + kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x0); + + /* + * Below CP registers are 0x0 by default, program init + * values based on a5xx flavor. + */ + if (adreno_is_a505_or_a506(adreno_dev) || adreno_is_a508(adreno_dev)) { + kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x20); + kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x400); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A); + } else if (adreno_is_a510(adreno_dev)) { + kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x20); + kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x20); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A); + } else if (adreno_is_a540(adreno_dev) || adreno_is_a512(adreno_dev)) { + kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x40); + kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x400); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16); + } else { + kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x40); + kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x40); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060); + kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16); + } + + /* + * vtxFifo and primFifo thresholds default values + * are different. + */ + if (adreno_is_a505_or_a506(adreno_dev) || adreno_is_a508(adreno_dev)) + kgsl_regwrite(device, A5XX_PC_DBG_ECO_CNTL, + (0x100 << 11 | 0x100 << 22)); + else if (adreno_is_a510(adreno_dev) || adreno_is_a512(adreno_dev)) + kgsl_regwrite(device, A5XX_PC_DBG_ECO_CNTL, + (0x200 << 11 | 0x200 << 22)); + else + kgsl_regwrite(device, A5XX_PC_DBG_ECO_CNTL, + (0x400 << 11 | 0x300 << 22)); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_TWO_PASS_USE_WFI)) { + /* + * Set TWOPASSUSEWFI in A5XX_PC_DBG_ECO_CNTL for + * microcodes after v77 + */ + if ((adreno_compare_pfp_version(adreno_dev, 0x5FF077) >= 0)) + kgsl_regrmw(device, A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8)); + } + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_DISABLE_RB_DP2CLOCKGATING)) { + /* + * Disable RB sampler datapath DP2 clock gating + * optimization for 1-SP GPU's, by default it is enabled. + */ + kgsl_regrmw(device, A5XX_RB_DBG_ECO_CNT, 0, (1 << 9)); + } + /* + * Disable UCHE global filter as SP can invalidate/flush + * independently + */ + kgsl_regwrite(device, A5XX_UCHE_MODE_CNTL, BIT(29)); + /* Set the USE_RETENTION_FLOPS chicken bit */ + kgsl_regwrite(device, A5XX_CP_CHICKEN_DBG, 0x02000000); + + /* Enable ISDB mode if requested */ + if (test_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv)) { + if (!adreno_active_count_get(adreno_dev)) { + /* + * Disable ME/PFP split timeouts when the debugger is + * enabled because the CP doesn't know when a shader is + * in active debug + */ + kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL1, 0x06FFFFFF); + + /* Force the SP0/SP1 clocks on to enable ISDB */ + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP0, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP1, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP2, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP3, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP0, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP1, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP2, 0x0); + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP3, 0x0); + + /* disable HWCG */ + kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL, 0x0); + kgsl_regwrite(device, A5XX_RBBM_ISDB_CNT, 0x0); + } else + dev_err(device->dev, + "Active count failed while turning on ISDB\n"); + } else { + /* if not in ISDB mode enable ME/PFP split notification */ + kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF); + } + + kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL2, 0x0000003F); + bit = adreno_dev->highest_bank_bit ? + (adreno_dev->highest_bank_bit - 13) & 0x03 : 0; + /* + * Program the highest DDR bank bit that was passed in + * from the DT in a handful of registers. Some of these + * registers will also be written by the UMD, but we + * want to program them in case we happen to use the + * UCHE before the UMD does + */ + + kgsl_regwrite(device, A5XX_TPL1_MODE_CNTL, bit << 7); + kgsl_regwrite(device, A5XX_RB_MODE_CNTL, bit << 1); + if (adreno_is_a540(adreno_dev) || adreno_is_a512(adreno_dev)) + kgsl_regwrite(device, A5XX_UCHE_DBG_ECO_CNTL_2, bit); + + /* Disable All flat shading optimization */ + kgsl_regrmw(device, A5XX_VPC_DBG_ECO_CNTL, 0, 0x1 << 10); + + /* + * VPC corner case with local memory load kill leads to corrupt + * internal state. Normal Disable does not work for all a5x chips. + * So do the following setting to disable it. + */ + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_DISABLE_LMLOADKILL)) { + kgsl_regrmw(device, A5XX_VPC_DBG_ECO_CNTL, 0, 0x1 << 23); + kgsl_regrmw(device, A5XX_HLSQ_DBG_ECO_CNTL, 0x1 << 18, 0); + } + + if (device->mmu.secured) { + kgsl_regwrite(device, A5XX_RBBM_SECVID_TSB_CNTL, 0x0); + kgsl_regwrite(device, A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, + lower_32_bits(KGSL_IOMMU_SECURE_BASE32)); + kgsl_regwrite(device, A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, + upper_32_bits(KGSL_IOMMU_SECURE_BASE32)); + kgsl_regwrite(device, A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, + FIELD_PREP(GENMASK(31, 12), + (KGSL_IOMMU_SECURE_SIZE(&device->mmu) / SZ_4K))); + } + + a5xx_preemption_start(adreno_dev); + a5xx_protect_init(adreno_dev); + + return 0; +} + +/* + * Follow the ME_INIT sequence with a preemption yield to allow the GPU to move + * to a different ringbuffer, if desired + */ +static int _preemption_init( + struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, unsigned int *cmds, + struct kgsl_context *context) +{ + unsigned int *cmds_orig = cmds; + uint64_t gpuaddr = rb->preemption_desc->gpuaddr; + + /* Turn CP protection OFF */ + cmds += cp_protected_mode(adreno_dev, cmds, 0); + /* + * CP during context switch will save context switch info to + * a5xx_cp_preemption_record pointed by CONTEXT_SWITCH_SAVE_ADDR + */ + *cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 1); + *cmds++ = lower_32_bits(gpuaddr); + *cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_HI, 1); + *cmds++ = upper_32_bits(gpuaddr); + + /* Turn CP protection ON */ + cmds += cp_protected_mode(adreno_dev, cmds, 1); + + *cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_GLOBAL, 1); + *cmds++ = 0; + + *cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_LOCAL, 1); + *cmds++ = 1; + + /* Enable yield in RB only */ + *cmds++ = cp_type7_packet(CP_YIELD_ENABLE, 1); + *cmds++ = 1; + + *cmds++ = cp_type7_packet(CP_CONTEXT_SWITCH_YIELD, 4); + cmds += cp_gpuaddr(adreno_dev, cmds, 0x0); + *cmds++ = 0; + /* generate interrupt on preemption completion */ + *cmds++ = 1; + + return cmds - cmds_orig; +} + +static int a5xx_post_start(struct adreno_device *adreno_dev) +{ + int ret; + unsigned int *cmds, *start; + struct adreno_ringbuffer *rb = adreno_dev->cur_rb; + + if (!adreno_is_a530(adreno_dev) && + !adreno_is_preemption_enabled(adreno_dev)) + return 0; + + cmds = adreno_ringbuffer_allocspace(rb, 42); + if (IS_ERR(cmds)) { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + dev_err(device->dev, + "error allocating preemtion init cmds\n"); + return PTR_ERR(cmds); + } + start = cmds; + + /* + * Send a pipeline stat event whenever the GPU gets powered up + * to cause misbehaving perf counters to start ticking + */ + if (adreno_is_a530(adreno_dev)) { + *cmds++ = cp_packet(adreno_dev, CP_EVENT_WRITE, 1); + *cmds++ = 0xF; + } + + if (adreno_is_preemption_enabled(adreno_dev)) { + cmds += _preemption_init(adreno_dev, rb, cmds, NULL); + rb->_wptr = rb->_wptr - (42 - (cmds - start)); + ret = a5xx_ringbuffer_submit(rb, NULL, false); + } else { + rb->_wptr = rb->_wptr - (42 - (cmds - start)); + ret = a5xx_ringbuffer_submit(rb, NULL, true); + } + + if (!ret) { + ret = adreno_spin_idle(adreno_dev, 2000); + if (ret) + a5xx_spin_idle_debug(adreno_dev, + "hw initialization failed to idle\n"); + } + + return ret; +} + +static int a5xx_gpmu_init(struct adreno_device *adreno_dev) +{ + int ret; + + /* Set up LM before initializing the GPMU */ + a5xx_lm_init(adreno_dev); + + /* Enable SPTP based power collapse before enabling GPMU */ + a5xx_enable_pc(adreno_dev); + + ret = a5xx_gpmu_start(adreno_dev); + if (ret) + return ret; + + /* Enable limits management */ + a5xx_lm_enable(adreno_dev); + return 0; +} + +static int a5xx_zap_shader_resume(struct kgsl_device *device) +{ + int ret = qcom_scm_set_remote_state(0, 13); + + if (ret) + dev_err(device->dev, + "SCM zap resume call failed: %d\n", ret); + + return ret; +} + +/* + * a5xx_microcode_load() - Load microcode + * @adreno_dev: Pointer to adreno device + */ +static int a5xx_microcode_load(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_firmware *pm4_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4); + struct adreno_firmware *pfp_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); + const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev); + uint64_t gpuaddr; + + gpuaddr = pm4_fw->memdesc->gpuaddr; + kgsl_regwrite(device, A5XX_CP_PM4_INSTR_BASE_LO, + lower_32_bits(gpuaddr)); + kgsl_regwrite(device, A5XX_CP_PM4_INSTR_BASE_HI, + upper_32_bits(gpuaddr)); + + gpuaddr = pfp_fw->memdesc->gpuaddr; + kgsl_regwrite(device, A5XX_CP_PFP_INSTR_BASE_LO, + lower_32_bits(gpuaddr)); + kgsl_regwrite(device, A5XX_CP_PFP_INSTR_BASE_HI, + upper_32_bits(gpuaddr)); + + /* + * Do not invoke to load zap shader if MMU does + * not support secure mode. + */ + if (!device->mmu.secured) + return 0; + + if (adreno_dev->zap_loaded && !(ADRENO_FEATURE(adreno_dev, + ADRENO_CPZ_RETENTION))) + return a5xx_zap_shader_resume(device); + + return adreno_zap_shader_load(adreno_dev, a5xx_core->zap_name); +} + +static int _me_init_ucode_workarounds(struct adreno_device *adreno_dev) +{ + switch (ADRENO_GPUREV(adreno_dev)) { + case ADRENO_REV_A510: + return 0x00000001; /* Ucode workaround for token end syncs */ + case ADRENO_REV_A505: + case ADRENO_REV_A506: + case ADRENO_REV_A530: + /* + * Ucode workarounds for token end syncs, + * WFI after every direct-render 3D mode draw and + * WFI after every 2D Mode 3 draw. + */ + return 0x0000000B; + default: + return 0x00000000; /* No ucode workarounds enabled */ + } +} + +/* + * CP_INIT_MAX_CONTEXT bit tells if the multiple hardware contexts can + * be used at once of if they should be serialized + */ +#define CP_INIT_MAX_CONTEXT BIT(0) + +/* Enables register protection mode */ +#define CP_INIT_ERROR_DETECTION_CONTROL BIT(1) + +/* Header dump information */ +#define CP_INIT_HEADER_DUMP BIT(2) /* Reserved */ + +/* Default Reset states enabled for PFP and ME */ +#define CP_INIT_DEFAULT_RESET_STATE BIT(3) + +/* Drawcall filter range */ +#define CP_INIT_DRAWCALL_FILTER_RANGE BIT(4) + +/* Ucode workaround masks */ +#define CP_INIT_UCODE_WORKAROUND_MASK BIT(5) + +#define CP_INIT_MASK (CP_INIT_MAX_CONTEXT | \ + CP_INIT_ERROR_DETECTION_CONTROL | \ + CP_INIT_HEADER_DUMP | \ + CP_INIT_DEFAULT_RESET_STATE | \ + CP_INIT_UCODE_WORKAROUND_MASK) + +static int a5xx_critical_packet_submit(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + unsigned int *cmds; + int ret; + + if (!critical_packet_constructed) + return 0; + + cmds = adreno_ringbuffer_allocspace(rb, 4); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + *cmds++ = cp_mem_packet(adreno_dev, CP_INDIRECT_BUFFER_PFE, 2, 1); + cmds += cp_gpuaddr(adreno_dev, cmds, adreno_dev->critpkts->gpuaddr); + *cmds++ = crit_pkts_dwords; + + ret = a5xx_ringbuffer_submit(rb, NULL, true); + if (!ret) { + ret = adreno_spin_idle(adreno_dev, 20); + if (ret) + a5xx_spin_idle_debug(adreno_dev, + "Critical packet submission failed to idle\n"); + } + + return ret; +} + +/* + * a5xx_send_me_init() - Initialize ringbuffer + * @adreno_dev: Pointer to adreno device + * @rb: Pointer to the ringbuffer of device + * + * Submit commands for ME initialization, + */ +static int a5xx_send_me_init(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + unsigned int *cmds; + int i = 0, ret; + + cmds = adreno_ringbuffer_allocspace(rb, 9); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + cmds[i++] = cp_type7_packet(CP_ME_INIT, 8); + + /* Enabled ordinal mask */ + cmds[i++] = CP_INIT_MASK; + + if (CP_INIT_MASK & CP_INIT_MAX_CONTEXT) + cmds[i++] = 0x00000003; + + if (CP_INIT_MASK & CP_INIT_ERROR_DETECTION_CONTROL) + cmds[i++] = 0x20000000; + + if (CP_INIT_MASK & CP_INIT_HEADER_DUMP) { + /* Header dump address */ + cmds[i++] = 0x00000000; + /* Header dump enable and dump size */ + cmds[i++] = 0x00000000; + } + + if (CP_INIT_MASK & CP_INIT_DRAWCALL_FILTER_RANGE) { + /* Start range */ + cmds[i++] = 0x00000000; + /* End range (inclusive) */ + cmds[i++] = 0x00000000; + } + + if (CP_INIT_MASK & CP_INIT_UCODE_WORKAROUND_MASK) + cmds[i++] = _me_init_ucode_workarounds(adreno_dev); + + ret = a5xx_ringbuffer_submit(rb, NULL, true); + if (!ret) { + ret = adreno_spin_idle(adreno_dev, 2000); + if (ret) + a5xx_spin_idle_debug(adreno_dev, + "CP initialization failed to idle\n"); + } + + return ret; +} + +/* + * a5xx_rb_start() - Start the ringbuffer + * @adreno_dev: Pointer to adreno device + */ +static int a5xx_rb_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_ringbuffer *rb; + uint64_t addr; + unsigned int *cmds; + int ret, i; + + /* Clear all the ringbuffers */ + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + memset(rb->buffer_desc->hostptr, 0xaa, KGSL_RB_SIZE); + kgsl_sharedmem_writel(device->scratch, + SCRATCH_RB_OFFSET(rb->id, rptr), 0); + + rb->wptr = 0; + rb->_wptr = 0; + rb->wptr_preempt_end = ~0; + } + + /* Set up the current ringbuffer */ + rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev); + addr = SCRATCH_RB_GPU_ADDR(device, rb->id, rptr); + + kgsl_regwrite(device, A5XX_CP_RB_RPTR_ADDR_LO, lower_32_bits(addr)); + kgsl_regwrite(device, A5XX_CP_RB_RPTR_ADDR_HI, upper_32_bits(addr)); + + /* + * The size of the ringbuffer in the hardware is the log2 + * representation of the size in quadwords (sizedwords / 2). + * Also disable the host RPTR shadow register as it might be unreliable + * in certain circumstances. + */ + + kgsl_regwrite(device, A5XX_CP_RB_CNTL, + A5XX_CP_RB_CNTL_DEFAULT); + + kgsl_regwrite(device, A5XX_CP_RB_BASE, + lower_32_bits(rb->buffer_desc->gpuaddr)); + kgsl_regwrite(device, A5XX_CP_RB_BASE_HI, + upper_32_bits(rb->buffer_desc->gpuaddr)); + + ret = a5xx_microcode_load(adreno_dev); + if (ret) + return ret; + + /* clear ME_HALT to start micro engine */ + + kgsl_regwrite(device, A5XX_CP_ME_CNTL, 0); + + ret = a5xx_send_me_init(adreno_dev, rb); + if (ret) + return ret; + + /* Run the critical packets if we need to */ + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_CRITICAL_PACKETS)) { + ret = a5xx_critical_packet_submit(adreno_dev, rb); + if (ret) + return ret; + } + + /* + * Try to execute the zap shader if it exists, otherwise just try + * directly writing to the control register + */ + if (!adreno_dev->zap_loaded) + kgsl_regwrite(device, A5XX_RBBM_SECVID_TRUST_CNTL, 0); + else { + cmds = adreno_ringbuffer_allocspace(rb, 2); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + *cmds++ = cp_packet(adreno_dev, CP_SET_SECURE_MODE, 1); + *cmds++ = 0; + + ret = a5xx_ringbuffer_submit(rb, NULL, true); + if (!ret) { + ret = adreno_spin_idle(adreno_dev, 2000); + if (ret) { + a5xx_spin_idle_debug(adreno_dev, + "Switch to unsecure failed to idle\n"); + return ret; + } + } + } + + ret = a5xx_gpmu_init(adreno_dev); + if (ret) + return ret; + + a5xx_post_start(adreno_dev); + + return 0; +} + +/* + * a5xx_microcode_read() - Read microcode + * @adreno_dev: Pointer to adreno device + */ +static int a5xx_microcode_read(struct adreno_device *adreno_dev) +{ + int ret; + struct adreno_firmware *pm4_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4); + struct adreno_firmware *pfp_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); + const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev); + + ret = adreno_get_firmware(adreno_dev, a5xx_core->pm4fw_name, pm4_fw); + if (ret) + return ret; + + ret = adreno_get_firmware(adreno_dev, a5xx_core->pfpfw_name, pfp_fw); + if (ret) + return ret; + + ret = _load_gpmu_firmware(adreno_dev); + if (ret) + return ret; + + _load_regfile(adreno_dev); + + return ret; +} + +/* Register offset defines for A5XX, in order of enum adreno_regs */ +static unsigned int a5xx_register_offsets[ADRENO_REG_REGISTER_MAX] = { + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE, A5XX_CP_RB_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE_HI, A5XX_CP_RB_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_LO, + A5XX_CP_RB_RPTR_ADDR_LO), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_HI, + A5XX_CP_RB_RPTR_ADDR_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR, A5XX_CP_RB_RPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_WPTR, A5XX_CP_RB_WPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_CNTL, A5XX_CP_ME_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_CNTL, A5XX_CP_RB_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE, A5XX_CP_IB1_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE_HI, A5XX_CP_IB1_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BUFSZ, A5XX_CP_IB1_BUFSZ), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE, A5XX_CP_IB2_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE_HI, A5XX_CP_IB2_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BUFSZ, A5XX_CP_IB2_BUFSZ), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PROTECT_REG_0, A5XX_CP_PROTECT_REG_0), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT, A5XX_CP_CONTEXT_SWITCH_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT_DEBUG, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT_DISABLE, ADRENO_REG_SKIP), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_LO, + A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_HI, + A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS, A5XX_RBBM_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS3, A5XX_RBBM_STATUS3), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_MASK, A5XX_RBBM_INT_0_MASK), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_CLOCK_CTL, A5XX_RBBM_CLOCK_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SW_RESET_CMD, A5XX_RBBM_SW_RESET_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_GPMU_POWER_COUNTER_ENABLE, + A5XX_GPMU_POWER_COUNTER_ENABLE), +}; + +static void a5xx_cp_hw_err_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int status1, status2; + + kgsl_regread(device, A5XX_CP_INTERRUPT_STATUS, &status1); + + if (status1 & BIT(A5XX_CP_OPCODE_ERROR)) { + unsigned int val; + + kgsl_regwrite(device, A5XX_CP_PFP_STAT_ADDR, 0); + + /* + * A5XX_CP_PFP_STAT_DATA is indexed, so read it twice to get the + * value we want + */ + kgsl_regread(device, A5XX_CP_PFP_STAT_DATA, &val); + kgsl_regread(device, A5XX_CP_PFP_STAT_DATA, &val); + + dev_crit_ratelimited(device->dev, + "ringbuffer opcode error | possible opcode=0x%8.8X\n", + val); + } + if (status1 & BIT(A5XX_CP_RESERVED_BIT_ERROR)) + dev_crit_ratelimited(device->dev, + "ringbuffer reserved bit error interrupt\n"); + if (status1 & BIT(A5XX_CP_HW_FAULT_ERROR)) { + kgsl_regread(device, A5XX_CP_HW_FAULT, &status2); + dev_crit_ratelimited(device->dev, + "CP | Ringbuffer HW fault | status=%x\n", + status2); + } + if (status1 & BIT(A5XX_CP_DMA_ERROR)) + dev_crit_ratelimited(device->dev, "CP | DMA error\n"); + if (status1 & BIT(A5XX_CP_REGISTER_PROTECTION_ERROR)) { + kgsl_regread(device, A5XX_CP_PROTECT_STATUS, &status2); + dev_crit_ratelimited(device->dev, + "CP | Protected mode error| %s | addr=%x | status=%x\n", + status2 & (1 << 24) ? "WRITE" : "READ", + (status2 & 0xFFFFF) >> 2, status2); + } + if (status1 & BIT(A5XX_CP_AHB_ERROR)) { + kgsl_regread(device, A5XX_CP_AHB_FAULT, &status2); + dev_crit_ratelimited(device->dev, + "ringbuffer AHB error interrupt | status=%x\n", + status2); + } +} + +static void a5xx_err_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg; + + switch (bit) { + case A5XX_INT_RBBM_AHB_ERROR: { + kgsl_regread(device, A5XX_RBBM_AHB_ERROR_STATUS, ®); + + /* + * Return the word address of the erroring register so that it + * matches the register specification + */ + dev_crit_ratelimited(device->dev, + "RBBM | AHB bus error | %s | addr=%x | ports=%x:%x\n", + reg & (1 << 28) ? "WRITE" : "READ", + (reg & 0xFFFFF) >> 2, + (reg >> 20) & 0x3, + (reg >> 24) & 0xF); + + /* Clear the error */ + kgsl_regwrite(device, A5XX_RBBM_AHB_CMD, (1 << 4)); + break; + } + case A5XX_INT_RBBM_TRANSFER_TIMEOUT: + dev_crit_ratelimited(device->dev, + "RBBM: AHB transfer timeout\n"); + break; + case A5XX_INT_RBBM_ME_MS_TIMEOUT: + kgsl_regread(device, A5XX_RBBM_AHB_ME_SPLIT_STATUS, ®); + dev_crit_ratelimited(device->dev, + "RBBM | ME master split timeout | status=%x\n", + reg); + break; + case A5XX_INT_RBBM_PFP_MS_TIMEOUT: + kgsl_regread(device, A5XX_RBBM_AHB_PFP_SPLIT_STATUS, ®); + dev_crit_ratelimited(device->dev, + "RBBM | PFP master split timeout | status=%x\n", + reg); + break; + case A5XX_INT_RBBM_ETS_MS_TIMEOUT: + dev_crit_ratelimited(device->dev, + "RBBM: ME master split timeout\n"); + break; + case A5XX_INT_RBBM_ATB_ASYNC_OVERFLOW: + dev_crit_ratelimited(device->dev, + "RBBM: ATB ASYNC overflow\n"); + break; + case A5XX_INT_RBBM_ATB_BUS_OVERFLOW: + dev_crit_ratelimited(device->dev, + "RBBM: ATB bus overflow\n"); + break; + case A5XX_INT_UCHE_OOB_ACCESS: + dev_crit_ratelimited(device->dev, + "UCHE: Out of bounds access\n"); + break; + case A5XX_INT_UCHE_TRAP_INTR: + dev_crit_ratelimited(device->dev, "UCHE: Trap interrupt\n"); + break; + case A5XX_INT_GPMU_VOLTAGE_DROOP: + dev_crit_ratelimited(device->dev, "GPMU: Voltage droop\n"); + break; + default: + dev_crit_ratelimited(device->dev, "Unknown interrupt %d\n", + bit); + } +} + +static void a5xx_irq_storm_worker(struct work_struct *work) +{ + struct adreno_device *adreno_dev = container_of(work, + struct adreno_device, irq_storm_work); + struct kgsl_device *device = &adreno_dev->dev; + unsigned int status; + + mutex_lock(&device->mutex); + + /* Wait for the storm to clear up */ + do { + kgsl_regwrite(device, A5XX_RBBM_INT_CLEAR_CMD, + BIT(A5XX_INT_CP_CACHE_FLUSH_TS)); + kgsl_regread(device, A5XX_RBBM_INT_0_STATUS, &status); + } while (status & BIT(A5XX_INT_CP_CACHE_FLUSH_TS)); + + /* Re-enable the interrupt bit in the mask */ + adreno_dev->irq_mask |= BIT(A5XX_INT_CP_CACHE_FLUSH_TS); + kgsl_regwrite(device, A5XX_RBBM_INT_0_MASK, adreno_dev->irq_mask); + clear_bit(ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED, &adreno_dev->priv); + + dev_warn(device->dev, "Re-enabled A5XX_INT_CP_CACHE_FLUSH_TS\n"); + mutex_unlock(&device->mutex); + + /* Reschedule just to make sure everything retires */ + adreno_dispatcher_schedule(device); +} + +static void a5xx_cp_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = &adreno_dev->dev; + unsigned int cur; + static unsigned int count; + static unsigned int prev; + + if (test_bit(ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED, &adreno_dev->priv)) + return; + + kgsl_sharedmem_readl(device->memstore, &cur, + KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL, + ref_wait_ts)); + + /* + * prev holds a previously read value + * from memory. It should be changed by the GPU with every + * interrupt. If the value we know about and the value we just + * read are the same, then we are likely in a storm. + * If this happens twice, disable the interrupt in the mask + * so the dispatcher can take care of the issue. It is then + * up to the dispatcher to re-enable the mask once all work + * is done and the storm has ended. + */ + if (prev == cur) { + count++; + if (count == 2) { + /* disable interrupt from the mask */ + set_bit(ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED, + &adreno_dev->priv); + + adreno_dev->irq_mask &= + ~BIT(A5XX_INT_CP_CACHE_FLUSH_TS); + + kgsl_regwrite(device, A5XX_RBBM_INT_0_MASK, + adreno_dev->irq_mask); + + kgsl_schedule_work(&adreno_dev->irq_storm_work); + + return; + } + } else { + count = 0; + prev = cur; + } + + a5xx_preemption_trigger(adreno_dev); + adreno_dispatcher_schedule(device); +} + +static const char *gpmu_int_msg[32] = { + [FW_INTR_INFO] = "FW_INTR_INFO", + [LLM_ACK_ERR_INTR] = "LLM_ACK_ERR_INTR", + [ISENS_TRIM_ERR_INTR] = "ISENS_TRIM_ERR_INTR", + [ISENS_ERR_INTR] = "ISENS_ERR_INTR", + [ISENS_IDLE_ERR_INTR] = "ISENS_IDLE_ERR_INTR", + [ISENS_PWR_ON_ERR_INTR] = "ISENS_PWR_ON_ERR_INTR", + [6 ... 30] = "", + [WDOG_EXPITED] = "WDOG_EXPITED"}; + +static void a5xx_gpmu_int_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg, i; + + kgsl_regread(device, A5XX_GPMU_RBBM_INTR_INFO, ®); + + if (reg & (~VALID_GPMU_IRQ)) { + dev_crit_ratelimited(device->dev, + "GPMU: Unknown IRQ mask 0x%08lx in 0x%08x\n", + reg & (~VALID_GPMU_IRQ), reg); + } + + for (i = 0; i < 32; i++) + switch (reg & BIT(i)) { + case BIT(WDOG_EXPITED): + if (test_and_clear_bit(ADRENO_DEVICE_GPMU_INITIALIZED, + &adreno_dev->priv)) { + /* Stop GPMU */ + kgsl_regwrite(device, + A5XX_GPMU_CM3_SYSRESET, 1); + kgsl_schedule_work(&adreno_dev->gpmu_work); + } + fallthrough; + case BIT(FW_INTR_INFO): + fallthrough; + case BIT(LLM_ACK_ERR_INTR): + fallthrough; + case BIT(ISENS_TRIM_ERR_INTR): + fallthrough; + case BIT(ISENS_ERR_INTR): + fallthrough; + case BIT(ISENS_IDLE_ERR_INTR): + fallthrough; + case BIT(ISENS_PWR_ON_ERR_INTR): + dev_crit_ratelimited(device->dev, + "GPMU: interrupt %s(%08lx)\n", + gpmu_int_msg[i], + BIT(i)); + break; + } +} + +/* + * a5x_gpc_err_int_callback() - Isr for GPC error interrupts + * @adreno_dev: Pointer to device + * @bit: Interrupt bit + */ +static void a5x_gpc_err_int_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* + * GPC error is typically the result of mistake SW programming. + * Force GPU fault for this interrupt so that we can debug it + * with help of register dump. + */ + + dev_crit(device->dev, "RBBM: GPC error\n"); + adreno_irqctrl(adreno_dev, 0); + + /* Trigger a fault in the dispatcher - this will effect a restart */ + adreno_dispatcher_fault(adreno_dev, ADRENO_SOFT_FAULT); + adreno_dispatcher_schedule(device); +} + +u64 a5xx_read_alwayson(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 lo = 0, hi = 0; + + kgsl_regread(device, A5XX_RBBM_ALWAYSON_COUNTER_LO, &lo); + + /* The upper 32 bits are only reliable on A540 targets */ + if (adreno_is_a540(adreno_dev)) + kgsl_regread(device, A5XX_RBBM_ALWAYSON_COUNTER_HI, &hi); + + return (((u64) hi) << 32) | lo; +} + + +static const struct adreno_irq_funcs a5xx_irq_funcs[32] = { + ADRENO_IRQ_CALLBACK(NULL), /* 0 - RBBM_GPU_IDLE */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 1 - RBBM_AHB_ERROR */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 2 - RBBM_TRANSFER_TIMEOUT */ + /* 3 - RBBM_ME_MASTER_SPLIT_TIMEOUT */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), + /* 4 - RBBM_PFP_MASTER_SPLIT_TIMEOUT */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), + /* 5 - RBBM_ETS_MASTER_SPLIT_TIMEOUT */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), + /* 6 - RBBM_ATB_ASYNC_OVERFLOW */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), + ADRENO_IRQ_CALLBACK(a5x_gpc_err_int_callback), /* 7 - GPC_ERR */ + ADRENO_IRQ_CALLBACK(a5xx_preempt_callback),/* 8 - CP_SW */ + ADRENO_IRQ_CALLBACK(a5xx_cp_hw_err_callback), /* 9 - CP_HW_ERROR */ + /* 10 - CP_CCU_FLUSH_DEPTH_TS */ + ADRENO_IRQ_CALLBACK(NULL), + /* 11 - CP_CCU_FLUSH_COLOR_TS */ + ADRENO_IRQ_CALLBACK(NULL), + /* 12 - CP_CCU_RESOLVE_TS */ + ADRENO_IRQ_CALLBACK(NULL), + ADRENO_IRQ_CALLBACK(NULL), /* 13 - CP_IB2_INT */ + ADRENO_IRQ_CALLBACK(NULL), /* 14 - CP_IB1_INT */ + ADRENO_IRQ_CALLBACK(NULL), /* 15 - CP_RB_INT */ + /* 16 - CCP_UNUSED_1 */ + ADRENO_IRQ_CALLBACK(NULL), + ADRENO_IRQ_CALLBACK(NULL), /* 17 - CP_RB_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 18 - CP_WT_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 19 - UNKNOWN_1 */ + ADRENO_IRQ_CALLBACK(a5xx_cp_callback), /* 20 - CP_CACHE_FLUSH_TS */ + /* 21 - UNUSED_2 */ + ADRENO_IRQ_CALLBACK(NULL), + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 22 - RBBM_ATB_BUS_OVERFLOW */ + /* 23 - MISC_HANG_DETECT */ + ADRENO_IRQ_CALLBACK(adreno_hang_int_callback), + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 24 - UCHE_OOB_ACCESS */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 25 - UCHE_TRAP_INTR */ + ADRENO_IRQ_CALLBACK(NULL), /* 26 - DEBBUS_INTR_0 */ + ADRENO_IRQ_CALLBACK(NULL), /* 27 - DEBBUS_INTR_1 */ + ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 28 - GPMU_VOLTAGE_DROOP */ + ADRENO_IRQ_CALLBACK(a5xx_gpmu_int_callback), /* 29 - GPMU_FIRMWARE */ + ADRENO_IRQ_CALLBACK(NULL), /* 30 - ISDB_CPU_IRQ */ + ADRENO_IRQ_CALLBACK(NULL), /* 31 - ISDB_UNDER_DEBUG */ +}; + +static irqreturn_t a5xx_irq_handler(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + irqreturn_t ret; + u32 status; + + kgsl_regread(device, A5XX_RBBM_INT_0_STATUS, &status); + + /* + * Clear all the interrupt bits except A5XX_INT_RBBM_AHB_ERROR. + * The interrupt will stay asserted until it is cleared by the handler + * so don't touch it yet to avoid a storm + */ + kgsl_regwrite(device, A5XX_RBBM_INT_CLEAR_CMD, + status & ~A5XX_INT_RBBM_AHB_ERROR); + + /* Call the helper function for callbacks */ + ret = adreno_irq_callbacks(adreno_dev, a5xx_irq_funcs, status); + + trace_kgsl_a5xx_irq_status(adreno_dev, status); + + /* Now chear AHB_ERROR if it was set */ + if (status & A5XX_INT_RBBM_AHB_ERROR) + kgsl_regwrite(device, A5XX_RBBM_INT_CLEAR_CMD, + A5XX_INT_RBBM_AHB_ERROR); + + return ret; +} + +static bool a5xx_hw_isidle(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 status; + + /* + * Due to CRC idle throttling the GPU idle hysteresis on a540 can take + * up to 5uS to expire + */ + if (adreno_is_a540(adreno_dev)) + udelay(5); + + kgsl_regread(device, A5XX_RBBM_STATUS, &status); + + if (status & 0xfffffffe) + return false; + + kgsl_regread(device, A5XX_RBBM_INT_0_STATUS, &status); + + /* Return busy if a interrupt is pending */ + return !((status & adreno_dev->irq_mask) || + atomic_read(&adreno_dev->pending_irq_refcnt)); +} + +static int a5xx_clear_pending_transactions(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 mask = A5XX_VBIF_XIN_HALT_CTRL0_MASK; + int ret; + + kgsl_regwrite(device, A5XX_VBIF_XIN_HALT_CTRL0, mask); + ret = adreno_wait_for_halt_ack(device, A5XX_VBIF_XIN_HALT_CTRL1, mask); + kgsl_regwrite(device, A5XX_VBIF_XIN_HALT_CTRL0, 0); + + return ret; +} + +static bool a5xx_is_hw_collapsible(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg; + + if (!adreno_isidle(adreno_dev)) + return false; + + /* If feature is not supported or enabled, no worry */ + if (!adreno_dev->sptp_pc_enabled) + return true; + kgsl_regread(device, A5XX_GPMU_SP_PWR_CLK_STATUS, ®); + if (reg & BIT(20)) + return false; + kgsl_regread(device, A5XX_GPMU_RBCCU_PWR_CLK_STATUS, ®); + return !(reg & BIT(20)); +} + +static void a5xx_remove(struct adreno_device *adreno_dev) +{ + if (adreno_preemption_feature_set(adreno_dev)) + del_timer(&adreno_dev->preempt.timer); +} + +static void a5xx_power_stats(struct adreno_device *adreno_dev, + struct kgsl_power_stats *stats) +{ + static u32 rbbm0_hi; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + s64 freq = kgsl_pwrctrl_active_freq(&device->pwrctrl) / 1000000; + struct adreno_busy_data *busy = &adreno_dev->busy_data; + s64 gpu_busy = 0; + u32 lo, hi; + s64 adj; + + /* Sometimes this counter can go backwards, so try to detect that */ + kgsl_regread(device, A5XX_RBBM_PERFCTR_RBBM_0_LO, &lo); + kgsl_regread(device, A5XX_RBBM_PERFCTR_RBBM_0_HI, &hi); + + if (busy->gpu_busy) { + if (lo < busy->gpu_busy) { + if (hi == rbbm0_hi) { + dev_warn_once(device->dev, + "abmormal value from RBBM_0 perfcounter: %x %x\n", + lo, busy->gpu_busy); + gpu_busy = 0; + } else { + gpu_busy = (UINT_MAX - busy->gpu_busy) + lo; + rbbm0_hi = hi; + } + } else + gpu_busy = lo - busy->gpu_busy; + } else { + gpu_busy = 0; + rbbm0_hi = 0; + } + + busy->gpu_busy = lo; + + adj = a5xx_read_throttling_counters(adreno_dev); + if (-adj <= gpu_busy) + gpu_busy += adj; + else + gpu_busy = 0; + + stats->busy_time = gpu_busy / freq; + + if (adreno_is_a530(adreno_dev) && adreno_dev->lm_threshold_count) + kgsl_regread(device, adreno_dev->lm_threshold_count, + &adreno_dev->lm_threshold_cross); + else if (adreno_is_a540(adreno_dev)) + adreno_dev->lm_threshold_cross = adj; + + if (!device->pwrctrl.bus_control) + return; + + stats->ram_time = counter_delta(device, adreno_dev->ram_cycles_lo, + &busy->bif_ram_cycles); + + stats->ram_wait = counter_delta(device, adreno_dev->starved_ram_lo, + &busy->bif_starved_ram); +} + +static int a5xx_setproperty(struct kgsl_device_private *dev_priv, + u32 type, void __user *value, u32 sizebytes) +{ + struct kgsl_device *device = dev_priv->device; + u32 enable; + + if (type != KGSL_PROP_PWRCTRL) + return -ENODEV; + + if (sizebytes != sizeof(enable)) + return -EINVAL; + + if (copy_from_user(&enable, value, sizeof(enable))) + return -EFAULT; + + mutex_lock(&device->mutex); + + if (enable) { + device->pwrctrl.ctrl_flags = 0; + kgsl_pwrscale_enable(device); + } else { + kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); + device->pwrctrl.ctrl_flags = KGSL_PWR_ON; + kgsl_pwrscale_disable(device, true); + } + + mutex_unlock(&device->mutex); + + return 0; +} + +const struct adreno_gpudev adreno_a5xx_gpudev = { + .reg_offsets = a5xx_register_offsets, + .probe = a5xx_probe, + .start = a5xx_start, + .snapshot = a5xx_snapshot, + .init = a5xx_init, + .irq_handler = a5xx_irq_handler, + .rb_start = a5xx_rb_start, + .regulator_enable = a5xx_regulator_enable, + .regulator_disable = a5xx_regulator_disable, + .pwrlevel_change_settings = a5xx_pwrlevel_change_settings, + .preemption_schedule = a5xx_preemption_schedule, + .read_alwayson = a5xx_read_alwayson, + .hw_isidle = a5xx_hw_isidle, + .power_ops = &adreno_power_operations, + .clear_pending_transactions = a5xx_clear_pending_transactions, + .remove = a5xx_remove, + .ringbuffer_submitcmd = a5xx_ringbuffer_submitcmd, + .is_hw_collapsible = a5xx_is_hw_collapsible, + .power_stats = a5xx_power_stats, + .setproperty = a5xx_setproperty, +}; diff --git a/qcom/opensource/graphics-kernel/adreno_a5xx.h b/qcom/opensource/graphics-kernel/adreno_a5xx.h new file mode 100644 index 0000000000..dd8f701f0a --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a5xx.h @@ -0,0 +1,314 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2015-2017,2019-2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022,2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _ADRENO_A5XX_H_ +#define _ADRENO_A5XX_H_ + +#include "a5xx_reg.h" + +/** + * struct adreno_a5xx_core - a5xx specific GPU core definitions + */ +struct adreno_a5xx_core { + /** @base: Container for the generic &struct adreno_gpu_core */ + struct adreno_gpu_core base; + /** @gpmu_tsens: ID for the temperature sensor used by the GPMU */ + unsigned int gpmu_tsens; + /** @max_power: Max possible power draw of a core */ + unsigned int max_power; + /** pm4fw_name: Name of the PM4 microcode file */ + const char *pm4fw_name; + /** pfpfw_name: Name of the PFP microcode file */ + const char *pfpfw_name; + /** gpmufw_name: Name of the GPMU microcode file */ + const char *gpmufw_name; + /** @regfw_name: Filename for the LM registers if applicable */ + const char *regfw_name; + /** @zap_name: Name of the CPZ zap file */ + const char *zap_name; + /** @hwcg: List of registers and values to write for HWCG */ + const struct kgsl_regmap_list *hwcg; + /** @hwcg_count: Number of registers in @hwcg */ + u32 hwcg_count; + /** @vbif: List of registers and values to write for VBIF */ + const struct kgsl_regmap_list *vbif; + /** @vbif_count: Number of registers in @vbif */ + u32 vbif_count; + /** @highest_bank_bit: The bit of the highest DDR bank */ + u32 highest_bank_bit; +}; + +#define A5XX_CP_CTXRECORD_MAGIC_REF 0x27C4BAFCUL +/* Size of each CP preemption record */ +#define A5XX_CP_CTXRECORD_SIZE_IN_BYTES 0x10000 +/* Size of the preemption counter block (in bytes) */ +#define A5XX_CP_CTXRECORD_PREEMPTION_COUNTER_SIZE (16 * 4) + +/** + * struct a5xx_cp_preemption_record - CP context record for + * preemption. + * @magic: (00) Value at this offset must be equal to + * A5XX_CP_CTXRECORD_MAGIC_REF. + * @info: (04) Type of record. Written non-zero (usually) by CP. + * we must set to zero for all ringbuffers. + * @data: (08) DATA field in SET_RENDER_MODE or checkpoint packets. + * Written by CP when switching out. Not used on switch-in. + * we must initialize to zero. + * @cntl: (12) RB_CNTL, saved and restored by CP. + * @rptr: (16) RB_RPTR, saved and restored by CP. + * @wptr: (20) RB_WPTR, saved and restored by CP. + * @rptr_addr: (24) RB_RPTR_ADDR_LO|HI saved and restored. + * rbase: (32) RB_BASE_LO|HI saved and restored. + * counter: (40) Pointer to preemption counter + */ +struct a5xx_cp_preemption_record { + uint32_t magic; + uint32_t info; + uint32_t data; + uint32_t cntl; + uint32_t rptr; + uint32_t wptr; + uint64_t rptr_addr; + uint64_t rbase; + uint64_t counter; +}; + +#define A5XX_CP_SMMU_INFO_MAGIC_REF 0x3618CDA3UL + +/** + * struct a5xx_cp_smmu_info - CP preemption SMMU info. + * @magic: (00) The value at this offset must be equal to + * A5XX_CP_SMMU_INFO_MAGIC_REF. + * @_pad4: (04) Reserved/padding + * @ttbr0: (08) Base address of the page table for the + * incoming context. + * @context_idr: (16) Context Identification Register value. + */ +struct a5xx_cp_smmu_info { + uint32_t magic; + uint32_t _pad4; + uint64_t ttbr0; + uint32_t asid; + uint32_t context_idr; +}; + +void a5xx_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot); +unsigned int a5xx_num_registers(void); + +void a5xx_crashdump_init(struct adreno_device *adreno_dev); + +void a5xx_hwcg_set(struct adreno_device *adreno_dev, bool on); + +#define A5XX_CP_RB_CNTL_DEFAULT ((1 << 27) | ((ilog2(4) << 8) & 0x1F00) | \ + (ilog2(KGSL_RB_DWORDS >> 1) & 0x3F)) +/* GPMU interrupt multiplexor */ +#define FW_INTR_INFO (0) +#define LLM_ACK_ERR_INTR (1) +#define ISENS_TRIM_ERR_INTR (2) +#define ISENS_ERR_INTR (3) +#define ISENS_IDLE_ERR_INTR (4) +#define ISENS_PWR_ON_ERR_INTR (5) +#define WDOG_EXPITED (31) + +#define VALID_GPMU_IRQ (\ + BIT(FW_INTR_INFO) | \ + BIT(LLM_ACK_ERR_INTR) | \ + BIT(ISENS_TRIM_ERR_INTR) | \ + BIT(ISENS_ERR_INTR) | \ + BIT(ISENS_IDLE_ERR_INTR) | \ + BIT(ISENS_PWR_ON_ERR_INTR) | \ + BIT(WDOG_EXPITED)) + +/* A5XX_GPMU_GPMU_LLM_GLM_SLEEP_CTRL */ +#define STATE_OF_CHILD GENMASK(5, 4) +#define STATE_OF_CHILD_01 BIT(4) +#define STATE_OF_CHILD_11 (BIT(4) | BIT(5)) +#define IDLE_FULL_LM_SLEEP BIT(0) + +/* A5XX_GPMU_GPMU_LLM_GLM_SLEEP_STATUS */ +#define WAKEUP_ACK BIT(1) +#define IDLE_FULL_ACK BIT(0) + +/* A5XX_GPMU_GPMU_ISENSE_CTRL */ +#define ISENSE_CGC_EN_DISABLE BIT(0) + +/* A5XX_GPMU_TEMP_SENSOR_CONFIG */ +#define GPMU_BCL_ENABLED BIT(4) +#define GPMU_LLM_ENABLED BIT(9) +#define GPMU_ISENSE_STATUS GENMASK(3, 0) +#define GPMU_ISENSE_END_POINT_CAL_ERR BIT(0) + +#define AMP_CALIBRATION_RETRY_CNT 3 +#define AMP_CALIBRATION_TIMEOUT 6 + +/* A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK */ +#define VOLTAGE_INTR_EN BIT(0) + +/* A5XX_GPMU_GPMU_PWR_THRESHOLD */ +#define PWR_THRESHOLD_VALID 0x80000000 + +/* A5XX_GPMU_GPMU_SP_CLOCK_CONTROL */ +#define CNTL_IP_CLK_ENABLE BIT(0) +/* AGC */ +#define AGC_INIT_BASE A5XX_GPMU_DATA_RAM_BASE +#define AGC_INIT_MSG_MAGIC (AGC_INIT_BASE + 5) +#define AGC_MSG_BASE (AGC_INIT_BASE + 7) + +#define AGC_MSG_STATE (AGC_MSG_BASE + 0) +#define AGC_MSG_COMMAND (AGC_MSG_BASE + 1) +#define AGC_MSG_PAYLOAD_SIZE (AGC_MSG_BASE + 3) +#define AGC_MSG_PAYLOAD (AGC_MSG_BASE + 5) + +#define AGC_INIT_MSG_VALUE 0xBABEFACE +#define AGC_POWER_CONFIG_PRODUCTION_ID 1 + +#define AGC_LM_CONFIG (136/4) +#define AGC_LM_CONFIG_ENABLE_GPMU_ADAPTIVE (1) + +#define AGC_LM_CONFIG_ENABLE_ERROR (3 << 4) +#define AGC_LM_CONFIG_ISENSE_ENABLE (1 << 4) + +#define AGC_THROTTLE_SEL_DCS (1 << 8) +#define AGC_THROTTLE_DISABLE (2 << 8) + + +#define AGC_LLM_ENABLED (1 << 16) +#define AGC_GPU_VERSION_MASK GENMASK(18, 17) +#define AGC_GPU_VERSION_SHIFT 17 +#define AGC_BCL_DISABLED (1 << 24) + + +#define AGC_LEVEL_CONFIG (140/4) + +#define LM_DCVS_LIMIT 1 +/* FW file tages */ +#define GPMU_FIRMWARE_ID 2 +#define GPMU_SEQUENCE_ID 3 +#define GPMU_INST_RAM_SIZE 0xFFF + +#define HEADER_MAJOR 1 +#define HEADER_MINOR 2 +#define HEADER_DATE 3 +#define HEADER_TIME 4 +#define HEADER_SEQUENCE 5 + +#define MAX_HEADER_SIZE 10 + +#define LM_SEQUENCE_ID 1 +#define MAX_SEQUENCE_ID 3 + +#define GPMU_ISENSE_SAVE (A5XX_GPMU_DATA_RAM_BASE + 200/4) +/* LM defaults */ +#define LM_DEFAULT_LIMIT 6000 +#define A530_DEFAULT_LEAKAGE 0x004E001A + +/** + * to_a5xx_core - return the a5xx specific GPU core struct + * @adreno_dev: An Adreno GPU device handle + * + * Returns: + * A pointer to the a5xx specific GPU core struct + */ +static inline const struct adreno_a5xx_core * +to_a5xx_core(struct adreno_device *adreno_dev) +{ + const struct adreno_gpu_core *core = adreno_dev->gpucore; + + return container_of(core, struct adreno_a5xx_core, base); +} + +/* Preemption functions */ +void a5xx_preemption_trigger(struct adreno_device *adreno_dev); +void a5xx_preemption_schedule(struct adreno_device *adreno_dev); +void a5xx_preemption_start(struct adreno_device *adreno_dev); +int a5xx_preemption_init(struct adreno_device *adreno_dev); + +/** + * a5xx_preemption_post_ibsubmit - Insert commands following a submission + * @adreno_dev: Adreno GPU handle + * @cmds: Pointer to the ringbuffer to insert opcodes + * + * Return: The number of opcodes written to @cmds + */ +u32 a5xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev, u32 *cmds); + +/** + * a5xx_preemption_post_ibsubmit - Insert opcodes before a submission + * @adreno_dev: Adreno GPU handle + * @rb: The ringbuffer being written + * @drawctxt: The draw context being written + * @cmds: Pointer to the ringbuffer to insert opcodes + * + * Return: The number of opcodes written to @cmds + */ +u32 a5xx_preemption_pre_ibsubmit(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, struct adreno_context *drawctxt, + u32 *cmds); + +void a5xx_preempt_callback(struct adreno_device *adreno_dev, int bit); + +u64 a5xx_read_alwayson(struct adreno_device *adreno_dev); + +extern const struct adreno_perfcounters adreno_a5xx_perfcounters; + +/** + * a5xx_ringbuffer_init - Initialize the ringbuffers + * @adreno_dev: An Adreno GPU handle + * + * Initialize the ringbuffer(s) for a5xx. + * Return: 0 on success or negative on failure + */ +int a5xx_ringbuffer_init(struct adreno_device *adreno_dev); + +/** + * a5xx_ringbuffer_addcmds - Submit a command to the ringbuffer + * @adreno_dev: An Adreno GPU handle + * @rb: Pointer to the ringbuffer to submit on + * @drawctxt: Pointer to the draw context for the submission, or NULL for + * internal submissions + * @flags: Flags for the submission + * @in: Commands to write to the ringbuffer + * @dwords: Size of @in (in dwords) + * @timestamp: Timestamp for the submission + * @time: Optional pointer to a submit time structure + * + * Submit a command to the ringbuffer. + * Return: 0 on success or negative on failure + */ +int a5xx_ringbuffer_addcmds(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, struct adreno_context *drawctxt, + u32 flags, u32 *in, u32 dwords, u32 timestamp, + struct adreno_submit_time *time); + +/** + * a5xx_ringbuffer_submitcmd - Submit a user command to the ringbuffer + * @adreno_dev: An Adreno GPU handle + * @cmdobj: Pointer to a user command object + * @flags: Internal submit flags + * @time: Optional pointer to a adreno_submit_time container + * + * Return: 0 on success or negative on failure + */ +int a5xx_ringbuffer_submitcmd(struct adreno_device *adreno_dev, + struct kgsl_drawobj_cmd *cmdobj, u32 flags, + struct adreno_submit_time *time); + +int a5xx_ringbuffer_submit(struct adreno_ringbuffer *rb, + struct adreno_submit_time *time, bool sync); + +static inline bool a5xx_has_gpmu(struct adreno_device *adreno_dev) +{ + return (adreno_is_a530(adreno_dev) || adreno_is_a540(adreno_dev)); +} + +#ifdef CONFIG_QCOM_KGSL_CORESIGHT +void a5xx_coresight_init(struct adreno_device *device); +#else +static inline void a5xx_coresight_init(struct adreno_device *device) { } +#endif + +#endif diff --git a/qcom/opensource/graphics-kernel/adreno_a5xx_coresight.c b/qcom/opensource/graphics-kernel/adreno_a5xx_coresight.c new file mode 100644 index 0000000000..4a74686cb1 --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a5xx_coresight.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include "adreno.h" +#include "adreno_a5xx.h" +#include "adreno_coresight.h" + +static struct adreno_coresight_register a5xx_coresight_registers[] = { + { A5XX_RBBM_CFG_DBGBUS_SEL_A }, + { A5XX_RBBM_CFG_DBGBUS_SEL_B }, + { A5XX_RBBM_CFG_DBGBUS_SEL_C }, + { A5XX_RBBM_CFG_DBGBUS_SEL_D }, + { A5XX_RBBM_CFG_DBGBUS_CNTLT }, + { A5XX_RBBM_CFG_DBGBUS_CNTLM }, + { A5XX_RBBM_CFG_DBGBUS_OPL }, + { A5XX_RBBM_CFG_DBGBUS_OPE }, + { A5XX_RBBM_CFG_DBGBUS_IVTL_0 }, + { A5XX_RBBM_CFG_DBGBUS_IVTL_1 }, + { A5XX_RBBM_CFG_DBGBUS_IVTL_2 }, + { A5XX_RBBM_CFG_DBGBUS_IVTL_3 }, + { A5XX_RBBM_CFG_DBGBUS_MASKL_0 }, + { A5XX_RBBM_CFG_DBGBUS_MASKL_1 }, + { A5XX_RBBM_CFG_DBGBUS_MASKL_2 }, + { A5XX_RBBM_CFG_DBGBUS_MASKL_3 }, + { A5XX_RBBM_CFG_DBGBUS_BYTEL_0 }, + { A5XX_RBBM_CFG_DBGBUS_BYTEL_1 }, + { A5XX_RBBM_CFG_DBGBUS_IVTE_0 }, + { A5XX_RBBM_CFG_DBGBUS_IVTE_1 }, + { A5XX_RBBM_CFG_DBGBUS_IVTE_2 }, + { A5XX_RBBM_CFG_DBGBUS_IVTE_3 }, + { A5XX_RBBM_CFG_DBGBUS_MASKE_0 }, + { A5XX_RBBM_CFG_DBGBUS_MASKE_1 }, + { A5XX_RBBM_CFG_DBGBUS_MASKE_2 }, + { A5XX_RBBM_CFG_DBGBUS_MASKE_3 }, + { A5XX_RBBM_CFG_DBGBUS_NIBBLEE }, + { A5XX_RBBM_CFG_DBGBUS_PTRC0 }, + { A5XX_RBBM_CFG_DBGBUS_PTRC1 }, + { A5XX_RBBM_CFG_DBGBUS_LOADREG }, + { A5XX_RBBM_CFG_DBGBUS_IDX }, + { A5XX_RBBM_CFG_DBGBUS_CLRC }, + { A5XX_RBBM_CFG_DBGBUS_LOADIVT }, + { A5XX_RBBM_CFG_DBGBUS_EVENT_LOGIC }, + { A5XX_RBBM_CFG_DBGBUS_OVER }, + { A5XX_RBBM_CFG_DBGBUS_COUNT0 }, + { A5XX_RBBM_CFG_DBGBUS_COUNT1 }, + { A5XX_RBBM_CFG_DBGBUS_COUNT2 }, + { A5XX_RBBM_CFG_DBGBUS_COUNT3 }, + { A5XX_RBBM_CFG_DBGBUS_COUNT4 }, + { A5XX_RBBM_CFG_DBGBUS_COUNT5 }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_ADDR }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_BUF0 }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_BUF1 }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_BUF2 }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_BUF3 }, + { A5XX_RBBM_CFG_DBGBUS_TRACE_BUF4 }, + { A5XX_RBBM_CFG_DBGBUS_MISR0 }, + { A5XX_RBBM_CFG_DBGBUS_MISR1 }, + { A5XX_RBBM_AHB_DBG_CNTL }, + { A5XX_RBBM_READ_AHB_THROUGH_DBG }, + { A5XX_RBBM_DBG_LO_HI_GPIO }, + { A5XX_RBBM_EXT_TRACE_BUS_CNTL }, + { A5XX_RBBM_EXT_VBIF_DBG_CNTL }, +}; + +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_a, &a5xx_coresight_registers[0]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_b, &a5xx_coresight_registers[1]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_c, &a5xx_coresight_registers[2]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_sel_d, &a5xx_coresight_registers[3]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_cntlt, &a5xx_coresight_registers[4]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_cntlm, &a5xx_coresight_registers[5]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_opl, &a5xx_coresight_registers[6]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ope, &a5xx_coresight_registers[7]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_0, &a5xx_coresight_registers[8]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_1, &a5xx_coresight_registers[9]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_2, &a5xx_coresight_registers[10]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivtl_3, &a5xx_coresight_registers[11]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_0, &a5xx_coresight_registers[12]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_1, &a5xx_coresight_registers[13]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_2, &a5xx_coresight_registers[14]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maskl_3, &a5xx_coresight_registers[15]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_bytel_0, &a5xx_coresight_registers[16]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_bytel_1, &a5xx_coresight_registers[17]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_0, &a5xx_coresight_registers[18]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_1, &a5xx_coresight_registers[19]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_2, &a5xx_coresight_registers[20]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ivte_3, &a5xx_coresight_registers[21]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_0, &a5xx_coresight_registers[22]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_1, &a5xx_coresight_registers[23]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_2, &a5xx_coresight_registers[24]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_maske_3, &a5xx_coresight_registers[25]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_nibblee, &a5xx_coresight_registers[26]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ptrc0, &a5xx_coresight_registers[27]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_ptrc1, &a5xx_coresight_registers[28]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_loadreg, &a5xx_coresight_registers[29]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_idx, &a5xx_coresight_registers[30]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_clrc, &a5xx_coresight_registers[31]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_loadivt, &a5xx_coresight_registers[32]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_event_logic, + &a5xx_coresight_registers[33]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_over, &a5xx_coresight_registers[34]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count0, &a5xx_coresight_registers[35]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count1, &a5xx_coresight_registers[36]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count2, &a5xx_coresight_registers[37]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count3, &a5xx_coresight_registers[38]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count4, &a5xx_coresight_registers[39]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_count5, &a5xx_coresight_registers[40]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_addr, + &a5xx_coresight_registers[41]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf0, + &a5xx_coresight_registers[42]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf1, + &a5xx_coresight_registers[43]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf2, + &a5xx_coresight_registers[44]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf3, + &a5xx_coresight_registers[45]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_trace_buf4, + &a5xx_coresight_registers[46]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_misr0, &a5xx_coresight_registers[47]); +static ADRENO_CORESIGHT_ATTR(cfg_dbgbus_misr1, &a5xx_coresight_registers[48]); +static ADRENO_CORESIGHT_ATTR(ahb_dbg_cntl, &a5xx_coresight_registers[49]); +static ADRENO_CORESIGHT_ATTR(read_ahb_through_dbg, + &a5xx_coresight_registers[50]); +static ADRENO_CORESIGHT_ATTR(dbg_lo_hi_gpio, &a5xx_coresight_registers[51]); +static ADRENO_CORESIGHT_ATTR(ext_trace_bus_cntl, &a5xx_coresight_registers[52]); +static ADRENO_CORESIGHT_ATTR(ext_vbif_dbg_cntl, &a5xx_coresight_registers[53]); + +static struct attribute *a5xx_coresight_attrs[] = { + &coresight_attr_cfg_dbgbus_sel_a.attr.attr, + &coresight_attr_cfg_dbgbus_sel_b.attr.attr, + &coresight_attr_cfg_dbgbus_sel_c.attr.attr, + &coresight_attr_cfg_dbgbus_sel_d.attr.attr, + &coresight_attr_cfg_dbgbus_cntlt.attr.attr, + &coresight_attr_cfg_dbgbus_cntlm.attr.attr, + &coresight_attr_cfg_dbgbus_opl.attr.attr, + &coresight_attr_cfg_dbgbus_ope.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_0.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_1.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_2.attr.attr, + &coresight_attr_cfg_dbgbus_ivtl_3.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_0.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_1.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_2.attr.attr, + &coresight_attr_cfg_dbgbus_maskl_3.attr.attr, + &coresight_attr_cfg_dbgbus_bytel_0.attr.attr, + &coresight_attr_cfg_dbgbus_bytel_1.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_0.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_1.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_2.attr.attr, + &coresight_attr_cfg_dbgbus_ivte_3.attr.attr, + &coresight_attr_cfg_dbgbus_maske_0.attr.attr, + &coresight_attr_cfg_dbgbus_maske_1.attr.attr, + &coresight_attr_cfg_dbgbus_maske_2.attr.attr, + &coresight_attr_cfg_dbgbus_maske_3.attr.attr, + &coresight_attr_cfg_dbgbus_nibblee.attr.attr, + &coresight_attr_cfg_dbgbus_ptrc0.attr.attr, + &coresight_attr_cfg_dbgbus_ptrc1.attr.attr, + &coresight_attr_cfg_dbgbus_loadreg.attr.attr, + &coresight_attr_cfg_dbgbus_idx.attr.attr, + &coresight_attr_cfg_dbgbus_clrc.attr.attr, + &coresight_attr_cfg_dbgbus_loadivt.attr.attr, + &coresight_attr_cfg_dbgbus_event_logic.attr.attr, + &coresight_attr_cfg_dbgbus_over.attr.attr, + &coresight_attr_cfg_dbgbus_count0.attr.attr, + &coresight_attr_cfg_dbgbus_count1.attr.attr, + &coresight_attr_cfg_dbgbus_count2.attr.attr, + &coresight_attr_cfg_dbgbus_count3.attr.attr, + &coresight_attr_cfg_dbgbus_count4.attr.attr, + &coresight_attr_cfg_dbgbus_count5.attr.attr, + &coresight_attr_cfg_dbgbus_trace_addr.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf0.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf1.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf2.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf3.attr.attr, + &coresight_attr_cfg_dbgbus_trace_buf4.attr.attr, + &coresight_attr_cfg_dbgbus_misr0.attr.attr, + &coresight_attr_cfg_dbgbus_misr1.attr.attr, + &coresight_attr_ahb_dbg_cntl.attr.attr, + &coresight_attr_read_ahb_through_dbg.attr.attr, + &coresight_attr_dbg_lo_hi_gpio.attr.attr, + &coresight_attr_ext_trace_bus_cntl.attr.attr, + &coresight_attr_ext_vbif_dbg_cntl.attr.attr, + NULL, +}; + +static const struct attribute_group a5xx_coresight_group = { + .attrs = a5xx_coresight_attrs, +}; + +static const struct attribute_group *a5xx_coresight_groups[] = { + &a5xx_coresight_group, + NULL, +}; + +static const struct adreno_coresight a5xx_coresight = { + .registers = a5xx_coresight_registers, + .count = ARRAY_SIZE(a5xx_coresight_registers), + .groups = a5xx_coresight_groups, +}; + +void a5xx_coresight_init(struct adreno_device *adreno_dev) +{ + adreno_coresight_add_device(adreno_dev, "coresight-gfx", + &a5xx_coresight, &adreno_dev->gx_coresight); +} diff --git a/qcom/opensource/graphics-kernel/adreno_a5xx_packets.h b/qcom/opensource/graphics-kernel/adreno_a5xx_packets.h new file mode 100644 index 0000000000..55276e46bc --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a5xx_packets.h @@ -0,0 +1,1406 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2016,2019, The Linux Foundation. All rights reserved. + */ + +struct adreno_critical_fixup { + unsigned int lo_offset; + unsigned int hi_offset; + int buffer; + uint64_t mem_offset; +}; + +static unsigned int _a5xx_critical_pkts[] = { + 0x400E0601, /* [0x0000] == TYPE4 == */ + 0x00000002, /* [0x0001] A5X_HLSQ_MODE_CNTL (0x0E06)*/ + 0x40E78A01, /* [0x0002] == TYPE4 == */ + 0x000FFFFF, /* [0x0003] A5X_HLSQ_UPDATE_CNTL_CTX_0 (0xE78A)*/ + 0x48E78401, /* [0x0004] == TYPE4 == */ + 0x00000005, /* [0x0005] A5X_HLSQ_CNTL_0_CTX_0 (0xE784)*/ + 0x40E78501, /* [0x0006] == TYPE4 == */ + 0x00000009, /* [0x0007] A5X_HLSQ_CNTL_1_CTX_0 (0xE785)*/ + 0x48E78B85, /* [0x0008] == TYPE4 == */ + 0x00000001, /* [0x0009] A5X_HLSQ_VS_CONFIG_CTX_0 (0xE78B)*/ + 0x00002085, /* [0x000A] A5X_HLSQ_PS_CONFIG_CTX_0 (0xE78C)*/ + 0x00002084, /* [0x000B] A5X_HLSQ_HS_CONFIG_CTX_0 (0xE78D)*/ + 0x00002084, /* [0x000C] A5X_HLSQ_DS_CONFIG_CTX_0 (0xE78E)*/ + 0x00002084, /* [0x000D] A5X_HLSQ_GS_CONFIG_CTX_0 (0xE78F)*/ + 0x40E58485, /* [0x000E] == TYPE4 == */ + 0x00000001, /* [0x000F] A5X_SP_VS_CONFIG_CTX_0 (0xE584)*/ + 0x00002085, /* [0x0010] A5X_SP_PS_CONFIG_CTX_0 (0xE585)*/ + 0x00002084, /* [0x0011] A5X_SP_HS_CONFIG_CTX_0 (0xE586)*/ + 0x00002084, /* [0x0012] A5X_SP_DS_CONFIG_CTX_0 (0xE587)*/ + 0x00002084, /* [0x0013] A5X_SP_GS_CONFIG_CTX_0 (0xE588)*/ + 0x40E79101, /* [0x0014] == TYPE4 == */ + 0x00000004, /* [0x0015] A5X_HLSQ_VS_CNTL_CTX_0 (0xE791)*/ + 0x40E79201, /* [0x0016] == TYPE4 == */ + 0x00000002, /* [0x0017] A5X_HLSQ_PS_CNTL_CTX_0 (0xE792)*/ + 0x48E58001, /* [0x0018] == TYPE4 == */ + 0x00000010, /* [0x0019] A5X_SP_SP_CNTL_CTX_0 (0xE580)*/ + 0x70B00043, /* [0x001A] == TYPE7: LOAD_STATE (30) == */ + 0x00A00000, /* [0x001B] */ + 0x00000000, /* [0x001C] */ + 0x00000000, /* [0x001D] */ + 0x20020003, /* [0x001E] */ + 0x56D81803, /* [0x001F] */ + 0x00000003, /* [0x0020] */ + 0x20150000, /* [0x0021] */ + 0x00000000, /* [0x0022] */ + 0x00000200, /* [0x0023] */ + 0x00000000, /* [0x0024] */ + 0x201100F4, /* [0x0025] */ + 0x00000000, /* [0x0026] */ + 0x00000500, /* [0x0027] */ + 0x00000C21, /* [0x0028] */ + 0x20154004, /* [0x0029] */ + 0x00000C20, /* [0x002A] */ + 0x20154003, /* [0x002B] */ + 0x00000C23, /* [0x002C] */ + 0x20154008, /* [0x002D] */ + 0x00000C22, /* [0x002E] */ + 0x20156007, /* [0x002F] */ + 0x00000000, /* [0x0030] */ + 0x20554005, /* [0x0031] */ + 0x3F800000, /* [0x0032] */ + 0x20554006, /* [0x0033] */ + 0x00000000, /* [0x0034] */ + 0x03000000, /* [0x0035] */ + 0x20050000, /* [0x0036] */ + 0x46F00009, /* [0x0037] */ + 0x201F0000, /* [0x0038] */ + 0x4398000A, /* [0x0039] */ + 0x201F0009, /* [0x003A] */ + 0x43980809, /* [0x003B] */ + 0x20180009, /* [0x003C] */ + 0x46100809, /* [0x003D] */ + 0x00091014, /* [0x003E] */ + 0x62050009, /* [0x003F] */ + 0x00000000, /* [0x0040] */ + 0x00000500, /* [0x0041] */ + 0x04800006, /* [0x0042] */ + 0xC2C61300, /* [0x0043] */ + 0x0280000E, /* [0x0044] */ + 0xC2C61310, /* [0x0045] */ + 0x00000000, /* [0x0046] */ + 0x04800000, /* [0x0047] */ + 0x00000000, /* [0x0048] */ + 0x05000000, /* [0x0049] */ + 0x00000000, /* [0x004A] */ + 0x00000000, /* [0x004B] */ + 0x00000000, /* [0x004C] */ + 0x00000000, /* [0x004D] */ + 0x00000000, /* [0x004E] */ + 0x00000000, /* [0x004F] */ + 0x00000000, /* [0x0050] */ + 0x00000000, /* [0x0051] */ + 0x00000000, /* [0x0052] */ + 0x00000000, /* [0x0053] */ + 0x00000000, /* [0x0054] */ + 0x00000000, /* [0x0055] */ + 0x00000000, /* [0x0056] */ + 0x00000000, /* [0x0057] */ + 0x00000000, /* [0x0058] */ + 0x00000000, /* [0x0059] */ + 0x00000000, /* [0x005A] */ + 0x00000000, /* [0x005B] */ + 0x00000000, /* [0x005C] */ + 0x00000000, /* [0x005D] */ + 0x70B00023, /* [0x005E] == TYPE7: LOAD_STATE (30) == */ + 0x00700000, /* [0x005F] */ + 0x00000000, /* [0x0060] */ + 0x00000000, /* [0x0061] */ + 0x00000000, /* [0x0062] */ + 0x03000000, /* [0x0063] */ + 0x00000000, /* [0x0064] */ + 0x00000000, /* [0x0065] */ + 0x00000000, /* [0x0066] */ + 0x00000000, /* [0x0067] */ + 0x00000000, /* [0x0068] */ + 0x00000000, /* [0x0069] */ + 0x00000000, /* [0x006A] */ + 0x00000000, /* [0x006B] */ + 0x00000000, /* [0x006C] */ + 0x00000000, /* [0x006D] */ + 0x00000000, /* [0x006E] */ + 0x00000000, /* [0x006F] */ + 0x00000000, /* [0x0070] */ + 0x00000000, /* [0x0071] */ + 0x00000000, /* [0x0072] */ + 0x00000000, /* [0x0073] */ + 0x00000000, /* [0x0074] */ + 0x00000000, /* [0x0075] */ + 0x00000000, /* [0x0076] */ + 0x00000000, /* [0x0077] */ + 0x00000000, /* [0x0078] */ + 0x00000000, /* [0x0079] */ + 0x00000000, /* [0x007A] */ + 0x00000000, /* [0x007B] */ + 0x00000000, /* [0x007C] */ + 0x00000000, /* [0x007D] */ + 0x00000000, /* [0x007E] */ + 0x00000000, /* [0x007F] */ + 0x00000000, /* [0x0080] */ + 0x00000000, /* [0x0081] */ + 0x70B08003, /* [0x0082] == TYPE7: LOAD_STATE (30) == */ + 0x00620000, /* [0x0083] */ + 0x00000000, /* [0x0084] */ + 0x00000000, /* [0x0085] */ + 0x70B08003, /* [0x0086] == TYPE7: LOAD_STATE (30) == */ + 0x01220008, /* [0x0087] */ + 0x00000000, /* [0x0088] */ + 0x00000000, /* [0x0089] */ + 0x70B0000B, /* [0x008A] == TYPE7: LOAD_STATE (30) == */ + 0x01180000, /* [0x008B] */ + 0x00000001, /* [0x008C] */ + 0x00000000, /* [0x008D] */ + 0x00000000, /* [0x008E] */ + 0x00000000, /* [0x008F] */ + 0x00000000, /* [0x0090] */ + 0x00000000, /* [0x0091] */ + 0x00000000, /* [0x0092] */ + 0x00000000, /* [0x0093] */ + 0x00000000, /* [0x0094] */ + 0x01400000, /* [0x0095] */ + 0x70460001, /* [0x0096] == TYPE7: EVENT_WRITE (46) == */ + 0x00000019, /* [0x0097] */ + 0x70460004, /* [0x0098] == TYPE7: EVENT_WRITE (46) == */ + 0x0000001D, /* [0x0099] */ + 0x00000000, /* [0x009A] */ + 0x00000000, /* [0x009B] */ + 0x00000001, /* [0x009C] */ + 0x70460004, /* [0x009D] == TYPE7: EVENT_WRITE (46) == */ + 0x0000001C, /* [0x009E] */ + 0x00000000, /* [0x009F] */ + 0x00000000, /* [0x00A0] */ + 0x00000001, /* [0x00A1] */ + 0x480E9185, /* [0x00A2] == TYPE4 == */ + 0x00000000, /* [0x00A3] A5X_UCHE_CACHE_INVALIDATE_MIN_LO (0x0E91)*/ + 0x00000000, /* [0x00A4] A5X_UCHE_CACHE_INVALIDATE_MIN_HI (0x0E92)*/ + 0x00000000, /* [0x00A5] A5X_UCHE_CACHE_INVALIDATE_MAX_LO (0x0E93)*/ + 0x00000000, /* [0x00A6] A5X_UCHE_CACHE_INVALIDATE_MAX_HI (0x0E94)*/ + 0x00000012, /* [0x00A7] A5X_UCHE_CACHE_INVALIDATE (0x0E95)*/ + 0x70268000, /* [0x00A8] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x40E78A01, /* [0x00A9] == TYPE4 == */ + 0x000FFFFF, /* [0x00AA] A5X_HLSQ_UPDATE_CNTL_CTX_0 (0xE78A)*/ + 0x70D08003, /* [0x00AB] == TYPE7: PERFCOUNTER_ACTION (50) == */ + 0x00000000, /* [0x00AC] */ + 0x00000000, /* [0x00AD] */ + 0x00000000, /* [0x00AE] */ + 0x70D08003, /* [0x00AF] == TYPE7: PERFCOUNTER_ACTION (50) == */ + 0x00000010, /* [0x00B0] */ + 0x00000000, /* [0x00B1] */ + 0x00000000, /* [0x00B2] */ + 0x70268000, /* [0x00B3] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x48E38C01, /* [0x00B4] == TYPE4 == */ + 0xFFFFFFFF, /* [0x00B5] A5X_PC_RESTART_INDEX_CTX_0 (0xE38C)*/ + 0x40E38801, /* [0x00B6] == TYPE4 == */ + 0x00000012, /* [0x00B7] A5X_PC_RASTER_CNTL_CTX_0 (0xE388)*/ + 0x48E09102, /* [0x00B8] == TYPE4 == */ + 0xFFC00010, /* [0x00B9] A5X_GRAS_SU_POINT_MINMAX_CTX_0 (0xE091)*/ + 0x00000008, /* [0x00BA] A5X_GRAS_SU_POINT_SIZE_CTX_0 (0xE092)*/ + 0x40E09901, /* [0x00BB] == TYPE4 == */ + 0x00000000, /* [0x00BC] A5X_GRAS_SU_CONSERVATIVE_RAS_CNTL_CTX_0 + * (0xE099) + */ + 0x48E0A401, /* [0x00BD] == TYPE4 == */ + 0x00000000, /* [0x00BE] A5X_GRAS_SC_SCREEN_SCISSOR_CNTL_CTX_0 (0xE0A4)*/ + 0x48E58A01, /* [0x00BF] == TYPE4 == */ + 0x00000000, /* [0x00C0] A5X_SP_VS_CONFIG_MAX_CONST_CTX_0 (0xE58A)*/ + 0x40E58B01, /* [0x00C1] == TYPE4 == */ + 0x00000000, /* [0x00C2] A5X_SP_PS_CONFIG_MAX_CONST_CTX_0 (0xE58B)*/ + 0x480CC601, /* [0x00C3] == TYPE4 == */ + 0x00000044, /* [0x00C4] A5X_RB_MODE_CNTL (0x0CC6)*/ + 0x400CC401, /* [0x00C5] == TYPE4 == */ + 0x00100000, /* [0x00C6] A5X_RB_DBG_ECO_CNTL (0x0CC4)*/ + 0x400E4201, /* [0x00C7] == TYPE4 == */ + 0x00000000, /* [0x00C8] A5X_VFD_MODE_CNTL (0x0E42)*/ + 0x480D0201, /* [0x00C9] == TYPE4 == */ + 0x0000001F, /* [0x00CA] A5X_PC_MODE_CNTL (0x0D02)*/ + 0x480EC201, /* [0x00CB] == TYPE4 == */ + 0x0000001E, /* [0x00CC] A5X_SP_MODE_CNTL (0x0EC2)*/ + 0x400EC001, /* [0x00CD] == TYPE4 == */ + 0x40000800, /* [0x00CE] A5X_SP_DBG_ECO_CNTL (0x0EC0)*/ + 0x400F0201, /* [0x00CF] == TYPE4 == */ + 0x00000544, /* [0x00D0] A5X_TPL1_MODE_CNTL (0x0F02)*/ + 0x400E0002, /* [0x00D1] == TYPE4 == */ + 0x00000080, /* [0x00D2] A5X_HLSQ_TIMEOUT_THRESHOLD_0 (0x0E00)*/ + 0x00000000, /* [0x00D3] A5X_HLSQ_TIMEOUT_THRESHOLD_1 (0x0E01)*/ + 0x400E6001, /* [0x00D4] == TYPE4 == */ + 0x00000400, /* [0x00D5] A5X_VPC_DBG_ECO_CNTL (0x0E60)*/ + 0x400E0601, /* [0x00D6] == TYPE4 == */ + 0x00000001, /* [0x00D7] A5X_HLSQ_MODE_CNTL (0x0E06)*/ + 0x480E6201, /* [0x00D8] == TYPE4 == */ + 0x00000000, /* [0x00D9] A5X_VPC_MODE_CNTL (0x0E62)*/ + 0x70EC8005, /* [0x00DA] == TYPE7: SET_RENDER_MODE (6C) == */ + 0x00000002, /* [0x00DB] */ + 0x00000000, /* [0x00DC] */ + 0x00000000, /* [0x00DD] */ + 0x00000008, /* [0x00DE] */ + 0x00000001, /* [0x00DF] */ + 0x40E14001, /* [0x00E0] == TYPE4 == */ + 0x00000204, /* [0x00E1] A5X_RB_CNTL_CTX_0 (0xE140)*/ + 0x709D0001, /* [0x00E2] == TYPE7: SKIP_IB2_ENABLE_GLOBAL (1D) == */ + 0x00000000, /* [0x00E3] */ + 0x48E0EA02, /* [0x00E4] == TYPE4 == */ + 0x00000000, /* [0x00E5] A5X_GRAS_SC_WINDOW_SCISSOR_TL_CTX_0 (0xE0EA)*/ + 0x001F0073, /* [0x00E6] A5X_GRAS_SC_WINDOW_SCISSOR_BR_CTX_0 (0xE0EB)*/ + 0x48E21102, /* [0x00E7] == TYPE4 == */ + 0x00000000, /* [0x00E8] A5X_RB_RESOLVE_CNTL_1_CTX_0 (0xE211)*/ + 0x00000000, /* [0x00E9] A5X_RB_RESOLVE_CNTL_2_CTX_0 (0xE212)*/ + 0x480BC283, /* [0x00EA] == TYPE4 == */ + 0x00000204, /* [0x00EB] UNKNOWN (0x0BC2)*/ + 0x00000000, /* [0x00EC] UNKNOWN (0x0BC3)*/ + 0x00000000, /* [0x00ED] UNKNOWN (0x0BC4)*/ + 0x400BC502, /* [0x00EE] == TYPE4 == */ + 0x00000000, /* [0x00EF] UNKNOWN (0x0BC5)*/ + 0x00000000, /* [0x00F0] UNKNOWN (0x0BC6)*/ + 0x480BD001, /* [0x00F1] == TYPE4 == */ + 0x01100000, /* [0x00F2] UNKNOWN (0x0BD0)*/ + 0x480BE002, /* [0x00F3] == TYPE4 == */ + 0x00000000, /* [0x00F4] UNKNOWN (0x0BE0)*/ + 0x00000000, /* [0x00F5] UNKNOWN (0x0BE1)*/ + 0x480C0001, /* [0x00F6] == TYPE4 == */ + 0x00000020, /* [0x00F7] A5X_VSC_PIPE_DATA_LENGTH_0 (0x0C00)*/ + 0x48E3B001, /* [0x00F8] == TYPE4 == */ + 0x00000003, /* [0x00F9] A5X_PC_POWER_CNTL_CTX_0 (0xE3B0)*/ + 0x48E4F001, /* [0x00FA] == TYPE4 == */ + 0x00000003, /* [0x00FB] A5X_VFD_POWER_CNTL_CTX_0 (0xE4F0)*/ + 0x480E6201, /* [0x00FC] == TYPE4 == */ + 0x00000001, /* [0x00FD] A5X_VPC_MODE_CNTL (0x0E62)*/ + 0x70460001, /* [0x00FE] == TYPE7: EVENT_WRITE (46) == */ + 0x0000002C, /* [0x00FF] */ + 0x40E1D001, /* [0x0100] == TYPE4 == */ + 0x00000000, /* [0x0101] A5X_RB_WINDOW_OFFSET_CTX_0 (0xE1D0)*/ + 0x70BF8003, /* [0x0102] == TYPE7: INDIRECT_BUFFER_PFE (3F) == */ + 0x00000000, /* [0x0103] */ + 0x00000000, /* [0x0104] */ + 0x000000A0, /* [0x0105] */ + 0x70460001, /* [0x0106] == TYPE7: EVENT_WRITE (46) == */ + 0x0000002D, /* [0x0107] */ + 0x70460004, /* [0x0108] == TYPE7: EVENT_WRITE (46) == */ + 0x00000004, /* [0x0109] */ + 0x00000000, /* [0x010A] */ + 0x00000000, /* [0x010B] */ + 0x00000000, /* [0x010C] */ + 0x70268000, /* [0x010D] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x480E6201, /* [0x010E] == TYPE4 == */ + 0x00000000, /* [0x010F] A5X_VPC_MODE_CNTL (0x0E62)*/ + 0x48E3B001, /* [0x0110] == TYPE4 == */ + 0x00000003, /* [0x0111] A5X_PC_POWER_CNTL_CTX_0 (0xE3B0)*/ + 0x48E4F001, /* [0x0112] == TYPE4 == */ + 0x00000003, /* [0x0113] A5X_VFD_POWER_CNTL_CTX_0 (0xE4F0)*/ + 0x70268000, /* [0x0114] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x400CC701, /* [0x0115] == TYPE4 == */ + 0x7C13C080, /* [0x0116] A5X_RB_CCU_CNTL (0x0CC7)*/ + 0x70EC8005, /* [0x0117] == TYPE7: SET_RENDER_MODE (6C) == */ + 0x00000001, /* [0x0118] */ + 0x00000000, /* [0x0119] */ + 0x00000000, /* [0x011A] */ + 0x00000010, /* [0x011B] */ + 0x00000001, /* [0x011C] */ + 0x70EA0001, /* [0x011D] == TYPE7: PREEMPT_ENABLE_LOCAL (6A) == */ + 0x00000000, /* [0x011E] */ + 0x48E0EA02, /* [0x011F] == TYPE4 == */ + 0x00000000, /* [0x0120] A5X_GRAS_SC_WINDOW_SCISSOR_TL_CTX_0 (0xE0EA)*/ + 0x001F0073, /* [0x0121] A5X_GRAS_SC_WINDOW_SCISSOR_BR_CTX_0 (0xE0EB)*/ + 0x48E21102, /* [0x0122] == TYPE4 == */ + 0x00000000, /* [0x0123] A5X_RB_RESOLVE_CNTL_1_CTX_0 (0xE211)*/ + 0x00030007, /* [0x0124] A5X_RB_RESOLVE_CNTL_2_CTX_0 (0xE212)*/ + 0x70138000, /* [0x0125] == TYPE7: WAIT_FOR_ME (13) == */ + 0x70640001, /* [0x0126] == TYPE7: SET_VISIBILITY_OVERRIDE (64) == */ + 0x00000000, /* [0x0127] */ + 0x702F8005, /* [0x0128] == TYPE7: SET_BIN_DATA (2F) == */ + 0x00010000, /* [0x0129] */ + 0x00000000, /* [0x012A] */ + 0x00000000, /* [0x012B] */ + 0x00000000, /* [0x012C] */ + 0x00000000, /* [0x012D] */ + 0x40E1D001, /* [0x012E] == TYPE4 == */ + 0x00000000, /* [0x012F] A5X_RB_WINDOW_OFFSET_CTX_0 (0xE1D0)*/ + 0x40E2A201, /* [0x0130] == TYPE4 == */ + 0x00000001, /* [0x0131] A5X_VPC_SO_OVERRIDE_CTX_0 (0xE2A2)*/ + 0x70640001, /* [0x0132] == TYPE7: SET_VISIBILITY_OVERRIDE (64) == */ + 0x00000000, /* [0x0133] */ + 0x48E1B285, /* [0x0134] == TYPE4 == */ + 0x00000001, /* [0x0135] A5X_RB_DEPTH_BUFFER_INFO_CTX_0 (0xE1B2)*/ + 0x00004000, /* [0x0136] A5X_RB_DEPTH_BUFFER_BASE_LO_CTX_0 (0xE1B3)*/ + 0x00000000, /* [0x0137] A5X_RB_DEPTH_BUFFER_BASE_HI_CTX_0 (0xE1B4)*/ + 0x00000004, /* [0x0138] A5X_RB_DEPTH_BUFFER_PITCH_CTX_0 (0xE1B5)*/ + 0x000000C0, /* [0x0139] A5X_RB_DEPTH_BUFFER_ARRAY_PITCH_CTX_0 (0xE1B6)*/ + 0x48E09801, /* [0x013A] == TYPE4 == */ + 0x00000001, /* [0x013B] A5X_GRAS_SU_DEPTH_BUFFER_INFO_CTX_0 (0xE098)*/ + 0x40E24083, /* [0x013C] == TYPE4 == */ + 0x00000000, /* [0x013D] A5X_RB_DEPTH_FLAG_BUFFER_BASE_LO_CTX_0 + * (0xE240) + */ + 0x00000000, /* [0x013E] A5X_RB_DEPTH_FLAG_BUFFER_BASE_HI_CTX_0 + * (0xE241) + */ + 0x00000000, /* [0x013F] A5X_RB_DEPTH_FLAG_BUFFER_PITCH_CTX_0 (0xE242)*/ + 0x40E15285, /* [0x0140] == TYPE4 == */ + 0x00001230, /* [0x0141] A5X_RB_MRT_BUFFER_INFO_0_CTX_0 (0xE152)*/ + 0x00000008, /* [0x0142] A5X_RB_MRT_BUFFER_PITCH_0_CTX_0 (0xE153)*/ + 0x00000100, /* [0x0143] A5X_RB_MRT_BUFFER_ARRAY_PITCH_0_CTX_0 (0xE154)*/ + 0x00000000, /* [0x0144] A5X_RB_MRT_BUFFER_BASE_LO_0_CTX_0 (0xE155)*/ + 0x00000000, /* [0x0145] A5X_RB_MRT_BUFFER_BASE_HI_0_CTX_0 (0xE156)*/ + 0x40E40801, /* [0x0146] == TYPE4 == */ + 0x00000000, /* [0x0147] A5X_VFD_INDEX_OFFSET_CTX_0 (0xE408)*/ + 0x48E40901, /* [0x0148] == TYPE4 == */ + 0x00000000, /* [0x0149] A5X_VFD_INSTANCE_START_OFFSET_CTX_0 (0xE409)*/ + 0x70BF8003, /* [0x014A] == TYPE7: INDIRECT_BUFFER_PFE (3F) == */ + 0x00000000, /* [0x014B] */ + 0x00000000, /* [0x014C] */ + 0x00000112, /* [0x014D] */ + 0x70230001, /* [0x014E] == TYPE7: SKIP_IB2_ENABLE_LOCAL (23) == */ + 0x00000000, /* [0x014F] */ + 0x70BF8003, /* [0x0150] == TYPE7: INDIRECT_BUFFER_PFE (3F) == */ + 0x00000000, /* [0x0151] */ + 0x00000000, /* [0x0152] */ + 0x0000001B, /* [0x0153] */ + 0x70EC8005, /* [0x0154] == TYPE7: SET_RENDER_MODE (6C) == */ + 0x00000001, /* [0x0155] */ + 0x00000000, /* [0x0156] */ + 0x00000000, /* [0x0157] */ + 0x00000000, /* [0x0158] */ + 0x00000001, /* [0x0159] */ + 0x70438003, /* [0x015A] == TYPE7: SET_DRAW_STATE (43) == */ + 0x00080059, /* [0x015B] */ + 0x00000000, /* [0x015C] */ + 0x00000000, /* [0x015D] */ + 0x70388003, /* [0x015E] == TYPE7: DRAW_INDX_OFFSET (38) == */ + 0x00000888, /* [0x015F] */ + 0x00000000, /* [0x0160] */ + 0x00000002, /* [0x0161] */ + 0x70A88003, /* [0x0162] == TYPE7: DRAW_INDIRECT (28) == */ + 0x00200884, /* [0x0163] */ + 0x00000000, /* [0x0164] */ + 0x00000000, /* [0x0165] */ + 0x70298006, /* [0x0166] == TYPE7: DRAW_INDX_INDIRECT (29) == */ + 0x00200404, /* [0x0167] */ + 0x00000000, /* [0x0168] */ + 0x00000000, /* [0x0169] */ + 0x00000006, /* [0x016A] */ + 0x00000000, /* [0x016B] */ + 0x00000000, /* [0x016C] */ + 0x40E2A783, /* [0x016D] == TYPE4 == */ + 0x00000000, /* [0x016E] A5X_VPC_SO_BUFFER_BASE_LO_0_CTX_0 (0xE2A7)*/ + 0x00000000, /* [0x016F] A5X_VPC_SO_BUFFER_BASE_HI_0_CTX_0 (0xE2A8)*/ + 0x00000004, /* [0x0170] A5X_VPC_SO_BUFFER_SIZE_0_CTX_0 (0xE2A9)*/ + 0x48E2AC02, /* [0x0171] == TYPE4 == */ + 0x00000000, /* [0x0172] A5X_VPC_SO_FLUSH_BASE_LO_0_CTX_0 (0xE2AC)*/ + 0x00000000, /* [0x0173] A5X_VPC_SO_FLUSH_BASE_HI_0_CTX_0 (0xE2AD)*/ + 0x70460001, /* [0x0174] == TYPE7: EVENT_WRITE (46) == */ + 0x00000011, /* [0x0175] */ + 0x48E10001, /* [0x0176] == TYPE4 == */ + 0x00000009, /* [0x0177] A5X_GRAS_LRZ_CNTL_CTX_0 (0xE100)*/ + 0x70460001, /* [0x0178] == TYPE7: EVENT_WRITE (46) == */ + 0x00000026, /* [0x0179] */ + 0x48E10001, /* [0x017A] == TYPE4 == */ + 0x00000008, /* [0x017B] A5X_GRAS_LRZ_CNTL_CTX_0 (0xE100)*/ + 0x40E10185, /* [0x017C] == TYPE4 == */ + 0x00000000, /* [0x017D] A5X_GRAS_LRZ_BUFFER_BASE_LO_CTX_0 (0xE101)*/ + 0x00000000, /* [0x017E] A5X_GRAS_LRZ_BUFFER_BASE_HI_CTX_0 (0xE102)*/ + 0x00000001, /* [0x017F] A5X_GRAS_LRZ_BUFFER_PITCH_CTX_0 (0xE103)*/ + 0x00000000, /* [0x0180] A5X_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_LO_CTX_0 + * (0xE104) + */ + 0x00000000, /* [0x0181] A5X_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_HI_CTX_0 + * (0xE105) + */ + 0x70460001, /* [0x0182] == TYPE7: EVENT_WRITE (46) == */ + 0x00000025, /* [0x0183] */ + 0x70460001, /* [0x0184] == TYPE7: EVENT_WRITE (46) == */ + 0x00000019, /* [0x0185] */ + 0x70460001, /* [0x0186] == TYPE7: EVENT_WRITE (46) == */ + 0x00000018, /* [0x0187] */ + 0x70EA0001, /* [0x0188] == TYPE7: PREEMPT_ENABLE_LOCAL (6A) == */ + 0x00000000, /* [0x0189] */ + 0x70EC0001, /* [0x018A] == TYPE7: SET_RENDER_MODE (6C) == */ + 0x00000006, /* [0x018B] */ + 0x70438003, /* [0x018C] == TYPE7: SET_DRAW_STATE (43) == */ + 0x00080059, /* [0x018D] */ + 0x00000000, /* [0x018E] */ + 0x00000000, /* [0x018F] */ + 0x70DC0002, /* [0x0190] == TYPE7: CONTEXT_REG_BUNCH (5C) == */ + 0x0000E2A1, /* [0x0191] */ + 0x00008001, /* [0x0192] */ + 0x709D0001, /* [0x0193] == TYPE7: SKIP_IB2_ENABLE_GLOBAL (1D) == */ + 0x00000000, /* [0x0194] */ + 0x70138000, /* [0x0195] == TYPE7: WAIT_FOR_ME (13) == */ + 0x70640001, /* [0x0196] == TYPE7: SET_VISIBILITY_OVERRIDE (64) == */ + 0x00000001, /* [0x0197] */ + 0x70380007, /* [0x0198] == TYPE7: DRAW_INDX_OFFSET (38) == */ + 0x00200506, /* [0x0199] */ + 0x00000000, /* [0x019A] */ + 0x00000004, /* [0x019B] */ + 0x00000000, /* [0x019C] */ + 0x00000000, /* [0x019D] */ + 0x00000000, /* [0x019E] */ + 0x00000004, /* [0x019F] */ + 0x703D8005, /* [0x01A0] == TYPE7: MEM_WRITE (3D) == */ + 0x00000000, /* [0x01A1] */ + 0x00000000, /* [0x01A2] */ + 0x00000001, /* [0x01A3] */ + 0x00000001, /* [0x01A4] */ + 0x00000001, /* [0x01A5] */ + 0x70928000, /* [0x01A6] == TYPE7: WAIT_MEM_WRITES (12) == */ + 0x70BF8003, /* [0x01A7] == TYPE7: INDIRECT_BUFFER_PFE (3F) == */ + 0x00000000, /* [0x01A8] */ + 0x00000000, /* [0x01A9] */ + 0x00000028, /* [0x01AA] */ + 0x70C48006, /* [0x01AB] == TYPE7: COND_EXEC (44) == */ + 0x00000000, /* [0x01AC] */ + 0x00000000, /* [0x01AD] */ + 0x00000000, /* [0x01AE] */ + 0x00000000, /* [0x01AF] */ + 0x00000001, /* [0x01B0] */ + 0x00000002, /* [0x01B1] */ + 0x70100001, /* [0x01B2] == TYPE7: NOP (10) == */ + 0x00000000, /* [0x01B3] */ + 0x70C28003, /* [0x01B4] == TYPE7: MEM_TO_REG (42) == */ + 0xC000E2AB, /* [0x01B5] */ + 0x00000000, /* [0x01B6] */ + 0x00000000, /* [0x01B7] */ + 0x70230001, /* [0x01B8] == TYPE7: SKIP_IB2_ENABLE_LOCAL (23) == */ + 0x00000000, /* [0x01B9] */ + 0x70E90001, /* [0x01BA] == TYPE7: PREEMPT_ENABLE_GLOBAL (69) == */ + 0x00000000, /* [0x01BB] */ + 0x70BC8006, /* [0x01BC] == TYPE7: WAIT_REG_MEM (3C) == */ + 0x00000010, /* [0x01BD] */ + 0x00000000, /* [0x01BE] */ + 0x00000000, /* [0x01BF] */ + 0x00000001, /* [0x01C0] */ + 0xFFFFFFFF, /* [0x01C1] */ + 0x00000001, /* [0x01C2] */ + 0x70738009, /* [0x01C3] == TYPE7: MEM_TO_MEM (73) == */ + 0x20000004, /* [0x01C4] */ + 0x00000000, /* [0x01C5] */ + 0x00000000, /* [0x01C6] */ + 0x00000000, /* [0x01C7] */ + 0x00000000, /* [0x01C8] */ + 0x00000000, /* [0x01C9] */ + 0x00000000, /* [0x01CA] */ + 0x00000000, /* [0x01CB] */ + 0x00000000, /* [0x01CC] */ + 0x70738009, /* [0x01CD] == TYPE7: MEM_TO_MEM (73) == */ + 0xE0000004, /* [0x01CE] */ + 0x00000000, /* [0x01CF] */ + 0x00000000, /* [0x01D0] */ + 0x00000000, /* [0x01D1] */ + 0x00000000, /* [0x01D2] */ + 0x00000000, /* [0x01D3] */ + 0x00000000, /* [0x01D4] */ + 0x00000000, /* [0x01D5] */ + 0x00000000, /* [0x01D6] */ + 0x70B50001, /* [0x01D7] == TYPE7: SET_SUBDRAW_SIZE (35) == */ + 0x00000001, /* [0x01D8] */ + 0x40E78A01, /* [0x01D9] == TYPE4 == */ + 0x000FFFFF, /* [0x01DA] A5X_HLSQ_UPDATE_CNTL_CTX_0 (0xE78A)*/ + 0x70268000, /* [0x01DB] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x400E0601, /* [0x01DC] == TYPE4 == */ + 0x00000001, /* [0x01DD] A5X_HLSQ_MODE_CNTL (0x0E06)*/ + 0x706E0004, /* [0x01DE] == TYPE7: COMPUTE_CHECKPOINT (6E) == */ + 0x00000000, /* [0x01DF] */ + 0x00000000, /* [0x01E0] */ + 0x00000018, /* [0x01E1] */ + 0x00000001, /* [0x01E2] */ + 0x40E14001, /* [0x01E3] == TYPE4 == */ + 0x00020000, /* [0x01E4] A5X_RB_CNTL_CTX_0 (0xE140)*/ + 0x40E78A01, /* [0x01E5] == TYPE4 == */ + 0x01F00000, /* [0x01E6] A5X_HLSQ_UPDATE_CNTL_CTX_0 (0xE78A)*/ + 0x70268000, /* [0x01E7] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x48E38C01, /* [0x01E8] == TYPE4 == */ + 0xFFFFFFFF, /* [0x01E9] A5X_PC_RESTART_INDEX_CTX_0 (0xE38C)*/ + 0x480D0201, /* [0x01EA] == TYPE4 == */ + 0x0000001F, /* [0x01EB] A5X_PC_MODE_CNTL (0x0D02)*/ + 0x480EC201, /* [0x01EC] == TYPE4 == */ + 0x0000001E, /* [0x01ED] A5X_SP_MODE_CNTL (0x0EC2)*/ + 0x48E58001, /* [0x01EE] == TYPE4 == */ + 0x00000000, /* [0x01EF] A5X_SP_SP_CNTL_CTX_0 (0xE580)*/ + 0x40E2A201, /* [0x01F0] == TYPE4 == */ + 0x00000001, /* [0x01F1] A5X_VPC_SO_OVERRIDE_CTX_0 (0xE2A2)*/ + 0x70640001, /* [0x01F2] == TYPE7: SET_VISIBILITY_OVERRIDE (64) == */ + 0x00000001, /* [0x01F3] */ + 0x48E78401, /* [0x01F4] == TYPE4 == */ + 0x00000881, /* [0x01F5] A5X_HLSQ_CNTL_0_CTX_0 (0xE784)*/ + 0x40E5F001, /* [0x01F6] == TYPE4 == */ + 0x00000C06, /* [0x01F7] A5X_SP_CS_CNTL_0_CTX_0 (0xE5F0)*/ + 0x48E79001, /* [0x01F8] == TYPE4 == */ + 0x00000001, /* [0x01F9] A5X_HLSQ_CS_CONFIG_CTX_0 (0xE790)*/ + 0x48E79601, /* [0x01FA] == TYPE4 == */ + 0x00000005, /* [0x01FB] A5X_HLSQ_CS_CNTL_CTX_0 (0xE796)*/ + 0x48E58901, /* [0x01FC] == TYPE4 == */ + 0x00000001, /* [0x01FD] A5X_SP_CS_CONFIG_CTX_0 (0xE589)*/ + 0x40E7DC01, /* [0x01FE] == TYPE4 == */ + 0x00000030, /* [0x01FF] A5X_HLSQ_CONTEXT_SWITCH_CS_SW_3_CTX_0 (0xE7DC)*/ + 0x48E7DD01, /* [0x0200] == TYPE4 == */ + 0x00000002, /* [0x0201] A5X_HLSQ_CONTEXT_SWITCH_CS_SW_4_CTX_0 (0xE7DD)*/ + 0x40E7B001, /* [0x0202] == TYPE4 == */ + 0x00000003, /* [0x0203] A5X_HLSQ_CS_NDRANGE_0_CTX_0 (0xE7B0)*/ + 0x48E7B702, /* [0x0204] == TYPE4 == */ + 0x00FCC0CF, /* [0x0205] A5X_HLSQ_CS_CNTL_0_CTX_0 (0xE7B7)*/ + 0x00000000, /* [0x0206] A5X_HLSQ_CS_CNTL_1_CTX_0 (0xE7B8)*/ + 0x40E7B983, /* [0x0207] == TYPE4 == */ + 0x00000001, /* [0x0208] A5X_HLSQ_CS_KERNEL_GROUP_X_CTX_0 (0xE7B9)*/ + 0x00000001, /* [0x0209] A5X_HLSQ_CS_KERNEL_GROUP_Y_CTX_0 (0xE7BA)*/ + 0x00000001, /* [0x020A] A5X_HLSQ_CS_KERNEL_GROUP_Z_CTX_0 (0xE7BB)*/ + 0x70B08003, /* [0x020B] == TYPE7: LOAD_STATE (30) == */ + 0x00B60000, /* [0x020C] */ + 0x00000000, /* [0x020D] */ + 0x00000000, /* [0x020E] */ + 0x70B08003, /* [0x020F] == TYPE7: LOAD_STATE (30) == */ + 0x01360008, /* [0x0210] */ + 0x00000000, /* [0x0211] */ + 0x00000000, /* [0x0212] */ + 0x70B0000B, /* [0x0213] == TYPE7: LOAD_STATE (30) == */ + 0x00BC0000, /* [0x0214] */ + 0x00000000, /* [0x0215] */ + 0x00000000, /* [0x0216] */ + 0x00000000, /* [0x0217] */ + 0x00000000, /* [0x0218] */ + 0x00000000, /* [0x0219] */ + 0x00000000, /* [0x021A] */ + 0x00000000, /* [0x021B] */ + 0x00000000, /* [0x021C] */ + 0x00000000, /* [0x021D] */ + 0x00000000, /* [0x021E] */ + 0x70B00007, /* [0x021F] == TYPE7: LOAD_STATE (30) == */ + 0x00BC0000, /* [0x0220] */ + 0x00000001, /* [0x0221] */ + 0x00000000, /* [0x0222] */ + 0x00040000, /* [0x0223] */ + 0x00000000, /* [0x0224] */ + 0x00040000, /* [0x0225] */ + 0x00000000, /* [0x0226] */ + 0x70B00007, /* [0x0227] == TYPE7: LOAD_STATE (30) == */ + 0x00BC0000, /* [0x0228] */ + 0x00000002, /* [0x0229] */ + 0x00000000, /* [0x022A] */ + 0x00000000, /* [0x022B] */ + 0x00000000, /* [0x022C] */ + 0x00000000, /* [0x022D] */ + 0x00000000, /* [0x022E] */ + 0x48E7B186, /* [0x022F] == TYPE4 == */ + 0x00000001, /* [0x0230] A5X_HLSQ_CS_NDRANGE_1_CTX_0 (0xE7B1)*/ + 0x00000000, /* [0x0231] A5X_HLSQ_CS_NDRANGE_2_CTX_0 (0xE7B2)*/ + 0x00000001, /* [0x0232] A5X_HLSQ_CS_NDRANGE_3_CTX_0 (0xE7B3)*/ + 0x00000000, /* [0x0233] A5X_HLSQ_CS_NDRANGE_4_CTX_0 (0xE7B4)*/ + 0x00000001, /* [0x0234] A5X_HLSQ_CS_NDRANGE_5_CTX_0 (0xE7B5)*/ + 0x00000000, /* [0x0235] A5X_HLSQ_CS_NDRANGE_6_CTX_0 (0xE7B6)*/ + 0x70B30004, /* [0x0236] == TYPE7: EXEC_CS (33) == */ + 0x00000000, /* [0x0237] */ + 0x00000001, /* [0x0238] */ + 0x00000001, /* [0x0239] */ + 0x00000001, /* [0x023A] */ + 0x480E6201, /* [0x023B] == TYPE4 == */ + 0x00000001, /* [0x023C] A5X_VPC_MODE_CNTL (0x0E62)*/ +}; + +/* + * These are fixups for the addresses _a5xx_critical_pkts[]. The first two + * numbers are the dword offsets into the buffer above. The third enum is a + * clue as to which buffer is being patched in and the final number is an offset + * in said buffer. + */ +static const struct adreno_critical_fixup critical_pkt_fixups[] = { + { 132, 133, 2, 0x0000 }, + { 136, 137, 2, 0x0001 }, + { 154, 155, 2, 0x0100 }, + { 159, 160, 2, 0x0104 }, + { 173, 174, 2, 0x0200 }, + { 177, 178, 2, 0x0300 }, + { 236, 237, 0, 0x0000 }, + { 244, 245, 0, 0x0040 }, + { 259, 260, 3, 0x0000 }, + { 266, 267, 2, 0x0108 }, + { 298, 299, 0, 0x0040 }, + { 300, 301, 2, 0x0080 }, + { 331, 332, 3, 0x02A0 }, + { 337, 338, 3, 0x0700 }, + { 348, 349, 3, 0x0920 }, + { 356, 357, 1, 0x008C }, + { 360, 361, 1, 0x0080 }, + { 363, 364, 1, 0x008C }, + { 366, 367, 0, 0x0100 }, + { 370, 371, 0, 0x0120 }, + { 381, 382, 1, 0x0480 }, + { 384, 385, 1, 0x0400 }, + { 398, 399, 3, 0x0920 }, + { 413, 414, 1, 0x0080 }, + { 417, 418, 1, 0x0300 }, + { 424, 425, 3, 0x0880 }, + { 428, 429, 1, 0x0300 }, + { 430, 431, 1, 0x0300 }, + { 438, 439, 1, 0x0300 }, + { 446, 447, 1, 0x0300 }, + { 453, 454, 1, 0x0320 }, + { 455, 456, 1, 0x0300 }, + { 457, 458, 1, 0x0304 }, + { 459, 460, 1, 0x0308 }, + { 463, 464, 1, 0x0320 }, + { 465, 466, 1, 0x0300 }, + { 467, 468, 1, 0x0304 }, + { 469, 470, 1, 0x0308 }, + { 525, 526, 1, 0x0160 }, + { 529, 530, 1, 0x0101 }, + { 535, 536, 1, 0x0140 }, + { 539, 540, 0, 0x0800 }, + { 555, 556, 1, 0x0140 }, + { 557, 558, 0, 0x0800 }, +}; + +static unsigned int _a5xx_critical_pkts_mem01[] = { + 0xBECCCCCD, 0x00000000, 0x3ECCCCCD, 0x00000000, 0x3ECCCCCD, 0x3ECCCCCD, + 0xBECCCCCD, 0x00000000, 0xBECCCCCD, 0x3ECCCCCD, 0x3ECCCCCD, 0x00000000, + 0xBECCCCCD, 0x00000000, 0x3ECCCCCD, 0x00000000, 0x3ECCCCCD, 0xBECCCCCD, + 0xBECCCCCD, 0x00000000, 0xBECCCCCD, 0xBECCCCCD, 0x3ECCCCCD, 0x00000000, + 0x3ECCCCCD, 0x00000000, 0x3ECCCCCD, 0x00000000, 0x3ECCCCCD, 0x00000000, + 0x00000000, 0x00000000, 0x00040003, 0x00090005, 0x000B000A, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000003, 0x00000001, + 0x00000006, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000003, 0x00000001, 0x00000003, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x3EF5C28F, 0x3ED1EB85, 0x3E6147AE, 0x3F800000, + 0x3E947AE1, 0x3E6147AE, 0x3D4CCCCD, 0x3F800000, 0x00000000, 0x20554002, + 0x3F800000, 0x20444003, 0x000000CF, 0x20044904, 0x00000000, 0x00000200, + 0x00050001, 0x42300001, 0x00000002, 0x20154005, 0x00000020, 0x20244006, + 0x00000000, 0x00000000, 0x10200001, 0x46500007, 0x20030004, 0x46D00004, + 0x00000000, 0x20554008, 0x00070001, 0x61830806, 0x00061020, 0x61808001, + 0x00040000, 0x42380800, 0x00010000, 0x42380800, 0x20040000, 0x46D80800, + 0x00000000, 0x20154007, 0x20020000, 0x46F80000, 0x00000007, 0x20154001, + 0x00000000, 0x00000200, 0x60030001, 0x43900004, 0x60030001, 0x43900001, + 0x00000000, 0x00000400, 0x00013600, 0xC6E20004, 0x40040003, 0x50180104, + 0x40060003, 0x40180803, 0x00000003, 0x20044006, 0x00000000, 0x00000500, + 0x00003609, 0xC7260201, 0x00000000, 0x03000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, +}; + +static unsigned int _a5xx_critical_pkts_mem02[] = { + 0x00000000, 0x03000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x0000000C, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x8ACFE7F3, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +static unsigned int _a5xx_critical_pkts_mem03[] = { + 0x70438003, /* [0x0000] == TYPE7: SET_DRAW_STATE (43) == */ + 0x0008003A, /* [0x0001] */ + 0x00000000, /* [0x0002] */ + 0x00000000, /* [0x0003] */ + 0x70B08003, /* [0x0004] == TYPE7: LOAD_STATE (30) == */ + 0x00620000, /* [0x0005] */ + 0x00000000, /* [0x0006] */ + 0x00000000, /* [0x0007] */ + 0x40E29801, /* [0x0008] == TYPE4 == */ + 0x0000FFFF, /* [0x0009] A5X_VPC_GS_SIV_CNTL_CTX_0 (0xE298)*/ + 0x48E2A001, /* [0x000A] == TYPE4 == */ + 0x000000FF, /* [0x000B] A5X_VPC_PS_PRIMITIVEID_CNTL_CTX_0 (0xE2A0)*/ + 0x40E40185, /* [0x000C] == TYPE4 == */ + 0x00FCFCFC, /* [0x000D] A5X_VFD_CNTL_1_CTX_0 (0xE401)*/ + 0x0000FCFC, /* [0x000E] A5X_VFD_CNTL_2_CTX_0 (0xE402)*/ + 0x0000FCFC, /* [0x000F] A5X_VFD_CNTL_3_CTX_0 (0xE403)*/ + 0x000000FC, /* [0x0010] A5X_VFD_CNTL_4_CTX_0 (0xE404)*/ + 0x00000000, /* [0x0011] A5X_VFD_CNTL_5_CTX_0 (0xE405)*/ + 0x48E38F01, /* [0x0012] == TYPE4 == */ + 0x00000000, /* [0x0013] A5X_PC_HS_PARAM_CTX_0 (0xE38F)*/ + 0x48E58001, /* [0x0014] == TYPE4 == */ + 0x00000010, /* [0x0015] A5X_SP_SP_CNTL_CTX_0 (0xE580)*/ + 0x40E00001, /* [0x0016] == TYPE4 == */ + 0x00000080, /* [0x0017] A5X_GRAS_CL_CNTL_CTX_0 (0xE000)*/ + 0x40E09583, /* [0x0018] == TYPE4 == */ + 0x00000000, /* [0x0019] A5X_GRAS_SU_POLY_OFFSET_SCALE_CTX_0 (0xE095)*/ + 0x00000000, /* [0x001A] A5X_GRAS_SU_POLY_OFFSET_OFFSET_CTX_0 (0xE096)*/ + 0x00000000, /* [0x001B] A5X_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP_CTX_0 + * (0xE097) + */ + 0x40E09001, /* [0x001C] == TYPE4 == */ + 0x00000010, /* [0x001D] A5X_GRAS_SU_CNTL_CTX_0 (0xE090)*/ + 0x40E0AA02, /* [0x001E] == TYPE4 == */ + 0x00000000, /* [0x001F] A5X_GRAS_SC_SCREEN_SCISSOR_TL_0_CTX_0 (0xE0AA)*/ + 0x001F0073, /* [0x0020] A5X_GRAS_SC_SCREEN_SCISSOR_BR_0_CTX_0 (0xE0AB)*/ + 0x48E01086, /* [0x0021] == TYPE4 == */ + 0x42680000, /* [0x0022] A5X_GRAS_CL_VIEWPORT_XOFFSET_0_CTX_0 (0xE010)*/ + 0x42680000, /* [0x0023] A5X_GRAS_CL_VIEWPORT_XSCALE_0_CTX_0 (0xE011)*/ + 0x41800000, /* [0x0024] A5X_GRAS_CL_VIEWPORT_YOFFSET_0_CTX_0 (0xE012)*/ + 0xC1800000, /* [0x0025] A5X_GRAS_CL_VIEWPORT_YSCALE_0_CTX_0 (0xE013)*/ + 0x3EFFFEE0, /* [0x0026] A5X_GRAS_CL_VIEWPORT_ZOFFSET_0_CTX_0 (0xE014)*/ + 0x3EFFFEE0, /* [0x0027] A5X_GRAS_CL_VIEWPORT_ZSCALE_0_CTX_0 (0xE015)*/ + 0x40E0CA02, /* [0x0028] == TYPE4 == */ + 0x00000000, /* [0x0029] A5X_GRAS_SC_VIEWPORT_SCISSOR_TL_0_CTX_0 + * (0xE0CA) + */ + 0x001F0073, /* [0x002A] A5X_GRAS_SC_VIEWPORT_SCISSOR_BR_0_CTX_0 + * (0xE0CB) + */ + 0x40E00601, /* [0x002B] == TYPE4 == */ + 0x0007FDFF, /* [0x002C] A5X_GRAS_CL_GUARDBAND_CLIP_ADJ_CTX_0 (0xE006)*/ + 0x40E70401, /* [0x002D] == TYPE4 == */ + 0x00000000, /* [0x002E] A5X_TPL1_TP_RAS_MSAA_CNTL_CTX_0 (0xE704)*/ + 0x48E70501, /* [0x002F] == TYPE4 == */ + 0x00000004, /* [0x0030] A5X_TPL1_TP_DEST_MSAA_CNTL_CTX_0 (0xE705)*/ + 0x48E14201, /* [0x0031] == TYPE4 == */ + 0x00000000, /* [0x0032] A5X_RB_RAS_MSAA_CNTL_CTX_0 (0xE142)*/ + 0x40E14301, /* [0x0033] == TYPE4 == */ + 0x00000004, /* [0x0034] A5X_RB_DEST_MSAA_CNTL_CTX_0 (0xE143)*/ + 0x40E78683, /* [0x0035] == TYPE4 == */ + 0xFCFCFCFC, /* [0x0036] A5X_HLSQ_CNTL_2_CTX_0 (0xE786)*/ + 0xFCFCFCFC, /* [0x0037] A5X_HLSQ_CNTL_3_CTX_0 (0xE787)*/ + 0xFCFCFCFC, /* [0x0038] A5X_HLSQ_CNTL_4_CTX_0 (0xE788)*/ + 0x48E0A201, /* [0x0039] == TYPE4 == */ + 0x00000000, /* [0x003A] A5X_GRAS_SC_RAS_MSAA_CNTL_CTX_0 (0xE0A2)*/ + 0x40E0A301, /* [0x003B] == TYPE4 == */ + 0x00000004, /* [0x003C] A5X_GRAS_SC_DEST_MSAA_CNTL_CTX_0 (0xE0A3)*/ + 0x48E14101, /* [0x003D] == TYPE4 == */ + 0x0000C089, /* [0x003E] A5X_RB_RENDER_CNTL_CTX_0 (0xE141)*/ + 0x40E0A001, /* [0x003F] == TYPE4 == */ + 0x00000009, /* [0x0040] A5X_GRAS_SC_CNTL_CTX_0 (0xE0A0)*/ + 0x40E28001, /* [0x0041] == TYPE4 == */ + 0x00010004, /* [0x0042] A5X_VPC_CNTL_0_CTX_0 (0xE280)*/ + 0x40E38401, /* [0x0043] == TYPE4 == */ + 0x00000404, /* [0x0044] A5X_PC_PRIMITIVE_CNTL_CTX_0 (0xE384)*/ + 0x40E78501, /* [0x0045] == TYPE4 == */ + 0x0000003F, /* [0x0046] A5X_HLSQ_CNTL_1_CTX_0 (0xE785)*/ + 0x48E5D301, /* [0x0047] == TYPE4 == */ + 0x00000030, /* [0x0048] A5X_SP_PS_MRT_0_CTX_0 (0xE5D3)*/ + 0x48E5CB01, /* [0x0049] == TYPE4 == */ + 0x00000100, /* [0x004A] A5X_SP_PS_OUTPUT_0_CTX_0 (0xE5CB)*/ + 0x40E5CA01, /* [0x004B] == TYPE4 == */ + 0x001F9F81, /* [0x004C] A5X_SP_PS_OUTPUT_CNTL_CTX_0 (0xE5CA)*/ + 0x40E14601, /* [0x004D] == TYPE4 == */ + 0x00000001, /* [0x004E] A5X_RB_PS_OUTPUT_CNTL_CTX_0 (0xE146)*/ + 0x40E38E01, /* [0x004F] == TYPE4 == */ + 0x00000000, /* [0x0050] A5X_PC_GS_PARAM_CTX_0 (0xE38E)*/ + 0x40E28A01, /* [0x0051] == TYPE4 == */ + 0x00000000, /* [0x0052] A5X_VPC_VARYING_REPLACE_MODE_0_CTX_0 (0xE28A)*/ + 0x48E1A901, /* [0x0053] == TYPE4 == */ + 0xFFFF0100, /* [0x0054] A5X_RB_BLEND_CNTL_CTX_0 (0xE1A9)*/ + 0x40E5C901, /* [0x0055] == TYPE4 == */ + 0x00000100, /* [0x0056] A5X_SP_BLEND_CNTL_CTX_0 (0xE5C9)*/ + 0x40E76401, /* [0x0057] == TYPE4 == */ + 0x00000000, /* [0x0058] A5X_TPL1_TP_PS_ROTATION_CNTL_CTX_0 (0xE764)*/ + 0x48E09401, /* [0x0059] == TYPE4 == */ + 0x00000000, /* [0x005A] A5X_GRAS_SU_DEPTH_PLANE_CNTL_CTX_0 (0xE094)*/ + 0x40E1B001, /* [0x005B] == TYPE4 == */ + 0x00000000, /* [0x005C] A5X_RB_DEPTH_PLANE_CNTL_CTX_0 (0xE1B0)*/ + 0x48E1B101, /* [0x005D] == TYPE4 == */ + 0x00000000, /* [0x005E] A5X_RB_DEPTH_CNTL_CTX_0 (0xE1B1)*/ + 0x48E40001, /* [0x005F] == TYPE4 == */ + 0x00000001, /* [0x0060] A5X_VFD_CNTL_0_CTX_0 (0xE400)*/ + 0x48E40A04, /* [0x0061] == TYPE4 == */ + 0x00000000, /* [0x0062] A5X_VFD_VERTEX_BUFFER_BASE_LO_0_CTX_0 (0xE40A)*/ + 0x00000000, /* [0x0063] A5X_VFD_VERTEX_BUFFER_BASE_HI_0_CTX_0 (0xE40B)*/ + 0x00000078, /* [0x0064] A5X_VFD_VERTEX_BUFFER_SIZE_0_CTX_0 (0xE40C)*/ + 0x00000008, /* [0x0065] A5X_VFD_VERTEX_BUFFER_STRIDE_0_CTX_0 (0xE40D)*/ + 0x40E48A02, /* [0x0066] == TYPE4 == */ + 0xC6700000, /* [0x0067] A5X_VFD_FETCH_INSTR_0_CTX_0 (0xE48A)*/ + 0x00000001, /* [0x0068] A5X_VFD_FETCH_INSTR_STEP_RATE_0_CTX_0 (0xE48B)*/ + 0x48E4CA01, /* [0x0069] == TYPE4 == */ + 0x0000000F, /* [0x006A] A5X_VFD_DEST_CNTL_0_CTX_0 (0xE4CA)*/ + 0x48E10001, /* [0x006B] == TYPE4 == */ + 0x00000008, /* [0x006C] A5X_GRAS_LRZ_CNTL_CTX_0 (0xE100)*/ + 0x48E0A101, /* [0x006D] == TYPE4 == */ + 0x00000004, /* [0x006E] A5X_GRAS_SC_BIN_CNTL_CTX_0 (0xE0A1)*/ + 0x40E10185, /* [0x006F] == TYPE4 == */ + 0x00000000, /* [0x0070] A5X_GRAS_LRZ_BUFFER_BASE_LO_CTX_0 (0xE101)*/ + 0x00000000, /* [0x0071] A5X_GRAS_LRZ_BUFFER_BASE_HI_CTX_0 (0xE102)*/ + 0x00000001, /* [0x0072] A5X_GRAS_LRZ_BUFFER_PITCH_CTX_0 (0xE103)*/ + 0x00000000, /* [0x0073] A5X_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_LO_CTX_0 + * (0xE104) + */ + 0x00000000, /* [0x0074] A5X_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_HI_CTX_0 + * (0xE105) + */ + 0x70388003, /* [0x0075] == TYPE7: DRAW_INDX_OFFSET (38) == */ + 0x00200884, /* [0x0076] */ + 0x00000001, /* [0x0077] */ + 0x00000003, /* [0x0078] */ + 0x70380007, /* [0x0079] == TYPE7: DRAW_INDX_OFFSET (38) == */ + 0x00200404, /* [0x007A] */ + 0x00000001, /* [0x007B] */ + 0x00000003, /* [0x007C] */ + 0x00000000, /* [0x007D] */ + 0x00000000, /* [0x007E] */ + 0x00000000, /* [0x007F] */ + 0x00000006, /* [0x0080] */ + 0x70460004, /* [0x0081] == TYPE7: EVENT_WRITE (46) == */ + 0x00000004, /* [0x0082] */ + 0x00000000, /* [0x0083] */ + 0x00000000, /* [0x0084] */ + 0x00000001, /* [0x0085] */ + 0x70268000, /* [0x0086] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x70A88003, /* [0x0087] == TYPE7: DRAW_INDIRECT (28) == */ + 0x00200884, /* [0x0088] */ + 0x00000000, /* [0x0089] */ + 0x00000000, /* [0x008A] */ + 0x70460004, /* [0x008B] == TYPE7: EVENT_WRITE (46) == */ + 0x00000004, /* [0x008C] */ + 0x00000000, /* [0x008D] */ + 0x00000000, /* [0x008E] */ + 0x00000001, /* [0x008F] */ + 0x70268000, /* [0x0090] == TYPE7: WAIT_FOR_IDLE (26) == */ + 0x70298006, /* [0x0091] == TYPE7: DRAW_INDX_INDIRECT (29) == */ + 0x00200404, /* [0x0092] */ + 0x00000000, /* [0x0093] */ + 0x00000000, /* [0x0094] */ + 0x00000006, /* [0x0095] */ + 0x00000000, /* [0x0096] */ + 0x00000000, /* [0x0097] */ + 0x40E40801, /* [0x0098] == TYPE4 == */ + 0x0000000D, /* [0x0099] A5X_VFD_INDEX_OFFSET_CTX_0 (0xE408)*/ + 0x48E40901, /* [0x009A] == TYPE4 == */ + 0x00000000, /* [0x009B] A5X_VFD_INSTANCE_START_OFFSET_CTX_0 (0xE409)*/ + 0x70388003, /* [0x009C] == TYPE7: DRAW_INDX_OFFSET (38) == */ + 0x00200884, /* [0x009D] */ + 0x00000001, /* [0x009E] */ + 0x00000003, /* [0x009F] */ + 0x00000000, /* [0x00A0] */ + 0x00000000, /* [0x00A1] */ + 0x00000000, /* [0x00A2] */ + 0x00000000, /* [0x00A3] */ + 0x00000000, /* [0x00A4] */ + 0x00000000, /* [0x00A5] */ + 0x00000000, /* [0x00A6] */ + 0x00000000, /* [0x00A7] */ + 0x48E78401, /* [0x00A8] */ + 0x00000881, /* [0x00A9] */ + 0x40E5C001, /* [0x00AA] */ + 0x0004001E, /* [0x00AB] */ + 0x70438003, /* [0x00AC] */ + 0x0000003A, /* [0x00AD] */ + 0x00000000, /* [0x00AE] */ + 0x00000000, /* [0x00AF] */ + 0x70B00023, /* [0x00B0] */ + 0x00600000, /* [0x00B1] */ + 0x00000000, /* [0x00B2] */ + 0x00000000, /* [0x00B3] */ + 0x00000000, /* [0x00B4] */ + 0x03000000, /* [0x00B5] */ + 0x00000000, /* [0x00B6] */ + 0x00000000, /* [0x00B7] */ + 0x00000000, /* [0x00B8] */ + 0x00000000, /* [0x00B9] */ + 0x00000000, /* [0x00BA] */ + 0x00000000, /* [0x00BB] */ + 0x00000000, /* [0x00BC] */ + 0x00000000, /* [0x00BD] */ + 0x00000000, /* [0x00BE] */ + 0x00000000, /* [0x00BF] */ + 0x00000000, /* [0x00C0] */ + 0x00000000, /* [0x00C1] */ + 0x00000000, /* [0x00C2] */ + 0x00000000, /* [0x00C3] */ + 0x00000000, /* [0x00C4] */ + 0x00000000, /* [0x00C5] */ + 0x00000000, /* [0x00C6] */ + 0x00000000, /* [0x00C7] */ + 0x00000000, /* [0x00C8] */ + 0x00000000, /* [0x00C9] */ + 0x00000000, /* [0x00CA] */ + 0x00000000, /* [0x00CB] */ + 0x00000000, /* [0x00CC] */ + 0x00000000, /* [0x00CD] */ + 0x00000000, /* [0x00CE] */ + 0x00000000, /* [0x00CF] */ + 0x00000000, /* [0x00D0] */ + 0x00000000, /* [0x00D1] */ + 0x00000000, /* [0x00D2] */ + 0x00000000, /* [0x00D3] */ + 0x40E09301, /* [0x00D4] */ + 0x00000000, /* [0x00D5] */ + 0x40E38D01, /* [0x00D6] */ + 0x00000000, /* [0x00D7] */ + 0x40E29801, /* [0x00D8] */ + 0x0000FFFF, /* [0x00D9] */ + 0x48E28201, /* [0x00DA] */ + 0xEAEAEAEA, /* [0x00DB] */ + 0x40E29404, /* [0x00DC] */ + 0xFFFFFFFF, /* [0x00DD] */ + 0xFFFFFFFF, /* [0x00DE] */ + 0xFFFFFFFF, /* [0x00DF] */ + 0xFFFFFFFF, /* [0x00E0] */ + 0x40E5DB01, /* [0x00E1] */ + 0x00000000, /* [0x00E2] */ + 0x48E14701, /* [0x00E3] */ + 0x0000000F, /* [0x00E4] */ + 0x70B00023, /* [0x00E5] */ + 0x00700000, /* [0x00E6] */ + 0x00000000, /* [0x00E7] */ + 0x00000000, /* [0x00E8] */ + 0x00003C00, /* [0x00E9] */ + 0x20400000, /* [0x00EA] */ + 0x00000000, /* [0x00EB] */ + 0x20400001, /* [0x00EC] */ + 0x00000000, /* [0x00ED] */ + 0x20400002, /* [0x00EE] */ + 0x00003C00, /* [0x00EF] */ + 0x20400003, /* [0x00F0] */ + 0x00000000, /* [0x00F1] */ + 0x03000000, /* [0x00F2] */ + 0x00000000, /* [0x00F3] */ + 0x00000000, /* [0x00F4] */ + 0x00000000, /* [0x00F5] */ + 0x00000000, /* [0x00F6] */ + 0x00000000, /* [0x00F7] */ + 0x00000000, /* [0x00F8] */ + 0x00000000, /* [0x00F9] */ + 0x00000000, /* [0x00FA] */ + 0x00000000, /* [0x00FB] */ + 0x00000000, /* [0x00FC] */ + 0x00000000, /* [0x00FD] */ + 0x00000000, /* [0x00FE] */ + 0x00000000, /* [0x00FF] */ + 0x00000000, /* [0x0100] */ + 0x00000000, /* [0x0101] */ + 0x00000000, /* [0x0102] */ + 0x00000000, /* [0x0103] */ + 0x00000000, /* [0x0104] */ + 0x00000000, /* [0x0105] */ + 0x00000000, /* [0x0106] */ + 0x00000000, /* [0x0107] */ + 0x00000000, /* [0x0108] */ + 0x48E2A001, /* [0x0109] */ + 0x000000FF, /* [0x010A] */ + 0x40E40185, /* [0x010B] */ + 0x00FCFCFC, /* [0x010C] */ + 0x0000FCFC, /* [0x010D] */ + 0x0000FCFC, /* [0x010E] */ + 0x000000FC, /* [0x010F] */ + 0x00000000, /* [0x0110] */ + 0x48E38F01, /* [0x0111] */ + 0x00000000, /* [0x0112] */ + 0x48E58001, /* [0x0113] */ + 0x00000010, /* [0x0114] */ + 0x40E1A801, /* [0x0115] */ + 0x00000E00, /* [0x0116] */ + 0x48E15001, /* [0x0117] */ + 0x000007E0, /* [0x0118] */ + 0x40E15101, /* [0x0119] */ + 0x00000000, /* [0x011A] */ + 0x40E00001, /* [0x011B] */ + 0x00000080, /* [0x011C] */ + 0x40E09583, /* [0x011D] */ + 0x00000000, /* [0x011E] */ + 0x00000000, /* [0x011F] */ + 0x00000000, /* [0x0120] */ + 0x40E09001, /* [0x0121] */ + 0x00000010, /* [0x0122] */ + 0x40E0AA02, /* [0x0123] */ + 0x00000000, /* [0x0124] */ + 0x001F0073, /* [0x0125] */ + 0x48E01086, /* [0x0126] */ + 0x42680000, /* [0x0127] */ + 0x42680000, /* [0x0128] */ + 0x41800000, /* [0x0129] */ + 0xC1800000, /* [0x012A] */ + 0x3EFFFEE0, /* [0x012B] */ + 0x3EFFFEE0, /* [0x012C] */ + 0x40E0CA02, /* [0x012D] */ + 0x00000000, /* [0x012E] */ + 0x001F0073, /* [0x012F] */ + 0x40E00601, /* [0x0130] */ + 0x0007FDFF, /* [0x0131] */ + 0x40E70401, /* [0x0132] */ + 0x00000000, /* [0x0133] */ + 0x48E70501, /* [0x0134] */ + 0x00000004, /* [0x0135] */ + 0x48E14201, /* [0x0136] */ + 0x00000000, /* [0x0137] */ + 0x40E14301, /* [0x0138] */ + 0x00000004, /* [0x0139] */ + 0x40E78683, /* [0x013A] */ + 0xFCFCFCFC, /* [0x013B] */ + 0xFCFCFCFC, /* [0x013C] */ + 0xFCFCFCFC, /* [0x013D] */ + 0x48E0A201, /* [0x013E] */ + 0x00000000, /* [0x013F] */ + 0x40E0A301, /* [0x0140] */ + 0x00000004, /* [0x0141] */ + 0x48E1B285, /* [0x0142] */ + 0x00000001, /* [0x0143] */ + 0x00004000, /* [0x0144] */ + 0x00000000, /* [0x0145] */ + 0x00000004, /* [0x0146] */ + 0x000000C0, /* [0x0147] */ + 0x48E09801, /* [0x0148] */ + 0x00000001, /* [0x0149] */ + 0x48E00401, /* [0x014A] */ + 0x00000000, /* [0x014B] */ + 0x480CDD02, /* [0x014C] */ + 0x00200074, /* [0x014D] */ + 0x00000000, /* [0x014E] */ + 0x40E15285, /* [0x014F] */ + 0x00000A30, /* [0x0150] */ + 0x00000008, /* [0x0151] */ + 0x00000100, /* [0x0152] */ + 0x00000000, /* [0x0153] */ + 0x00000000, /* [0x0154] */ + 0x48E14101, /* [0x0155] */ + 0x0000C008, /* [0x0156] */ + 0x40E0A001, /* [0x0157] */ + 0x00000008, /* [0x0158] */ + 0x40E28001, /* [0x0159] */ + 0x00010004, /* [0x015A] */ + 0x40E38401, /* [0x015B] */ + 0x00000404, /* [0x015C] */ + 0x40E78501, /* [0x015D] */ + 0x0000003F, /* [0x015E] */ + 0x48E5D301, /* [0x015F] */ + 0x00000030, /* [0x0160] */ + 0x48E5CB01, /* [0x0161] */ + 0x00000100, /* [0x0162] */ + 0x40E5CA01, /* [0x0163] */ + 0x001F9F81, /* [0x0164] */ + 0x40E14601, /* [0x0165] */ + 0x00000001, /* [0x0166] */ + 0x40E38E01, /* [0x0167] */ + 0x00000000, /* [0x0168] */ + 0x40E28A01, /* [0x0169] */ + 0x00000000, /* [0x016A] */ + 0x48E1A901, /* [0x016B] */ + 0xFFFF0100, /* [0x016C] */ + 0x40E5C901, /* [0x016D] */ + 0x00000100, /* [0x016E] */ + 0x40E76401, /* [0x016F] */ + 0x00000000, /* [0x0170] */ + 0x48E09401, /* [0x0171] */ + 0x00000000, /* [0x0172] */ + 0x40E1B001, /* [0x0173] */ + 0x00000000, /* [0x0174] */ + 0x48E1B101, /* [0x0175] */ + 0x00000006, /* [0x0176] */ + 0x48E40001, /* [0x0177] */ + 0x00000001, /* [0x0178] */ + 0x48E40A04, /* [0x0179] */ + 0x00000000, /* [0x017A] */ + 0x00000000, /* [0x017B] */ + 0x00000078, /* [0x017C] */ + 0x00000008, /* [0x017D] */ + 0x40E48A02, /* [0x017E] */ + 0xC6700000, /* [0x017F] */ + 0x00000001, /* [0x0180] */ + 0x48E4CA01, /* [0x0181] */ + 0x0000000F, /* [0x0182] */ + 0x48E10001, /* [0x0183] */ + 0x00000008, /* [0x0184] */ + 0x48E0A101, /* [0x0185] */ + 0x00000000, /* [0x0186] */ + 0x40E10185, /* [0x0187] */ + 0x00000000, /* [0x0188] */ + 0x00000000, /* [0x0189] */ + 0x00000001, /* [0x018A] */ + 0x00000000, /* [0x018B] */ + 0x00000000, /* [0x018C] */ + 0x70230001, /* [0x018D] */ + 0x00000000, /* [0x018E] */ + 0x70388003, /* [0x018F] */ + 0x00200984, /* [0x0190] */ + 0x00000001, /* [0x0191] */ + 0x00000003, /* [0x0192] */ + 0x70380007, /* [0x0193] */ + 0x00200504, /* [0x0194] */ + 0x00000001, /* [0x0195] */ + 0x00000003, /* [0x0196] */ + 0x00000000, /* [0x0197] */ + 0x00000000, /* [0x0198] */ + 0x00000000, /* [0x0199] */ + 0x00000006, /* [0x019A] */ + 0x70460004, /* [0x019B] */ + 0x00000004, /* [0x019C] */ + 0x00000000, /* [0x019D] */ + 0x00000000, /* [0x019E] */ + 0x00000000, /* [0x019F] */ + 0x70268000, /* [0x01A0] */ + 0x70A88003, /* [0x01A1] */ + 0x00200984, /* [0x01A2] */ + 0x00000000, /* [0x01A3] */ + 0x00000000, /* [0x01A4] */ + 0x70460004, /* [0x01A5] */ + 0x00000004, /* [0x01A6] */ + 0x00000000, /* [0x01A7] */ + 0x00000000, /* [0x01A8] */ + 0x00000001, /* [0x01A9] */ + 0x70268000, /* [0x01AA] */ + 0x70298006, /* [0x01AB] */ + 0x00200504, /* [0x01AC] */ + 0x00000000, /* [0x01AD] */ + 0x00000000, /* [0x01AE] */ + 0x00000006, /* [0x01AF] */ + 0x00000000, /* [0x01B0] */ + 0x00000000, /* [0x01B1] */ + 0x40E40801, /* [0x01B2] */ + 0x0000000D, /* [0x01B3] */ + 0x48E40901, /* [0x01B4] */ + 0x00000000, /* [0x01B5] */ + 0x70388003, /* [0x01B6] */ + 0x00200984, /* [0x01B7] */ + 0x00000001, /* [0x01B8] */ + 0x00000003, /* [0x01B9] */ + 0x00000000, /* [0x01BA] */ + 0x00000000, /* [0x01BB] */ + 0x00000000, /* [0x01BC] */ + 0x00000000, /* [0x01BD] */ + 0x00000000, /* [0x01BE] */ + 0x00000000, /* [0x01BF] */ + 0x70EA0001, /* [0x01C0] */ + 0x00000000, /* [0x01C1] */ + 0x40E78A01, /* [0x01C2] */ + 0x000FFFFF, /* [0x01C3] */ + 0x40E09001, /* [0x01C4] */ + 0x00000000, /* [0x01C5] */ + 0x40E00501, /* [0x01C6] */ + 0x00000000, /* [0x01C7] */ + 0x40E00001, /* [0x01C8] */ + 0x00000181, /* [0x01C9] */ + 0x48E10001, /* [0x01CA] */ + 0x00000000, /* [0x01CB] */ + 0x40E21385, /* [0x01CC] */ + 0x00000004, /* [0x01CD] */ + 0x00000000, /* [0x01CE] */ + 0x00000000, /* [0x01CF] */ + 0x00000001, /* [0x01D0] */ + 0x00000001, /* [0x01D1] */ + 0x40E21C01, /* [0x01D2] */ + 0x00000000, /* [0x01D3] */ + 0x40E21001, /* [0x01D4] */ + 0x00000000, /* [0x01D5] */ + 0x70460004, /* [0x01D6] */ + 0x0000001E, /* [0x01D7] */ + 0x00000000, /* [0x01D8] */ + 0x00000000, /* [0x01D9] */ + 0x00000001, /* [0x01DA] */ + 0x00000000, /* [0x01DB] */ + 0x00000000, /* [0x01DC] */ + 0x00000000, /* [0x01DD] */ + 0x00000000, /* [0x01DE] */ + 0x00000000, /* [0x01DF] */ + 0x40E78A01, /* [0x01E0] */ + 0x020FFFFF, /* [0x01E1] */ + 0x48E78B85, /* [0x01E2] */ + 0x00000001, /* [0x01E3] */ + 0x00003F05, /* [0x01E4] */ + 0x00003F04, /* [0x01E5] */ + 0x00003F04, /* [0x01E6] */ + 0x00003F04, /* [0x01E7] */ + 0x48E79001, /* [0x01E8] */ + 0x00000000, /* [0x01E9] */ + 0x40E79101, /* [0x01EA] */ + 0x00000002, /* [0x01EB] */ + 0x40E79201, /* [0x01EC] */ + 0x00000002, /* [0x01ED] */ + 0x40E58485, /* [0x01EE] */ + 0x00000001, /* [0x01EF] */ + 0x00003F05, /* [0x01F0] */ + 0x00003F04, /* [0x01F1] */ + 0x00003F04, /* [0x01F2] */ + 0x00003F04, /* [0x01F3] */ + 0x48E58901, /* [0x01F4] */ + 0x00000000, /* [0x01F5] */ + 0x48E7C302, /* [0x01F6] */ + 0x00000002, /* [0x01F7] */ + 0x00000001, /* [0x01F8] */ + 0x48E7D702, /* [0x01F9] */ + 0x00000002, /* [0x01FA] */ + 0x00000001, /* [0x01FB] */ + 0x40E7C802, /* [0x01FC] */ + 0x00000000, /* [0x01FD] */ + 0x00000000, /* [0x01FE] */ + 0x40E7CD02, /* [0x01FF] */ + 0x00000000, /* [0x0200] */ + 0x00000000, /* [0x0201] */ + 0x48E7D202, /* [0x0202] */ + 0x00000000, /* [0x0203] */ + 0x00000000, /* [0x0204] */ + 0x40E7DC02, /* [0x0205] */ + 0x00000000, /* [0x0206] */ + 0x00000000, /* [0x0207] */ + 0x48E38901, /* [0x0208] */ + 0x00000000, /* [0x0209] */ + 0x48E29A01, /* [0x020A] */ + 0x00FFFF00, /* [0x020B] */ + 0x48E00101, /* [0x020C] */ + 0x00000000, /* [0x020D] */ + 0x40E29D01, /* [0x020E] */ + 0x0000FF00, /* [0x020F] */ + 0x40E59001, /* [0x0210] */ + 0x00000406, /* [0x0211] */ + 0x48E59201, /* [0x0212] */ + 0x00000001, /* [0x0213] */ + 0x40E59301, /* [0x0214] */ + 0x00000F00, /* [0x0215] */ + 0x40E5A301, /* [0x0216] */ + 0x00000000, /* [0x0217] */ + 0x48E38501, /* [0x0218] */ + 0x00000000, /* [0x0219] */ + 0x00000000, /* [0x021A] */ + 0x00000000, /* [0x021B] */ + 0x00000000, /* [0x021C] */ + 0x00000000, /* [0x021D] */ + 0x00000000, /* [0x021E] */ + 0x00000000, /* [0x021F] */ + 0x48210001, /* [0x0220] */ + 0x86000000, /* [0x0221] */ + 0x40218001, /* [0x0222] */ + 0x86000000, /* [0x0223] */ + 0x40211089, /* [0x0224] */ + 0x00001331, /* [0x0225] */ + 0x00000000, /* [0x0226] */ + 0x00000000, /* [0x0227] */ + 0x00020001, /* [0x0228] */ + 0x00000000, /* [0x0229] */ + 0x00000000, /* [0x022A] */ + 0x00000000, /* [0x022B] */ + 0x00000000, /* [0x022C] */ + 0x00000000, /* [0x022D] */ + 0x48218201, /* [0x022E] */ + 0x00001331, /* [0x022F] */ + 0x40214383, /* [0x0230] */ + 0x00000000, /* [0x0231] */ + 0x00000000, /* [0x0232] */ + 0x00000001, /* [0x0233] */ + 0x40210789, /* [0x0234] */ + 0x00000021, /* [0x0235] */ + 0x00000000, /* [0x0236] */ + 0x00000000, /* [0x0237] */ + 0x00020001, /* [0x0238] */ + 0x00000000, /* [0x0239] */ + 0x00000000, /* [0x023A] */ + 0x00000000, /* [0x023B] */ + 0x00000000, /* [0x023C] */ + 0x00000000, /* [0x023D] */ + 0x48218101, /* [0x023E] */ + 0x00000021, /* [0x023F] */ + 0x48218401, /* [0x0240] */ + 0x00000001, /* [0x0241] */ + 0x702C8005, /* [0x0242] */ + 0x00000002, /* [0x0243] */ + 0x00000000, /* [0x0244] */ + 0x00010001, /* [0x0245] */ + 0x00000000, /* [0x0246] */ + 0x00010001, /* [0x0247] */ + 0x70B00023, /* [0x0248] */ + 0x00600000, /* [0x0249] */ + 0x00000000, /* [0x024A] */ + 0x00000000, /* [0x024B] */ + 0x00000000, /* [0x024C] */ + 0x03000000, /* [0x024D] */ + 0x00000000, /* [0x024E] */ + 0x00000000, /* [0x024F] */ + 0x00000000, /* [0x0250] */ + 0x00000000, /* [0x0251] */ + 0x00000000, /* [0x0252] */ + 0x00000000, /* [0x0253] */ + 0x00000000, /* [0x0254] */ + 0x00000000, /* [0x0255] */ + 0x00000000, /* [0x0256] */ + 0x00000000, /* [0x0257] */ + 0x00000000, /* [0x0258] */ + 0x00000000, /* [0x0259] */ + 0x00000000, /* [0x025A] */ + 0x00000000, /* [0x025B] */ + 0x00000000, /* [0x025C] */ + 0x00000000, /* [0x025D] */ + 0x00000000, /* [0x025E] */ + 0x00000000, /* [0x025F] */ + 0x00000000, /* [0x0260] */ + 0x00000000, /* [0x0261] */ + 0x00000000, /* [0x0262] */ + 0x00000000, /* [0x0263] */ + 0x00000000, /* [0x0264] */ + 0x00000000, /* [0x0265] */ + 0x00000000, /* [0x0266] */ + 0x00000000, /* [0x0267] */ + 0x00000000, /* [0x0268] */ + 0x00000000, /* [0x0269] */ + 0x00000000, /* [0x026A] */ + 0x00000000, /* [0x026B] */ + 0x40E09301, /* [0x026C] */ + 0x00000000, /* [0x026D] */ + 0x40E38D01, /* [0x026E] */ + 0x00000000, /* [0x026F] */ + 0x40E29801, /* [0x0270] */ + 0x0000FFFF, /* [0x0271] */ + 0x48E28201, /* [0x0272] */ + 0xEAEAEAEA, /* [0x0273] */ + 0x40E29404, /* [0x0274] */ + 0xFFFFFFFF, /* [0x0275] */ + 0xFFFFFFFF, /* [0x0276] */ + 0xFFFFFFFF, /* [0x0277] */ + 0xFFFFFFFF, /* [0x0278] */ + 0x40E5DB01, /* [0x0279] */ + 0x00000000, /* [0x027A] */ + 0x48E14701, /* [0x027B] */ + 0x0000000F, /* [0x027C] */ + 0x70B00023, /* [0x027D] */ + 0x00700000, /* [0x027E] */ + 0x00000000, /* [0x027F] */ + 0x00000000, /* [0x0280] */ + 0x00003C00, /* [0x0281] */ + 0x20400000, /* [0x0282] */ + 0x00000000, /* [0x0283] */ + 0x20400001, /* [0x0284] */ + 0x00000000, /* [0x0285] */ + 0x20400002, /* [0x0286] */ + 0x00003C00, /* [0x0287] */ + 0x20400003, /* [0x0288] */ + 0x00000000, /* [0x0289] */ + 0x03000000, /* [0x028A] */ + 0x00000000, /* [0x028B] */ + 0x00000000, /* [0x028C] */ + 0x00000000, /* [0x028D] */ + 0x00000000, /* [0x028E] */ + 0x00000000, /* [0x028F] */ + 0x00000000, /* [0x0290] */ + 0x00000000, /* [0x0291] */ + 0x00000000, /* [0x0292] */ + 0x00000000, /* [0x0293] */ + 0x00000000, /* [0x0294] */ + 0x00000000, /* [0x0295] */ + 0x00000000, /* [0x0296] */ + 0x00000000, /* [0x0297] */ + 0x00000000, /* [0x0298] */ + 0x00000000, /* [0x0299] */ + 0x00000000, /* [0x029A] */ + 0x00000000, /* [0x029B] */ + 0x00000000, /* [0x029C] */ + 0x00000000, /* [0x029D] */ + 0x00000000, /* [0x029E] */ + 0x00000000, /* [0x029F] */ + 0x00000000, /* [0x02A0] */ +}; + +/* Fixups for the IBs in _a5xx_critical_pkts_mem03 */ +static const struct adreno_critical_fixup critical_pkt_mem03_fixups[] = { + { 2, 3, 3, 0x0780 }, + { 6, 7, 2, 0x0000 }, + { 98, 99, 1, 0x0000 }, + { 112, 113, 1, 0x0480 }, + { 115, 116, 1, 0x0400 }, + { 126, 127, 1, 0x0080 }, + { 131, 132, 2, 0x0108 }, + { 137, 138, 1, 0x00A0 }, + { 141, 142, 2, 0x0108 }, + { 147, 148, 1, 0x0080 }, + { 150, 151, 1, 0x00C0 }, + { 174, 175, 3, 0x0780 }, + { 378, 379, 1, 0x0000 }, + { 392, 393, 1, 0x0480 }, + { 395, 396, 1, 0x0400 }, + { 408, 409, 1, 0x0080 }, + { 413, 414, 2, 0x0108 }, + { 419, 420, 1, 0x00A0 }, + { 423, 424, 2, 0x0108 }, + { 429, 430, 1, 0x0080 }, + { 432, 433, 1, 0x00C0 }, + { 462, 463, 0, 0x0700 }, + { 472, 473, 2, 0x0110 }, + { 550, 551, 1, 0x0500 }, + { 561, 562, 1, 0x0600 }, + { 566, 567, 1, 0x0700 }, +}; diff --git a/qcom/opensource/graphics-kernel/adreno_a5xx_perfcounter.c b/qcom/opensource/graphics-kernel/adreno_a5xx_perfcounter.c new file mode 100644 index 0000000000..8886ee24ba --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a5xx_perfcounter.c @@ -0,0 +1,695 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include "adreno.h" +#include "adreno_a5xx.h" +#include "adreno_perfcounter.h" +#include "adreno_pm4types.h" +#include "kgsl_device.h" + +#define VBIF2_PERF_CNT_SEL_MASK 0x7F +/* offset of clear register from select register */ +#define VBIF2_PERF_CLR_REG_SEL_OFF 8 +/* offset of enable register from select register */ +#define VBIF2_PERF_EN_REG_SEL_OFF 16 +/* offset of clear register from the enable register */ +#define VBIF2_PERF_PWR_CLR_REG_EN_OFF 8 + +static void a5xx_counter_load(struct adreno_device *adreno_dev, + struct adreno_perfcount_register *reg) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int index = reg->load_bit / 32; + u32 enable = BIT(reg->load_bit & 31); + + kgsl_regwrite(device, A5XX_RBBM_PERFCTR_LOAD_VALUE_LO, + lower_32_bits(reg->value)); + + kgsl_regwrite(device, A5XX_RBBM_PERFCTR_LOAD_VALUE_HI, + upper_32_bits(reg->value)); + + kgsl_regwrite(device, A5XX_RBBM_PERFCTR_LOAD_CMD0 + index, enable); +} + +static u64 a5xx_counter_read_norestore(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + u32 hi, lo; + + kgsl_regread(device, reg->offset, &lo); + kgsl_regread(device, reg->offset_hi, &hi); + + return ((((u64) hi) << 32) | lo) + reg->value; +} + +static int a5xx_counter_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + + kgsl_regwrite(device, reg->select, countable); + reg->value = 0; + + return 0; +} + +static int a5xx_counter_inline_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + struct adreno_ringbuffer *rb = &adreno_dev->ringbuffers[0]; + u32 cmds[3]; + int ret; + + if (!(device->state == KGSL_STATE_ACTIVE)) + return a5xx_counter_enable(adreno_dev, group, counter, + countable); + + cmds[0] = cp_type7_packet(CP_WAIT_FOR_IDLE, 0); + cmds[1] = cp_type4_packet(reg->select, 1); + cmds[2] = countable; + + /* submit to highest priority RB always */ + ret = a5xx_ringbuffer_addcmds(adreno_dev, rb, NULL, + F_NOTPROTECTED, cmds, 3, 0, NULL); + + if (ret) + return ret; + + /* + * schedule dispatcher to make sure rb[0] is run, because + * if the current RB is not rb[0] and gpu is idle then + * rb[0] will not get scheduled to run + */ + if (adreno_dev->cur_rb != rb) + adreno_dispatcher_schedule(device); + + /* wait for the above commands submitted to complete */ + ret = adreno_ringbuffer_waittimestamp(rb, rb->timestamp, + ADRENO_IDLE_TIMEOUT); + + if (ret) { + /* + * If we were woken up because of cancelling rb events + * either due to soft reset or adreno_stop, ignore the + * error and return 0 here. The perfcounter is already + * set up in software and it will be programmed in + * hardware when we wake up or come up after soft reset, + * by adreno_perfcounter_restore. + */ + if (ret == -EAGAIN) + ret = 0; + else + dev_err(device->dev, + "Perfcounter %s/%u/%u start via commands failed %d\n", + group->name, counter, countable, ret); + } + + if (!ret) + reg->value = 0; + + return ret; +} + +static int a5xx_counter_rbbm_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + if (adreno_is_a540(adreno_dev) && countable == A5XX_RBBM_ALWAYS_COUNT) + return -EINVAL; + + return a5xx_counter_inline_enable(adreno_dev, group, counter, + countable); +} + +static u64 a5xx_counter_read(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + u32 hi, lo; + + kgsl_regread(device, reg->offset, &lo); + kgsl_regread(device, reg->offset_hi, &hi); + + return (((u64) hi) << 32) | lo; +} + +static int a5xx_counter_vbif_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + + if (countable > VBIF2_PERF_CNT_SEL_MASK) + return -EINVAL; + + /* + * Write 1, followed by 0 to CLR register for + * clearing the counter + */ + kgsl_regwrite(device, + reg->select - VBIF2_PERF_CLR_REG_SEL_OFF, 1); + kgsl_regwrite(device, + reg->select - VBIF2_PERF_CLR_REG_SEL_OFF, 0); + kgsl_regwrite(device, + reg->select, countable & VBIF2_PERF_CNT_SEL_MASK); + /* enable reg is 8 DWORDS before select reg */ + kgsl_regwrite(device, + reg->select - VBIF2_PERF_EN_REG_SEL_OFF, 1); + + kgsl_regwrite(device, reg->select, countable); + + reg->value = 0; + return 0; +} + +static int a5xx_counter_vbif_pwr_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + + /* + * Write 1, followed by 0 to CLR register for + * clearing the counter + */ + kgsl_regwrite(device, reg->select + + VBIF2_PERF_PWR_CLR_REG_EN_OFF, 1); + kgsl_regwrite(device, reg->select + + VBIF2_PERF_PWR_CLR_REG_EN_OFF, 0); + kgsl_regwrite(device, reg->select, 1); + + reg->value = 0; + + return 0; +} + +static int a5xx_counter_alwayson_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + return 0; +} + +static u64 a5xx_counter_alwayson_read(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter) +{ + struct adreno_perfcount_register *reg = &group->regs[counter]; + + return a5xx_read_alwayson(adreno_dev) + reg->value; +} + +static int a5xx_counter_pwr_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + + kgsl_regwrite(device, reg->select, countable); + kgsl_regwrite(device, A5XX_GPMU_POWER_COUNTER_ENABLE, 1); + + reg->value = 0; + return 0; +} + +static int a5xx_counter_pwr_gpmu_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + unsigned int shift = (counter << 3) % (sizeof(unsigned int) * 8); + + if (adreno_is_a530(adreno_dev)) { + if (countable > 43) + return -EINVAL; + } else if (adreno_is_a540(adreno_dev)) { + if (countable > 47) + return -EINVAL; + } + + kgsl_regrmw(device, reg->select, 0xff << shift, countable << shift); + kgsl_regwrite(device, A5XX_GPMU_POWER_COUNTER_ENABLE, 1); + + reg->value = 0; + return 0; +} + +static int a5xx_counter_pwr_alwayson_enable(struct adreno_device *adreno_dev, + const struct adreno_perfcount_group *group, + unsigned int counter, unsigned int countable) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_perfcount_register *reg = &group->regs[counter]; + + kgsl_regwrite(device, A5XX_GPMU_ALWAYS_ON_COUNTER_RESET, 1); + + reg->value = 0; + return 0; +} + +static struct adreno_perfcount_register a5xx_perfcounters_cp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_0_LO, + A5XX_RBBM_PERFCTR_CP_0_HI, 0, A5XX_CP_PERFCTR_CP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_1_LO, + A5XX_RBBM_PERFCTR_CP_1_HI, 1, A5XX_CP_PERFCTR_CP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_2_LO, + A5XX_RBBM_PERFCTR_CP_2_HI, 2, A5XX_CP_PERFCTR_CP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_3_LO, + A5XX_RBBM_PERFCTR_CP_3_HI, 3, A5XX_CP_PERFCTR_CP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_4_LO, + A5XX_RBBM_PERFCTR_CP_4_HI, 4, A5XX_CP_PERFCTR_CP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_5_LO, + A5XX_RBBM_PERFCTR_CP_5_HI, 5, A5XX_CP_PERFCTR_CP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_6_LO, + A5XX_RBBM_PERFCTR_CP_6_HI, 6, A5XX_CP_PERFCTR_CP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CP_7_LO, + A5XX_RBBM_PERFCTR_CP_7_HI, 7, A5XX_CP_PERFCTR_CP_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_rbbm[] = { + /* + * A5XX_RBBM_PERFCTR_RBBM_0 is used for frequency scaling and omitted + * from the poool of available counters + */ + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RBBM_1_LO, + A5XX_RBBM_PERFCTR_RBBM_1_HI, 9, A5XX_RBBM_PERFCTR_RBBM_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RBBM_2_LO, + A5XX_RBBM_PERFCTR_RBBM_2_HI, 10, A5XX_RBBM_PERFCTR_RBBM_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RBBM_3_LO, + A5XX_RBBM_PERFCTR_RBBM_3_HI, 11, A5XX_RBBM_PERFCTR_RBBM_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_pc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_0_LO, + A5XX_RBBM_PERFCTR_PC_0_HI, 12, A5XX_PC_PERFCTR_PC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_1_LO, + A5XX_RBBM_PERFCTR_PC_1_HI, 13, A5XX_PC_PERFCTR_PC_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_2_LO, + A5XX_RBBM_PERFCTR_PC_2_HI, 14, A5XX_PC_PERFCTR_PC_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_3_LO, + A5XX_RBBM_PERFCTR_PC_3_HI, 15, A5XX_PC_PERFCTR_PC_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_4_LO, + A5XX_RBBM_PERFCTR_PC_4_HI, 16, A5XX_PC_PERFCTR_PC_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_5_LO, + A5XX_RBBM_PERFCTR_PC_5_HI, 17, A5XX_PC_PERFCTR_PC_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_6_LO, + A5XX_RBBM_PERFCTR_PC_6_HI, 18, A5XX_PC_PERFCTR_PC_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_PC_7_LO, + A5XX_RBBM_PERFCTR_PC_7_HI, 19, A5XX_PC_PERFCTR_PC_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_vfd[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_0_LO, + A5XX_RBBM_PERFCTR_VFD_0_HI, 20, A5XX_VFD_PERFCTR_VFD_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_1_LO, + A5XX_RBBM_PERFCTR_VFD_1_HI, 21, A5XX_VFD_PERFCTR_VFD_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_2_LO, + A5XX_RBBM_PERFCTR_VFD_2_HI, 22, A5XX_VFD_PERFCTR_VFD_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_3_LO, + A5XX_RBBM_PERFCTR_VFD_3_HI, 23, A5XX_VFD_PERFCTR_VFD_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_4_LO, + A5XX_RBBM_PERFCTR_VFD_4_HI, 24, A5XX_VFD_PERFCTR_VFD_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_5_LO, + A5XX_RBBM_PERFCTR_VFD_5_HI, 25, A5XX_VFD_PERFCTR_VFD_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_6_LO, + A5XX_RBBM_PERFCTR_VFD_6_HI, 26, A5XX_VFD_PERFCTR_VFD_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VFD_7_LO, + A5XX_RBBM_PERFCTR_VFD_7_HI, 27, A5XX_VFD_PERFCTR_VFD_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_hlsq[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_0_LO, + A5XX_RBBM_PERFCTR_HLSQ_0_HI, 28, A5XX_HLSQ_PERFCTR_HLSQ_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_1_LO, + A5XX_RBBM_PERFCTR_HLSQ_1_HI, 29, A5XX_HLSQ_PERFCTR_HLSQ_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_2_LO, + A5XX_RBBM_PERFCTR_HLSQ_2_HI, 30, A5XX_HLSQ_PERFCTR_HLSQ_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_3_LO, + A5XX_RBBM_PERFCTR_HLSQ_3_HI, 31, A5XX_HLSQ_PERFCTR_HLSQ_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_4_LO, + A5XX_RBBM_PERFCTR_HLSQ_4_HI, 32, A5XX_HLSQ_PERFCTR_HLSQ_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_5_LO, + A5XX_RBBM_PERFCTR_HLSQ_5_HI, 33, A5XX_HLSQ_PERFCTR_HLSQ_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_6_LO, + A5XX_RBBM_PERFCTR_HLSQ_6_HI, 34, A5XX_HLSQ_PERFCTR_HLSQ_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_HLSQ_7_LO, + A5XX_RBBM_PERFCTR_HLSQ_7_HI, 35, A5XX_HLSQ_PERFCTR_HLSQ_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_vpc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VPC_0_LO, + A5XX_RBBM_PERFCTR_VPC_0_HI, 36, A5XX_VPC_PERFCTR_VPC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VPC_1_LO, + A5XX_RBBM_PERFCTR_VPC_1_HI, 37, A5XX_VPC_PERFCTR_VPC_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VPC_2_LO, + A5XX_RBBM_PERFCTR_VPC_2_HI, 38, A5XX_VPC_PERFCTR_VPC_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VPC_3_LO, + A5XX_RBBM_PERFCTR_VPC_3_HI, 39, A5XX_VPC_PERFCTR_VPC_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_ccu[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CCU_0_LO, + A5XX_RBBM_PERFCTR_CCU_0_HI, 40, A5XX_RB_PERFCTR_CCU_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CCU_1_LO, + A5XX_RBBM_PERFCTR_CCU_1_HI, 41, A5XX_RB_PERFCTR_CCU_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CCU_2_LO, + A5XX_RBBM_PERFCTR_CCU_2_HI, 42, A5XX_RB_PERFCTR_CCU_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CCU_3_LO, + A5XX_RBBM_PERFCTR_CCU_3_HI, 43, A5XX_RB_PERFCTR_CCU_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_tse[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TSE_0_LO, + A5XX_RBBM_PERFCTR_TSE_0_HI, 44, A5XX_GRAS_PERFCTR_TSE_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TSE_1_LO, + A5XX_RBBM_PERFCTR_TSE_1_HI, 45, A5XX_GRAS_PERFCTR_TSE_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TSE_2_LO, + A5XX_RBBM_PERFCTR_TSE_2_HI, 46, A5XX_GRAS_PERFCTR_TSE_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TSE_3_LO, + A5XX_RBBM_PERFCTR_TSE_3_HI, 47, A5XX_GRAS_PERFCTR_TSE_SEL_3 }, +}; + + +static struct adreno_perfcount_register a5xx_perfcounters_ras[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RAS_0_LO, + A5XX_RBBM_PERFCTR_RAS_0_HI, 48, A5XX_GRAS_PERFCTR_RAS_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RAS_1_LO, + A5XX_RBBM_PERFCTR_RAS_1_HI, 49, A5XX_GRAS_PERFCTR_RAS_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RAS_2_LO, + A5XX_RBBM_PERFCTR_RAS_2_HI, 50, A5XX_GRAS_PERFCTR_RAS_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RAS_3_LO, + A5XX_RBBM_PERFCTR_RAS_3_HI, 51, A5XX_GRAS_PERFCTR_RAS_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_uche[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_0_LO, + A5XX_RBBM_PERFCTR_UCHE_0_HI, 52, A5XX_UCHE_PERFCTR_UCHE_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_1_LO, + A5XX_RBBM_PERFCTR_UCHE_1_HI, 53, A5XX_UCHE_PERFCTR_UCHE_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_2_LO, + A5XX_RBBM_PERFCTR_UCHE_2_HI, 54, A5XX_UCHE_PERFCTR_UCHE_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_3_LO, + A5XX_RBBM_PERFCTR_UCHE_3_HI, 55, A5XX_UCHE_PERFCTR_UCHE_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_4_LO, + A5XX_RBBM_PERFCTR_UCHE_4_HI, 56, A5XX_UCHE_PERFCTR_UCHE_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_5_LO, + A5XX_RBBM_PERFCTR_UCHE_5_HI, 57, A5XX_UCHE_PERFCTR_UCHE_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_6_LO, + A5XX_RBBM_PERFCTR_UCHE_6_HI, 58, A5XX_UCHE_PERFCTR_UCHE_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_UCHE_7_LO, + A5XX_RBBM_PERFCTR_UCHE_7_HI, 59, A5XX_UCHE_PERFCTR_UCHE_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_tp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_0_LO, + A5XX_RBBM_PERFCTR_TP_0_HI, 60, A5XX_TPL1_PERFCTR_TP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_1_LO, + A5XX_RBBM_PERFCTR_TP_1_HI, 61, A5XX_TPL1_PERFCTR_TP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_2_LO, + A5XX_RBBM_PERFCTR_TP_2_HI, 62, A5XX_TPL1_PERFCTR_TP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_3_LO, + A5XX_RBBM_PERFCTR_TP_3_HI, 63, A5XX_TPL1_PERFCTR_TP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_4_LO, + A5XX_RBBM_PERFCTR_TP_4_HI, 64, A5XX_TPL1_PERFCTR_TP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_5_LO, + A5XX_RBBM_PERFCTR_TP_5_HI, 65, A5XX_TPL1_PERFCTR_TP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_6_LO, + A5XX_RBBM_PERFCTR_TP_6_HI, 66, A5XX_TPL1_PERFCTR_TP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_TP_7_LO, + A5XX_RBBM_PERFCTR_TP_7_HI, 67, A5XX_TPL1_PERFCTR_TP_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_sp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_0_LO, + A5XX_RBBM_PERFCTR_SP_0_HI, 68, A5XX_SP_PERFCTR_SP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_1_LO, + A5XX_RBBM_PERFCTR_SP_1_HI, 69, A5XX_SP_PERFCTR_SP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_2_LO, + A5XX_RBBM_PERFCTR_SP_2_HI, 70, A5XX_SP_PERFCTR_SP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_3_LO, + A5XX_RBBM_PERFCTR_SP_3_HI, 71, A5XX_SP_PERFCTR_SP_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_4_LO, + A5XX_RBBM_PERFCTR_SP_4_HI, 72, A5XX_SP_PERFCTR_SP_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_5_LO, + A5XX_RBBM_PERFCTR_SP_5_HI, 73, A5XX_SP_PERFCTR_SP_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_6_LO, + A5XX_RBBM_PERFCTR_SP_6_HI, 74, A5XX_SP_PERFCTR_SP_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_7_LO, + A5XX_RBBM_PERFCTR_SP_7_HI, 75, A5XX_SP_PERFCTR_SP_SEL_7 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_8_LO, + A5XX_RBBM_PERFCTR_SP_8_HI, 76, A5XX_SP_PERFCTR_SP_SEL_8 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_9_LO, + A5XX_RBBM_PERFCTR_SP_9_HI, 77, A5XX_SP_PERFCTR_SP_SEL_9 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_10_LO, + A5XX_RBBM_PERFCTR_SP_10_HI, 78, A5XX_SP_PERFCTR_SP_SEL_10 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_SP_11_LO, + A5XX_RBBM_PERFCTR_SP_11_HI, 79, A5XX_SP_PERFCTR_SP_SEL_11 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_rb[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_0_LO, + A5XX_RBBM_PERFCTR_RB_0_HI, 80, A5XX_RB_PERFCTR_RB_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_1_LO, + A5XX_RBBM_PERFCTR_RB_1_HI, 81, A5XX_RB_PERFCTR_RB_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_2_LO, + A5XX_RBBM_PERFCTR_RB_2_HI, 82, A5XX_RB_PERFCTR_RB_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_3_LO, + A5XX_RBBM_PERFCTR_RB_3_HI, 83, A5XX_RB_PERFCTR_RB_SEL_3 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_4_LO, + A5XX_RBBM_PERFCTR_RB_4_HI, 84, A5XX_RB_PERFCTR_RB_SEL_4 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_5_LO, + A5XX_RBBM_PERFCTR_RB_5_HI, 85, A5XX_RB_PERFCTR_RB_SEL_5 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_6_LO, + A5XX_RBBM_PERFCTR_RB_6_HI, 86, A5XX_RB_PERFCTR_RB_SEL_6 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_RB_7_LO, + A5XX_RBBM_PERFCTR_RB_7_HI, 87, A5XX_RB_PERFCTR_RB_SEL_7 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_vsc[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VSC_0_LO, + A5XX_RBBM_PERFCTR_VSC_0_HI, 88, A5XX_VSC_PERFCTR_VSC_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_VSC_1_LO, + A5XX_RBBM_PERFCTR_VSC_1_HI, 89, A5XX_VSC_PERFCTR_VSC_SEL_1 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_lrz[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_LRZ_0_LO, + A5XX_RBBM_PERFCTR_LRZ_0_HI, 90, A5XX_GRAS_PERFCTR_LRZ_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_LRZ_1_LO, + A5XX_RBBM_PERFCTR_LRZ_1_HI, 91, A5XX_GRAS_PERFCTR_LRZ_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_LRZ_2_LO, + A5XX_RBBM_PERFCTR_LRZ_2_HI, 92, A5XX_GRAS_PERFCTR_LRZ_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_LRZ_3_LO, + A5XX_RBBM_PERFCTR_LRZ_3_HI, 93, A5XX_GRAS_PERFCTR_LRZ_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_cmp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CMP_0_LO, + A5XX_RBBM_PERFCTR_CMP_0_HI, 94, A5XX_RB_PERFCTR_CMP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CMP_1_LO, + A5XX_RBBM_PERFCTR_CMP_1_HI, 95, A5XX_RB_PERFCTR_CMP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CMP_2_LO, + A5XX_RBBM_PERFCTR_CMP_2_HI, 96, A5XX_RB_PERFCTR_CMP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_PERFCTR_CMP_3_LO, + A5XX_RBBM_PERFCTR_CMP_3_HI, 97, A5XX_RB_PERFCTR_CMP_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_vbif[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_CNT_LOW0, + A5XX_VBIF_PERF_CNT_HIGH0, -1, A5XX_VBIF_PERF_CNT_SEL0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_CNT_LOW1, + A5XX_VBIF_PERF_CNT_HIGH1, -1, A5XX_VBIF_PERF_CNT_SEL1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_CNT_LOW2, + A5XX_VBIF_PERF_CNT_HIGH2, -1, A5XX_VBIF_PERF_CNT_SEL2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_CNT_LOW3, + A5XX_VBIF_PERF_CNT_HIGH3, -1, A5XX_VBIF_PERF_CNT_SEL3 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_vbif_pwr[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_PWR_CNT_LOW0, + A5XX_VBIF_PERF_PWR_CNT_HIGH0, -1, A5XX_VBIF_PERF_PWR_CNT_EN0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_PWR_CNT_LOW1, + A5XX_VBIF_PERF_PWR_CNT_HIGH1, -1, A5XX_VBIF_PERF_PWR_CNT_EN1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_VBIF_PERF_PWR_CNT_LOW2, + A5XX_VBIF_PERF_PWR_CNT_HIGH2, -1, A5XX_VBIF_PERF_PWR_CNT_EN2 }, +}; + +static struct adreno_perfcount_register a5xx_perfcounters_alwayson[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RBBM_ALWAYSON_COUNTER_LO, + A5XX_RBBM_ALWAYSON_COUNTER_HI, -1 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_sp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_SP_POWER_COUNTER_0_LO, + A5XX_SP_POWER_COUNTER_0_HI, -1, A5XX_SP_POWERCTR_SP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_SP_POWER_COUNTER_1_LO, + A5XX_SP_POWER_COUNTER_1_HI, -1, A5XX_SP_POWERCTR_SP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_SP_POWER_COUNTER_2_LO, + A5XX_SP_POWER_COUNTER_2_HI, -1, A5XX_SP_POWERCTR_SP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_SP_POWER_COUNTER_3_LO, + A5XX_SP_POWER_COUNTER_3_HI, -1, A5XX_SP_POWERCTR_SP_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_tp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_TP_POWER_COUNTER_0_LO, + A5XX_TP_POWER_COUNTER_0_HI, -1, A5XX_TPL1_POWERCTR_TP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_TP_POWER_COUNTER_1_LO, + A5XX_TP_POWER_COUNTER_1_HI, -1, A5XX_TPL1_POWERCTR_TP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_TP_POWER_COUNTER_2_LO, + A5XX_TP_POWER_COUNTER_2_HI, -1, A5XX_TPL1_POWERCTR_TP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_TP_POWER_COUNTER_3_LO, + A5XX_TP_POWER_COUNTER_3_HI, -1, A5XX_TPL1_POWERCTR_TP_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_rb[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RB_POWER_COUNTER_0_LO, + A5XX_RB_POWER_COUNTER_0_HI, -1, A5XX_RB_POWERCTR_RB_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RB_POWER_COUNTER_1_LO, + A5XX_RB_POWER_COUNTER_1_HI, -1, A5XX_RB_POWERCTR_RB_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RB_POWER_COUNTER_2_LO, + A5XX_RB_POWER_COUNTER_2_HI, -1, A5XX_RB_POWERCTR_RB_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_RB_POWER_COUNTER_3_LO, + A5XX_RB_POWER_COUNTER_3_HI, -1, A5XX_RB_POWERCTR_RB_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_ccu[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CCU_POWER_COUNTER_0_LO, + A5XX_CCU_POWER_COUNTER_0_HI, -1, A5XX_RB_POWERCTR_CCU_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CCU_POWER_COUNTER_1_LO, + A5XX_CCU_POWER_COUNTER_1_HI, -1, A5XX_RB_POWERCTR_CCU_SEL_1 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_uche[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_UCHE_POWER_COUNTER_0_LO, + A5XX_UCHE_POWER_COUNTER_0_HI, -1, + A5XX_UCHE_POWERCTR_UCHE_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_UCHE_POWER_COUNTER_1_LO, + A5XX_UCHE_POWER_COUNTER_1_HI, -1, + A5XX_UCHE_POWERCTR_UCHE_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_UCHE_POWER_COUNTER_2_LO, + A5XX_UCHE_POWER_COUNTER_2_HI, -1, + A5XX_UCHE_POWERCTR_UCHE_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_UCHE_POWER_COUNTER_3_LO, + A5XX_UCHE_POWER_COUNTER_3_HI, -1, + A5XX_UCHE_POWERCTR_UCHE_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_cp[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CP_POWER_COUNTER_0_LO, + A5XX_CP_POWER_COUNTER_0_HI, -1, A5XX_CP_POWERCTR_CP_SEL_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CP_POWER_COUNTER_1_LO, + A5XX_CP_POWER_COUNTER_1_HI, -1, A5XX_CP_POWERCTR_CP_SEL_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CP_POWER_COUNTER_2_LO, + A5XX_CP_POWER_COUNTER_2_HI, -1, A5XX_CP_POWERCTR_CP_SEL_2 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_CP_POWER_COUNTER_3_LO, + A5XX_CP_POWER_COUNTER_3_HI, -1, A5XX_CP_POWERCTR_CP_SEL_3 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_gpmu[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_0_LO, + A5XX_GPMU_POWER_COUNTER_0_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_1_LO, + A5XX_GPMU_POWER_COUNTER_1_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_2_LO, + A5XX_GPMU_POWER_COUNTER_2_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_3_LO, + A5XX_GPMU_POWER_COUNTER_3_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_0 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_4_LO, + A5XX_GPMU_POWER_COUNTER_4_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_1 }, + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_POWER_COUNTER_5_LO, + A5XX_GPMU_POWER_COUNTER_5_HI, -1, + A5XX_GPMU_POWER_COUNTER_SELECT_1 }, +}; + +static struct adreno_perfcount_register a5xx_pwrcounters_alwayson[] = { + { KGSL_PERFCOUNTER_NOT_USED, 0, 0, A5XX_GPMU_ALWAYS_ON_COUNTER_LO, + A5XX_GPMU_ALWAYS_ON_COUNTER_HI, -1 }, +}; + +#define A5XX_PERFCOUNTER_GROUP(offset, name, enable, read, load) \ + ADRENO_PERFCOUNTER_GROUP(a5xx, offset, name, enable, read, load) + +#define A5XX_PERFCOUNTER_GROUP_FLAGS(offset, name, flags, enable, read, load) \ + ADRENO_PERFCOUNTER_GROUP_FLAGS(a5xx, offset, name, flags, enable, \ + read, load) + +#define A5XX_POWER_COUNTER_GROUP(offset, name, enable, read) \ + [KGSL_PERFCOUNTER_GROUP_##offset##_PWR] = { a5xx_pwrcounters_##name, \ + ARRAY_SIZE(a5xx_pwrcounters_##name), __stringify(name##_pwr), 0, \ + enable, read, NULL } + +#define A5XX_REGULAR_PERFCOUNTER_GROUP(offset, name) \ + A5XX_PERFCOUNTER_GROUP(offset, name, a5xx_counter_inline_enable, \ + a5xx_counter_read, a5xx_counter_load) + +static struct adreno_perfcount_group a5xx_perfcounter_groups + [KGSL_PERFCOUNTER_GROUP_MAX] = { + A5XX_REGULAR_PERFCOUNTER_GROUP(CP, cp), + A5XX_PERFCOUNTER_GROUP(RBBM, rbbm, + a5xx_counter_rbbm_enable, a5xx_counter_read, a5xx_counter_load), + A5XX_REGULAR_PERFCOUNTER_GROUP(PC, pc), + A5XX_REGULAR_PERFCOUNTER_GROUP(VFD, vfd), + A5XX_REGULAR_PERFCOUNTER_GROUP(HLSQ, hlsq), + A5XX_REGULAR_PERFCOUNTER_GROUP(VPC, vpc), + A5XX_REGULAR_PERFCOUNTER_GROUP(CCU, ccu), + A5XX_REGULAR_PERFCOUNTER_GROUP(CMP, cmp), + A5XX_REGULAR_PERFCOUNTER_GROUP(TSE, tse), + A5XX_REGULAR_PERFCOUNTER_GROUP(RAS, ras), + A5XX_REGULAR_PERFCOUNTER_GROUP(LRZ, lrz), + A5XX_REGULAR_PERFCOUNTER_GROUP(UCHE, uche), + A5XX_REGULAR_PERFCOUNTER_GROUP(TP, tp), + A5XX_REGULAR_PERFCOUNTER_GROUP(SP, sp), + A5XX_REGULAR_PERFCOUNTER_GROUP(RB, rb), + A5XX_REGULAR_PERFCOUNTER_GROUP(VSC, vsc), + A5XX_PERFCOUNTER_GROUP(VBIF, vbif, + a5xx_counter_vbif_enable, a5xx_counter_read_norestore, NULL), + A5XX_PERFCOUNTER_GROUP_FLAGS(VBIF_PWR, vbif_pwr, + ADRENO_PERFCOUNTER_GROUP_FIXED, + a5xx_counter_vbif_pwr_enable, + a5xx_counter_read_norestore, NULL), + A5XX_PERFCOUNTER_GROUP_FLAGS(ALWAYSON, alwayson, + ADRENO_PERFCOUNTER_GROUP_FIXED, + a5xx_counter_alwayson_enable, a5xx_counter_alwayson_read, NULL), + A5XX_POWER_COUNTER_GROUP(SP, sp, + a5xx_counter_pwr_enable, a5xx_counter_read_norestore), + A5XX_POWER_COUNTER_GROUP(TP, tp, + a5xx_counter_pwr_enable, a5xx_counter_read_norestore), + A5XX_POWER_COUNTER_GROUP(RB, rb, + a5xx_counter_pwr_enable, a5xx_counter_read_norestore), + A5XX_POWER_COUNTER_GROUP(CCU, ccu, + a5xx_counter_pwr_enable, a5xx_counter_read_norestore), + A5XX_POWER_COUNTER_GROUP(UCHE, uche, + a5xx_counter_pwr_enable, a5xx_counter_read_norestore), + A5XX_POWER_COUNTER_GROUP(CP, cp, + a5xx_counter_pwr_enable, a5xx_counter_read_norestore), + A5XX_POWER_COUNTER_GROUP(GPMU, gpmu, + a5xx_counter_pwr_gpmu_enable, a5xx_counter_read_norestore), + A5XX_POWER_COUNTER_GROUP(ALWAYSON, alwayson, + a5xx_counter_pwr_alwayson_enable, a5xx_counter_read_norestore), +}; + +const struct adreno_perfcounters adreno_a5xx_perfcounters = { + a5xx_perfcounter_groups, + ARRAY_SIZE(a5xx_perfcounter_groups), +}; diff --git a/qcom/opensource/graphics-kernel/adreno_a5xx_preempt.c b/qcom/opensource/graphics-kernel/adreno_a5xx_preempt.c new file mode 100644 index 0000000000..4428beb246 --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a5xx_preempt.c @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2014-2017,2021 The Linux Foundation. All rights reserved. + * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include "adreno.h" +#include "adreno_a5xx.h" +#include "adreno_pm4types.h" +#include "adreno_trace.h" + +#define PREEMPT_RECORD(_field) \ + offsetof(struct a5xx_cp_preemption_record, _field) + +#define PREEMPT_SMMU_RECORD(_field) \ + offsetof(struct a5xx_cp_smmu_info, _field) + +static void _update_wptr(struct adreno_device *adreno_dev, bool reset_timer) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_ringbuffer *rb = adreno_dev->cur_rb; + unsigned int wptr; + unsigned long flags; + + spin_lock_irqsave(&rb->preempt_lock, flags); + + kgsl_regread(device, A5XX_CP_RB_WPTR, &wptr); + + if (wptr != rb->wptr) { + kgsl_regwrite(device, A5XX_CP_RB_WPTR, rb->wptr); + /* + * In case something got submitted while preemption was on + * going, reset the timer. + */ + reset_timer = true; + } + + if (reset_timer) + rb->dispatch_q.expires = jiffies + + msecs_to_jiffies(adreno_drawobj_timeout); + + spin_unlock_irqrestore(&rb->preempt_lock, flags); +} + +static void _a5xx_preemption_done(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int status; + + /* + * In the very unlikely case that the power is off, do nothing - the + * state will be reset on power up and everybody will be happy + */ + + if (!kgsl_state_is_awake(device)) + return; + + kgsl_regread(device, A5XX_CP_CONTEXT_SWITCH_CNTL, &status); + + if (status != 0) { + dev_err(device->dev, + "Preemption not complete: status=%X cur=%d R/W=%X/%X next=%d R/W=%X/%X\n", + status, adreno_dev->cur_rb->id, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_dev->cur_rb->wptr, + adreno_dev->next_rb->id, + adreno_get_rptr(adreno_dev->next_rb), + adreno_dev->next_rb->wptr); + + /* Set a fault and restart */ + adreno_dispatcher_fault(adreno_dev, ADRENO_PREEMPT_FAULT); + + return; + } + + del_timer_sync(&adreno_dev->preempt.timer); + + trace_adreno_preempt_done(adreno_dev->cur_rb->id, adreno_dev->next_rb->id, 0, 0); + + /* Clean up all the bits */ + adreno_dev->prev_rb = adreno_dev->cur_rb; + adreno_dev->cur_rb = adreno_dev->next_rb; + adreno_dev->next_rb = NULL; + + /* Update the wptr for the new command queue */ + _update_wptr(adreno_dev, true); + + /* Update the dispatcher timer for the new command queue */ + mod_timer(&adreno_dev->dispatcher.timer, + adreno_dev->cur_rb->dispatch_q.expires); + + /* Clear the preempt state */ + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE); +} + +static void _a5xx_preemption_fault(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int status; + + /* + * If the power is on check the preemption status one more time - if it + * was successful then just transition to the complete state + */ + if (kgsl_state_is_awake(device)) { + kgsl_regread(device, A5XX_CP_CONTEXT_SWITCH_CNTL, &status); + + if (status == 0) { + adreno_set_preempt_state(adreno_dev, + ADRENO_PREEMPT_COMPLETE); + + adreno_dispatcher_schedule(device); + return; + } + } + + dev_err(device->dev, + "Preemption timed out: cur=%d R/W=%X/%X, next=%d R/W=%X/%X\n", + adreno_dev->cur_rb->id, + adreno_get_rptr(adreno_dev->cur_rb), + adreno_dev->cur_rb->wptr, + adreno_dev->next_rb->id, + adreno_get_rptr(adreno_dev->next_rb), + adreno_dev->next_rb->wptr); + + adreno_dispatcher_fault(adreno_dev, ADRENO_PREEMPT_FAULT); +} + +static void _a5xx_preemption_worker(struct work_struct *work) +{ + struct adreno_preemption *preempt = container_of(work, + struct adreno_preemption, work); + struct adreno_device *adreno_dev = container_of(preempt, + struct adreno_device, preempt); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* Need to take the mutex to make sure that the power stays on */ + mutex_lock(&device->mutex); + + if (adreno_in_preempt_state(adreno_dev, ADRENO_PREEMPT_FAULTED)) + _a5xx_preemption_fault(adreno_dev); + + mutex_unlock(&device->mutex); +} + +/* Find the highest priority active ringbuffer */ +static struct adreno_ringbuffer *a5xx_next_ringbuffer( + struct adreno_device *adreno_dev) +{ + struct adreno_ringbuffer *rb; + unsigned long flags; + unsigned int i; + + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + bool empty; + + spin_lock_irqsave(&rb->preempt_lock, flags); + empty = adreno_rb_empty(rb); + spin_unlock_irqrestore(&rb->preempt_lock, flags); + + if (!empty) + return rb; + } + + return NULL; +} + +void a5xx_preemption_trigger(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_iommu *iommu = KGSL_IOMMU(device); + struct adreno_ringbuffer *next; + uint64_t ttbr0; + unsigned int contextidr; + unsigned long flags; + + /* Put ourselves into a possible trigger state */ + if (!adreno_move_preempt_state(adreno_dev, + ADRENO_PREEMPT_NONE, ADRENO_PREEMPT_START)) + return; + + /* Get the next ringbuffer to preempt in */ + next = a5xx_next_ringbuffer(adreno_dev); + + /* + * Nothing to do if every ringbuffer is empty or if the current + * ringbuffer is the only active one + */ + if (next == NULL || next == adreno_dev->cur_rb) { + /* + * Update any critical things that might have been skipped while + * we were looking for a new ringbuffer + */ + + if (next != NULL) { + _update_wptr(adreno_dev, false); + + mod_timer(&adreno_dev->dispatcher.timer, + adreno_dev->cur_rb->dispatch_q.expires); + } + + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE); + return; + } + + /* Turn off the dispatcher timer */ + del_timer(&adreno_dev->dispatcher.timer); + + /* + * This is the most critical section - we need to take care not to race + * until we have programmed the CP for the switch + */ + + spin_lock_irqsave(&next->preempt_lock, flags); + + /* Get the pagetable from the pagetable info. */ + kgsl_sharedmem_readq(device->scratch, &ttbr0, + SCRATCH_RB_OFFSET(next->id, ttbr0)); + kgsl_sharedmem_readl(device->scratch, &contextidr, + SCRATCH_RB_OFFSET(next->id, contextidr)); + + kgsl_sharedmem_writel(next->preemption_desc, + PREEMPT_RECORD(wptr), next->wptr); + + spin_unlock_irqrestore(&next->preempt_lock, flags); + + /* And write it to the smmu info */ + if (kgsl_mmu_is_perprocess(&device->mmu)) { + kgsl_sharedmem_writeq(iommu->smmu_info, + PREEMPT_SMMU_RECORD(ttbr0), ttbr0); + kgsl_sharedmem_writel(iommu->smmu_info, + PREEMPT_SMMU_RECORD(context_idr), contextidr); + } + + kgsl_regwrite(device, A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO, + lower_32_bits(next->preemption_desc->gpuaddr)); + kgsl_regwrite(device, A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI, + upper_32_bits(next->preemption_desc->gpuaddr)); + + adreno_dev->next_rb = next; + + /* Start the timer to detect a stuck preemption */ + mod_timer(&adreno_dev->preempt.timer, + jiffies + msecs_to_jiffies(ADRENO_PREEMPT_TIMEOUT)); + + trace_adreno_preempt_trigger(adreno_dev->cur_rb->id, adreno_dev->next_rb->id, + 1, 0); + + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_TRIGGERED); + + /* Trigger the preemption */ + kgsl_regwrite(device, A5XX_CP_CONTEXT_SWITCH_CNTL, 1); +} + +void a5xx_preempt_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int status; + + if (!adreno_move_preempt_state(adreno_dev, + ADRENO_PREEMPT_TRIGGERED, ADRENO_PREEMPT_PENDING)) + return; + + kgsl_regread(device, A5XX_CP_CONTEXT_SWITCH_CNTL, &status); + + if (status != 0) { + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "preempt interrupt with non-zero status: %X\n", + status); + + /* + * Under the assumption that this is a race between the + * interrupt and the register, schedule the worker to clean up. + * If the status still hasn't resolved itself by the time we get + * there then we have to assume something bad happened + */ + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_COMPLETE); + adreno_dispatcher_schedule(device); + return; + } + + del_timer(&adreno_dev->preempt.timer); + + trace_adreno_preempt_done(adreno_dev->cur_rb->id, adreno_dev->next_rb->id, 0, 0); + + adreno_dev->prev_rb = adreno_dev->cur_rb; + adreno_dev->cur_rb = adreno_dev->next_rb; + adreno_dev->next_rb = NULL; + + /* Update the wptr if it changed while preemption was ongoing */ + _update_wptr(adreno_dev, true); + + /* Update the dispatcher timer for the new command queue */ + mod_timer(&adreno_dev->dispatcher.timer, + adreno_dev->cur_rb->dispatch_q.expires); + + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE); + + a5xx_preemption_trigger(adreno_dev); +} + +void a5xx_preemption_schedule(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!adreno_is_preemption_enabled(adreno_dev)) + return; + + mutex_lock(&device->mutex); + + if (adreno_in_preempt_state(adreno_dev, ADRENO_PREEMPT_COMPLETE)) + _a5xx_preemption_done(adreno_dev); + + a5xx_preemption_trigger(adreno_dev); + + mutex_unlock(&device->mutex); +} + +u32 a5xx_preemption_pre_ibsubmit(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, + struct adreno_context *drawctxt, u32 *cmds) +{ + unsigned int *cmds_orig = cmds; + uint64_t gpuaddr = rb->preemption_desc->gpuaddr; + unsigned int preempt_style = 0; + + if (!adreno_is_preemption_enabled(adreno_dev)) + return 0; + + if (drawctxt) { + /* + * Preemption from secure to unsecure needs Zap shader to be + * run to clear all secure content. CP does not know during + * preemption if it is switching between secure and unsecure + * contexts so restrict Secure contexts to be preempted at + * ringbuffer level. + */ + if (drawctxt->base.flags & KGSL_CONTEXT_SECURE) + preempt_style = KGSL_CONTEXT_PREEMPT_STYLE_RINGBUFFER; + else + preempt_style = FIELD_GET(KGSL_CONTEXT_PREEMPT_STYLE_MASK, + drawctxt->base.flags); + } + + /* + * CP_PREEMPT_ENABLE_GLOBAL(global preemption) can only be set by KMD + * in ringbuffer. + * 1) set global preemption to 0x0 to disable global preemption. + * Only RB level preemption is allowed in this mode + * 2) Set global preemption to defer(0x2) for finegrain preemption. + * when global preemption is set to defer(0x2), + * CP_PREEMPT_ENABLE_LOCAL(local preemption) determines the + * preemption point. Local preemption + * can be enabled by both UMD(within IB) and KMD. + */ + *cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_GLOBAL, 1); + *cmds++ = ((preempt_style == KGSL_CONTEXT_PREEMPT_STYLE_FINEGRAIN) + ? 2 : 0); + + /* Turn CP protection OFF */ + cmds += cp_protected_mode(adreno_dev, cmds, 0); + + /* + * CP during context switch will save context switch info to + * a5xx_cp_preemption_record pointed by CONTEXT_SWITCH_SAVE_ADDR + */ + *cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 1); + *cmds++ = lower_32_bits(gpuaddr); + *cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_HI, 1); + *cmds++ = upper_32_bits(gpuaddr); + + /* Turn CP protection ON */ + cmds += cp_protected_mode(adreno_dev, cmds, 1); + + /* + * Enable local preemption for finegrain preemption in case of + * a misbehaving IB + */ + if (preempt_style == KGSL_CONTEXT_PREEMPT_STYLE_FINEGRAIN) { + *cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_LOCAL, 1); + *cmds++ = 1; + } else { + *cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_LOCAL, 1); + *cmds++ = 0; + } + + /* Enable CP_CONTEXT_SWITCH_YIELD packets in the IB2s */ + *cmds++ = cp_type7_packet(CP_YIELD_ENABLE, 1); + *cmds++ = 2; + + return (unsigned int) (cmds - cmds_orig); +} + +unsigned int a5xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev, + unsigned int *cmds) +{ + int dwords = 0; + + if (!adreno_is_preemption_enabled(adreno_dev)) + return 0; + + cmds[dwords++] = cp_type7_packet(CP_CONTEXT_SWITCH_YIELD, 4); + /* Write NULL to the address to skip the data write */ + dwords += cp_gpuaddr(adreno_dev, &cmds[dwords], 0x0); + cmds[dwords++] = 1; + /* generate interrupt on preemption completion */ + cmds[dwords++] = 1; + + return dwords; +} + +void a5xx_preemption_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_iommu *iommu = KGSL_IOMMU(device); + struct adreno_ringbuffer *rb; + unsigned int i; + + if (!adreno_is_preemption_enabled(adreno_dev)) + return; + + /* Force the state to be clear */ + adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE); + + /* Only set up smmu info when per-process pagetables are enabled */ + + if (kgsl_mmu_is_perprocess(&device->mmu)) { + /* smmu_info is allocated and mapped in a5xx_preemption_iommu_init */ + kgsl_sharedmem_writel(iommu->smmu_info, + PREEMPT_SMMU_RECORD(magic), A5XX_CP_SMMU_INFO_MAGIC_REF); + kgsl_sharedmem_writeq(iommu->smmu_info, + PREEMPT_SMMU_RECORD(ttbr0), MMU_DEFAULT_TTBR0(device)); + + /* The CP doesn't use the asid record, so poison it */ + kgsl_sharedmem_writel(iommu->smmu_info, + PREEMPT_SMMU_RECORD(asid), 0xDECAFBAD); + kgsl_sharedmem_writel(iommu->smmu_info, + PREEMPT_SMMU_RECORD(context_idr), 0); + + kgsl_regwrite(device, A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO, + lower_32_bits(iommu->smmu_info->gpuaddr)); + + kgsl_regwrite(device, A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, + upper_32_bits(iommu->smmu_info->gpuaddr)); + } + + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + /* + * preemption_desc is allocated and mapped at init time, + * so no need to check sharedmem_writel return value + */ + kgsl_sharedmem_writel(rb->preemption_desc, + PREEMPT_RECORD(rptr), 0); + kgsl_sharedmem_writel(rb->preemption_desc, + PREEMPT_RECORD(wptr), 0); + + adreno_ringbuffer_set_pagetable(device, rb, + device->mmu.defaultpagetable); + } + +} + +static int a5xx_preemption_ringbuffer_init(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, uint64_t counteraddr) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (IS_ERR_OR_NULL(rb->preemption_desc)) + rb->preemption_desc = kgsl_allocate_global(device, + A5XX_CP_CTXRECORD_SIZE_IN_BYTES, SZ_16K, 0, + KGSL_MEMDESC_PRIVILEGED, "preemption_desc"); + + if (IS_ERR(rb->preemption_desc)) + return PTR_ERR(rb->preemption_desc); + + kgsl_sharedmem_writel(rb->preemption_desc, + PREEMPT_RECORD(magic), A5XX_CP_CTXRECORD_MAGIC_REF); + kgsl_sharedmem_writel(rb->preemption_desc, + PREEMPT_RECORD(info), 0); + kgsl_sharedmem_writel(rb->preemption_desc, + PREEMPT_RECORD(data), 0); + kgsl_sharedmem_writel(rb->preemption_desc, + PREEMPT_RECORD(cntl), A5XX_CP_RB_CNTL_DEFAULT); + kgsl_sharedmem_writel(rb->preemption_desc, + PREEMPT_RECORD(rptr), 0); + kgsl_sharedmem_writel(rb->preemption_desc, + PREEMPT_RECORD(wptr), 0); + kgsl_sharedmem_writeq(rb->preemption_desc, + PREEMPT_RECORD(rptr_addr), SCRATCH_RB_GPU_ADDR(device, + rb->id, rptr)); + kgsl_sharedmem_writeq(rb->preemption_desc, + PREEMPT_RECORD(rbase), rb->buffer_desc->gpuaddr); + kgsl_sharedmem_writeq(rb->preemption_desc, + PREEMPT_RECORD(counter), counteraddr); + + return 0; +} + +int a5xx_preemption_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_iommu *iommu = KGSL_IOMMU(device); + struct adreno_preemption *preempt = &adreno_dev->preempt; + struct adreno_ringbuffer *rb; + int ret; + unsigned int i; + uint64_t addr; + + /* We are dependent on IOMMU to make preemption go on the CP side */ + if (kgsl_mmu_get_mmutype(device) != KGSL_MMU_TYPE_IOMMU) + return -ENODEV; + + INIT_WORK(&preempt->work, _a5xx_preemption_worker); + + /* Allocate mem for storing preemption counters */ + if (IS_ERR_OR_NULL(preempt->scratch)) + preempt->scratch = kgsl_allocate_global(device, + adreno_dev->num_ringbuffers * + A5XX_CP_CTXRECORD_PREEMPTION_COUNTER_SIZE, 0, 0, 0, + "preemption_counters"); + + ret = PTR_ERR_OR_ZERO(preempt->scratch); + if (ret) + return ret; + + addr = preempt->scratch->gpuaddr; + + /* Allocate mem for storing preemption switch record */ + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + ret = a5xx_preemption_ringbuffer_init(adreno_dev, rb, addr); + if (ret) + return ret; + + addr += A5XX_CP_CTXRECORD_PREEMPTION_COUNTER_SIZE; + } + + /* Allocate mem for storing preemption smmu record */ + if (kgsl_mmu_is_perprocess(&device->mmu) && IS_ERR_OR_NULL(iommu->smmu_info)) + iommu->smmu_info = kgsl_allocate_global(device, PAGE_SIZE, 0, + KGSL_MEMFLAGS_GPUREADONLY, KGSL_MEMDESC_PRIVILEGED, + "smmu_info"); + + if (IS_ERR(iommu->smmu_info)) + return PTR_ERR(iommu->smmu_info); + + set_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv); + return 0; +} diff --git a/qcom/opensource/graphics-kernel/adreno_a5xx_ringbuffer.c b/qcom/opensource/graphics-kernel/adreno_a5xx_ringbuffer.c new file mode 100644 index 0000000000..feafeedcce --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a5xx_ringbuffer.c @@ -0,0 +1,530 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include "adreno.h" +#include "adreno_a5xx.h" +#include "adreno_pm4types.h" +#include "adreno_ringbuffer.h" +#include "adreno_trace.h" +#include "kgsl_trace.h" + +static int a5xx_rb_pagetable_switch(struct kgsl_device *device, + struct adreno_context *drawctxt, + struct adreno_ringbuffer *rb, + struct kgsl_pagetable *pagetable, u32 *cmds) +{ + u64 ttbr0 = kgsl_mmu_pagetable_get_ttbr0(pagetable); + u32 id = drawctxt ? drawctxt->base.id : 0; + + if (pagetable == device->mmu.defaultpagetable) + return 0; + + cmds[0] = cp_type7_packet(CP_SMMU_TABLE_UPDATE, 3); + cmds[1] = lower_32_bits(ttbr0); + cmds[2] = upper_32_bits(ttbr0); + cmds[3] = id; + + cmds[4] = cp_type7_packet(CP_WAIT_FOR_IDLE, 0); + cmds[5] = cp_type7_packet(CP_WAIT_FOR_ME, 0); + cmds[6] = cp_type4_packet(A5XX_CP_CNTL, 1); + cmds[7] = 1; + + cmds[8] = cp_type7_packet(CP_MEM_WRITE, 5); + cmds[9] = lower_32_bits(SCRATCH_RB_GPU_ADDR(device, + rb->id, ttbr0)); + cmds[10] = upper_32_bits(SCRATCH_RB_GPU_ADDR(device, + rb->id, ttbr0)); + cmds[11] = lower_32_bits(ttbr0); + cmds[12] = upper_32_bits(ttbr0); + cmds[13] = id; + + cmds[14] = cp_type7_packet(CP_WAIT_FOR_IDLE, 0); + cmds[15] = cp_type7_packet(CP_WAIT_FOR_ME, 0); + cmds[16] = cp_type4_packet(A5XX_CP_CNTL, 1); + cmds[17] = 0; + + return 18; +} + +#define RB_SOPTIMESTAMP(device, rb) \ + MEMSTORE_RB_GPU_ADDR(device, rb, soptimestamp) +#define CTXT_SOPTIMESTAMP(device, drawctxt) \ + MEMSTORE_ID_GPU_ADDR(device, (drawctxt)->base.id, soptimestamp) + +#define RB_EOPTIMESTAMP(device, rb) \ + MEMSTORE_RB_GPU_ADDR(device, rb, eoptimestamp) +#define CTXT_EOPTIMESTAMP(device, drawctxt) \ + MEMSTORE_ID_GPU_ADDR(device, (drawctxt)->base.id, eoptimestamp) + +int a5xx_ringbuffer_submit(struct adreno_ringbuffer *rb, + struct adreno_submit_time *time, bool sync) +{ + struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned long flags; + + adreno_get_submit_time(adreno_dev, rb, time); + adreno_profile_submit_time(time); + + if (sync) { + u32 *cmds = adreno_ringbuffer_allocspace(rb, 3); + + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + cmds[0] = cp_type7_packet(CP_WHERE_AM_I, 2); + cmds[1] = lower_32_bits(SCRATCH_RB_GPU_ADDR(device, rb->id, + rptr)); + cmds[2] = upper_32_bits(SCRATCH_RB_GPU_ADDR(device, rb->id, + rptr)); + } + + spin_lock_irqsave(&rb->preempt_lock, flags); + if (adreno_in_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE)) { + if (adreno_dev->cur_rb == rb) { + kgsl_pwrscale_busy(device); + kgsl_regwrite(device, A5XX_CP_RB_WPTR, rb->_wptr); + } + } + + rb->wptr = rb->_wptr; + spin_unlock_irqrestore(&rb->preempt_lock, flags); + + return 0; +} + +int a5xx_ringbuffer_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int i; + + if (IS_ERR_OR_NULL(device->scratch)) + device->scratch = kgsl_allocate_global(device, PAGE_SIZE, + 0, 0, KGSL_MEMDESC_RANDOM | KGSL_MEMDESC_PRIVILEGED, + "scratch"); + + if (IS_ERR(device->scratch)) + return PTR_ERR(device->scratch); + + adreno_dev->cur_rb = &(adreno_dev->ringbuffers[0]); + + if (!adreno_preemption_feature_set(adreno_dev)) { + adreno_dev->num_ringbuffers = 1; + return adreno_ringbuffer_setup(adreno_dev, + &adreno_dev->ringbuffers[0], 0); + } + + adreno_dev->num_ringbuffers = ARRAY_SIZE(adreno_dev->ringbuffers); + + for (i = 0; i < adreno_dev->num_ringbuffers; i++) { + int ret; + + ret = adreno_ringbuffer_setup(adreno_dev, + &adreno_dev->ringbuffers[i], i); + if (ret) + return ret; + } + + timer_setup(&adreno_dev->preempt.timer, adreno_preemption_timer, 0); + a5xx_preemption_init(adreno_dev); + return 0; +} + +#define A5XX_SUBMIT_MAX 64 + +int a5xx_ringbuffer_addcmds(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, struct adreno_context *drawctxt, + u32 flags, u32 *in, u32 dwords, u32 timestamp, + struct adreno_submit_time *time) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + static u32 sequence; + u32 size = A5XX_SUBMIT_MAX + dwords; + u32 *cmds, index = 0; + u64 profile_gpuaddr; + u32 profile_dwords; + + if (adreno_drawctxt_detached(drawctxt)) + return -ENOENT; + + if (adreno_gpu_fault(adreno_dev) != 0) + return -EPROTO; + + rb->timestamp++; + + if (drawctxt) + drawctxt->internal_timestamp = rb->timestamp; + + cmds = adreno_ringbuffer_allocspace(rb, size); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + /* Identify the start of a command */ + cmds[index++] = cp_type7_packet(CP_NOP, 1); + cmds[index++] = drawctxt ? CMD_IDENTIFIER : CMD_INTERNAL_IDENTIFIER; + + /* 14 dwords */ + index += a5xx_preemption_pre_ibsubmit(adreno_dev, rb, drawctxt, + &cmds[index]); + + profile_gpuaddr = adreno_profile_preib_processing(adreno_dev, + drawctxt, &profile_dwords); + + if (profile_gpuaddr) { + cmds[index++] = cp_type7_packet(CP_INDIRECT_BUFFER_PFE, 3); + cmds[index++] = lower_32_bits(profile_gpuaddr); + cmds[index++] = upper_32_bits(profile_gpuaddr); + cmds[index++] = profile_dwords; + } + + if (drawctxt) { + cmds[index++] = cp_type7_packet(CP_MEM_WRITE, 3); + cmds[index++] = lower_32_bits(CTXT_SOPTIMESTAMP(device, + drawctxt)); + cmds[index++] = upper_32_bits(CTXT_SOPTIMESTAMP(device, + drawctxt)); + cmds[index++] = timestamp; + } + + cmds[index++] = cp_type7_packet(CP_MEM_WRITE, 3); + cmds[index++] = lower_32_bits(RB_SOPTIMESTAMP(device, rb)); + cmds[index++] = upper_32_bits(RB_SOPTIMESTAMP(device, rb)); + cmds[index++] = rb->timestamp; + + if (IS_SECURE(flags)) { + cmds[index++] = cp_type7_packet(CP_SET_SECURE_MODE, 1); + cmds[index++] = 1; + } + + if (IS_NOTPROTECTED(flags)) { + cmds[index++] = cp_type7_packet(CP_SET_PROTECTED_MODE, 1); + cmds[index++] = 0; + } + + memcpy(&cmds[index], in, dwords << 2); + index += dwords; + + if (IS_NOTPROTECTED(flags)) { + cmds[index++] = cp_type7_packet(CP_SET_PROTECTED_MODE, 1); + cmds[index++] = 1; + } + + /* 4 dwords */ + profile_gpuaddr = adreno_profile_postib_processing(adreno_dev, + drawctxt, &profile_dwords); + + if (profile_gpuaddr) { + cmds[index++] = cp_type7_packet(CP_INDIRECT_BUFFER_PFE, 3); + cmds[index++] = lower_32_bits(profile_gpuaddr); + cmds[index++] = upper_32_bits(profile_gpuaddr); + cmds[index++] = profile_dwords; + } + + if (!adreno_is_a510(adreno_dev) && + test_bit(KGSL_FT_PAGEFAULT_GPUHALT_ENABLE, + &device->mmu.pfpolicy)) + cmds[index++] = cp_type7_packet(CP_WAIT_MEM_WRITES, 0); + + /* + * Do a unique memory write from the GPU to assist in early detection of + * interrupt storms + */ + + cmds[index++] = cp_type7_packet(CP_MEM_WRITE, 3); + cmds[index++] = lower_32_bits(MEMSTORE_ID_GPU_ADDR(device, + KGSL_MEMSTORE_GLOBAL, ref_wait_ts)); + cmds[index++] = upper_32_bits(MEMSTORE_ID_GPU_ADDR(device, + KGSL_MEMSTORE_GLOBAL, ref_wait_ts)); + cmds[index++] = ++sequence; + + /* + * If this is an internal command, just write the ringbuffer timestamp, + * otherwise, write both + */ + if (!drawctxt) { + cmds[index++] = cp_type7_packet(CP_EVENT_WRITE, 4); + cmds[index++] = CACHE_FLUSH_TS | (1 << 31); + cmds[index++] = lower_32_bits(RB_EOPTIMESTAMP(device, rb)); + cmds[index++] = upper_32_bits(RB_EOPTIMESTAMP(device, rb)); + cmds[index++] = rb->timestamp; + } else { + cmds[index++] = cp_type7_packet(CP_EVENT_WRITE, 4); + cmds[index++] = CACHE_FLUSH_TS | (1 << 31); + cmds[index++] = lower_32_bits(CTXT_EOPTIMESTAMP(device, + drawctxt)); + cmds[index++] = upper_32_bits(CTXT_EOPTIMESTAMP(device, + drawctxt)); + cmds[index++] = timestamp; + + cmds[index++] = cp_type7_packet(CP_EVENT_WRITE, 4); + cmds[index++] = CACHE_FLUSH_TS; + cmds[index++] = lower_32_bits(RB_EOPTIMESTAMP(device, rb)); + cmds[index++] = upper_32_bits(RB_EOPTIMESTAMP(device, rb)); + cmds[index++] = rb->timestamp; + } + + if (IS_WFI(flags)) + cmds[index++] = cp_type7_packet(CP_WAIT_FOR_IDLE, 0); + + if (IS_SECURE(flags)) { + cmds[index++] = cp_type7_packet(CP_SET_SECURE_MODE, 1); + cmds[index++] = 0; + } + + /* 5 dwords */ + index += a5xx_preemption_post_ibsubmit(adreno_dev, &cmds[index]); + + /* Adjust the thing for the number of bytes we actually wrote */ + rb->_wptr -= (size - index); + + a5xx_ringbuffer_submit(rb, time, + !adreno_is_preemption_enabled(adreno_dev)); + + return 0; +} + +static u32 a5xx_get_alwayson_counter(struct adreno_device *adreno_dev, + u32 *cmds, u64 gpuaddr) +{ + cmds[0] = cp_type7_packet(CP_REG_TO_MEM, 3); + cmds[1] = A5XX_RBBM_ALWAYSON_COUNTER_LO; + + /* On some targets the upper 32 bits are not reliable */ + if (ADRENO_GPUREV(adreno_dev) > ADRENO_REV_A530) + cmds[1] |= (1 << 30) | (2 << 18); + + cmds[2] = lower_32_bits(gpuaddr); + cmds[3] = upper_32_bits(gpuaddr); + + return 4; +} + +/* This is the maximum possible size for 64 bit targets */ +#define PROFILE_IB_DWORDS 4 +#define PROFILE_IB_SLOTS (PAGE_SIZE / (PROFILE_IB_DWORDS << 2)) + +static u64 a5xx_get_user_profiling_ib(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, struct kgsl_drawobj_cmd *cmdobj, + u32 target_offset, u32 *cmds) +{ + u32 offset, *ib, dwords; + u64 gpuaddr; + + if (IS_ERR(rb->profile_desc)) + return 0; + + offset = rb->profile_index * (PROFILE_IB_DWORDS << 2); + ib = rb->profile_desc->hostptr + offset; + gpuaddr = rb->profile_desc->gpuaddr + offset; + dwords = a5xx_get_alwayson_counter(adreno_dev, ib, + cmdobj->profiling_buffer_gpuaddr + target_offset); + + cmds[0] = cp_type7_packet(CP_INDIRECT_BUFFER_PFE, 3); + cmds[1] = lower_32_bits(gpuaddr); + cmds[2] = upper_32_bits(gpuaddr); + cmds[3] = dwords; + + rb->profile_index = (rb->profile_index + 1) % PROFILE_IB_SLOTS; + + return 4; +} + +static int a5xx_rb_context_switch(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, + struct adreno_context *drawctxt) +{ + struct kgsl_pagetable *pagetable = + adreno_drawctxt_get_pagetable(drawctxt); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int count = 0; + u32 cmds[32]; + + if (adreno_drawctxt_get_pagetable(rb->drawctxt_active) != pagetable) + count += a5xx_rb_pagetable_switch(device, drawctxt, + rb, pagetable, cmds); + + cmds[count++] = cp_type7_packet(CP_NOP, 1); + cmds[count++] = CONTEXT_TO_MEM_IDENTIFIER; + + cmds[count++] = cp_type7_packet(CP_MEM_WRITE, 3); + cmds[count++] = lower_32_bits(MEMSTORE_RB_GPU_ADDR(device, rb, + current_context)); + cmds[count++] = upper_32_bits(MEMSTORE_RB_GPU_ADDR(device, rb, + current_context)); + cmds[count++] = drawctxt->base.id; + + cmds[count++] = cp_type7_packet(CP_MEM_WRITE, 3); + cmds[count++] = lower_32_bits(MEMSTORE_ID_GPU_ADDR(device, + KGSL_MEMSTORE_GLOBAL, current_context)); + cmds[count++] = upper_32_bits(MEMSTORE_ID_GPU_ADDR(device, + KGSL_MEMSTORE_GLOBAL, current_context)); + cmds[count++] = drawctxt->base.id; + + cmds[count++] = cp_type4_packet(A5XX_UCHE_INVALIDATE0, 1); + cmds[count++] = 0x12; + + return a5xx_ringbuffer_addcmds(adreno_dev, rb, NULL, F_NOTPROTECTED, + cmds, count, 0, NULL); +} + +static int a5xx_drawctxt_switch(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, + struct adreno_context *drawctxt) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (rb->drawctxt_active == drawctxt) + return 0; + + if (kgsl_context_detached(&drawctxt->base)) + return -ENOENT; + + if (!_kgsl_context_get(&drawctxt->base)) + return -ENOENT; + + trace_adreno_drawctxt_switch(rb, drawctxt); + + a5xx_rb_context_switch(adreno_dev, rb, drawctxt); + + /* Release the current drawctxt as soon as the new one is switched */ + adreno_put_drawctxt_on_timestamp(device, rb->drawctxt_active, + rb, rb->timestamp); + + rb->drawctxt_active = drawctxt; + return 0; +} + + +#define A5XX_USER_PROFILE_IB(dev, rb, cmdobj, cmds, field) \ + a5xx_get_user_profiling_ib((dev), (rb), (cmdobj), \ + offsetof(struct kgsl_drawobj_profiling_buffer, field), \ + (cmds)) + +#define A5XX_KERNEL_PROFILE(dev, cmdobj, cmds, field) \ + a5xx_get_alwayson_counter((dev), (cmds), \ + (dev)->profile_buffer->gpuaddr + \ + ADRENO_DRAWOBJ_PROFILE_OFFSET((cmdobj)->profile_index, \ + field)) + +#define A5XX_COMMAND_DWORDS 32 + +int a5xx_ringbuffer_submitcmd(struct adreno_device *adreno_dev, + struct kgsl_drawobj_cmd *cmdobj, u32 flags, + struct adreno_submit_time *time) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); + struct adreno_context *drawctxt = ADRENO_CONTEXT(drawobj->context); + struct adreno_ringbuffer *rb = drawctxt->rb; + int ret = 0, numibs = 0, index = 0; + u32 *cmds; + + /* Count the number of IBs (if we are not skipping) */ + if (!IS_SKIP(flags)) { + struct list_head *tmp; + + list_for_each(tmp, &cmdobj->cmdlist) + numibs++; + } + + cmds = kmalloc((A5XX_COMMAND_DWORDS + (numibs * 5)) << 2, GFP_KERNEL); + if (!cmds) { + ret = -ENOMEM; + goto done; + } + + cmds[index++] = cp_type7_packet(CP_NOP, 1); + cmds[index++] = START_IB_IDENTIFIER; + + /* Kernel profiling: 4 dwords */ + if (IS_KERNEL_PROFILE(flags)) + index += A5XX_KERNEL_PROFILE(adreno_dev, cmdobj, &cmds[index], + started); + + /* User profiling: 4 dwords */ + if (IS_USER_PROFILE(flags)) + index += A5XX_USER_PROFILE_IB(adreno_dev, rb, cmdobj, + &cmds[index], gpu_ticks_submitted); + + if (numibs) { + struct kgsl_memobj_node *ib; + + list_for_each_entry(ib, &cmdobj->cmdlist, node) { + if (ib->priv & MEMOBJ_SKIP || + (ib->flags & KGSL_CMDLIST_CTXTSWITCH_PREAMBLE + && !IS_PREAMBLE(flags))) + cmds[index++] = cp_type7_packet(CP_NOP, 4); + + cmds[index++] = + cp_type7_packet(CP_INDIRECT_BUFFER_PFE, 3); + cmds[index++] = lower_32_bits(ib->gpuaddr); + cmds[index++] = upper_32_bits(ib->gpuaddr); + + /* Double check that IB_PRIV is never set */ + cmds[index++] = (ib->size >> 2) & 0xfffff; + } + } + + /* + * SRM -- set render mode (ex binning, direct render etc) + * SRM is set by UMD usually at start of IB to tell CP the type of + * preemption. + * KMD needs to set SRM to NULL to indicate CP that rendering is + * done by IB. + */ + cmds[index++] = cp_type7_packet(CP_SET_RENDER_MODE, 5); + cmds[index++] = 0; + cmds[index++] = 0; + cmds[index++] = 0; + cmds[index++] = 0; + cmds[index++] = 0; + + cmds[index++] = cp_type7_packet(CP_YIELD_ENABLE, 1); + cmds[index++] = 1; + + /* 4 dwords */ + if (IS_KERNEL_PROFILE(flags)) + index += A5XX_KERNEL_PROFILE(adreno_dev, cmdobj, &cmds[index], + retired); + + /* 4 dwords */ + if (IS_USER_PROFILE(flags)) + index += A5XX_USER_PROFILE_IB(adreno_dev, rb, cmdobj, + &cmds[index], gpu_ticks_retired); + + cmds[index++] = cp_type7_packet(CP_NOP, 1); + cmds[index++] = END_IB_IDENTIFIER; + + ret = a5xx_drawctxt_switch(adreno_dev, rb, drawctxt); + + /* + * In the unlikely event of an error in the drawctxt switch, + * treat it like a hang + */ + if (ret) { + /* + * It is "normal" to get a -ENOSPC or a -ENOENT. Don't log it, + * the upper layers know how to handle it + */ + if (ret != -ENOSPC && ret != -ENOENT) + dev_err(device->dev, + "Unable to switch draw context: %d\n", + ret); + goto done; + } + + adreno_drawobj_set_constraint(device, drawobj); + + ret = a5xx_ringbuffer_addcmds(adreno_dev, drawctxt->rb, drawctxt, + flags, cmds, index, drawobj->timestamp, time); + +done: + trace_kgsl_issueibcmds(device, drawctxt->base.id, numibs, + drawobj->timestamp, drawobj->flags, ret, drawctxt->type); + + kfree(cmds); + return ret; +} diff --git a/qcom/opensource/graphics-kernel/adreno_a5xx_snapshot.c b/qcom/opensource/graphics-kernel/adreno_a5xx_snapshot.c new file mode 100644 index 0000000000..232319a61e --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a5xx_snapshot.c @@ -0,0 +1,1219 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2015-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include "adreno.h" +#include "adreno_a5xx.h" +#include "adreno_snapshot.h" + +enum a5xx_rbbm_debbus_id { + A5XX_RBBM_DBGBUS_CP = 0x1, + A5XX_RBBM_DBGBUS_RBBM = 0x2, + A5XX_RBBM_DBGBUS_VBIF = 0x3, + A5XX_RBBM_DBGBUS_HLSQ = 0x4, + A5XX_RBBM_DBGBUS_UCHE = 0x5, + A5XX_RBBM_DBGBUS_DPM = 0x6, + A5XX_RBBM_DBGBUS_TESS = 0x7, + A5XX_RBBM_DBGBUS_PC = 0x8, + A5XX_RBBM_DBGBUS_VFDP = 0x9, + A5XX_RBBM_DBGBUS_VPC = 0xa, + A5XX_RBBM_DBGBUS_TSE = 0xb, + A5XX_RBBM_DBGBUS_RAS = 0xc, + A5XX_RBBM_DBGBUS_VSC = 0xd, + A5XX_RBBM_DBGBUS_COM = 0xe, + A5XX_RBBM_DBGBUS_DCOM = 0xf, + A5XX_RBBM_DBGBUS_LRZ = 0x10, + A5XX_RBBM_DBGBUS_A2D_DSP = 0x11, + A5XX_RBBM_DBGBUS_CCUFCHE = 0x12, + A5XX_RBBM_DBGBUS_GPMU = 0x13, + A5XX_RBBM_DBGBUS_RBP = 0x14, + A5XX_RBBM_DBGBUS_HM = 0x15, + A5XX_RBBM_DBGBUS_RBBM_CFG = 0x16, + A5XX_RBBM_DBGBUS_VBIF_CX = 0x17, + A5XX_RBBM_DBGBUS_GPC = 0x1d, + A5XX_RBBM_DBGBUS_LARC = 0x1e, + A5XX_RBBM_DBGBUS_HLSQ_SPTP = 0x1f, + A5XX_RBBM_DBGBUS_RB_0 = 0x20, + A5XX_RBBM_DBGBUS_RB_1 = 0x21, + A5XX_RBBM_DBGBUS_RB_2 = 0x22, + A5XX_RBBM_DBGBUS_RB_3 = 0x23, + A5XX_RBBM_DBGBUS_CCU_0 = 0x28, + A5XX_RBBM_DBGBUS_CCU_1 = 0x29, + A5XX_RBBM_DBGBUS_CCU_2 = 0x2a, + A5XX_RBBM_DBGBUS_CCU_3 = 0x2b, + A5XX_RBBM_DBGBUS_A2D_RAS_0 = 0x30, + A5XX_RBBM_DBGBUS_A2D_RAS_1 = 0x31, + A5XX_RBBM_DBGBUS_A2D_RAS_2 = 0x32, + A5XX_RBBM_DBGBUS_A2D_RAS_3 = 0x33, + A5XX_RBBM_DBGBUS_VFD_0 = 0x38, + A5XX_RBBM_DBGBUS_VFD_1 = 0x39, + A5XX_RBBM_DBGBUS_VFD_2 = 0x3a, + A5XX_RBBM_DBGBUS_VFD_3 = 0x3b, + A5XX_RBBM_DBGBUS_SP_0 = 0x40, + A5XX_RBBM_DBGBUS_SP_1 = 0x41, + A5XX_RBBM_DBGBUS_SP_2 = 0x42, + A5XX_RBBM_DBGBUS_SP_3 = 0x43, + A5XX_RBBM_DBGBUS_TPL1_0 = 0x48, + A5XX_RBBM_DBGBUS_TPL1_1 = 0x49, + A5XX_RBBM_DBGBUS_TPL1_2 = 0x4a, + A5XX_RBBM_DBGBUS_TPL1_3 = 0x4b +}; + +static const struct adreno_debugbus_block a5xx_debugbus_blocks[] = { + { A5XX_RBBM_DBGBUS_CP, 0x100, }, + { A5XX_RBBM_DBGBUS_RBBM, 0x100, }, + { A5XX_RBBM_DBGBUS_VBIF, 0x100, }, + { A5XX_RBBM_DBGBUS_HLSQ, 0x100, }, + { A5XX_RBBM_DBGBUS_UCHE, 0x100, }, + { A5XX_RBBM_DBGBUS_DPM, 0x100, }, + { A5XX_RBBM_DBGBUS_TESS, 0x100, }, + { A5XX_RBBM_DBGBUS_PC, 0x100, }, + { A5XX_RBBM_DBGBUS_VFDP, 0x100, }, + { A5XX_RBBM_DBGBUS_VPC, 0x100, }, + { A5XX_RBBM_DBGBUS_TSE, 0x100, }, + { A5XX_RBBM_DBGBUS_RAS, 0x100, }, + { A5XX_RBBM_DBGBUS_VSC, 0x100, }, + { A5XX_RBBM_DBGBUS_COM, 0x100, }, + { A5XX_RBBM_DBGBUS_DCOM, 0x100, }, + { A5XX_RBBM_DBGBUS_LRZ, 0x100, }, + { A5XX_RBBM_DBGBUS_A2D_DSP, 0x100, }, + { A5XX_RBBM_DBGBUS_CCUFCHE, 0x100, }, + { A5XX_RBBM_DBGBUS_GPMU, 0x100, }, + { A5XX_RBBM_DBGBUS_RBP, 0x100, }, + { A5XX_RBBM_DBGBUS_HM, 0x100, }, + { A5XX_RBBM_DBGBUS_RBBM_CFG, 0x100, }, + { A5XX_RBBM_DBGBUS_VBIF_CX, 0x100, }, + { A5XX_RBBM_DBGBUS_GPC, 0x100, }, + { A5XX_RBBM_DBGBUS_LARC, 0x100, }, + { A5XX_RBBM_DBGBUS_HLSQ_SPTP, 0x100, }, + { A5XX_RBBM_DBGBUS_RB_0, 0x100, }, + { A5XX_RBBM_DBGBUS_RB_1, 0x100, }, + { A5XX_RBBM_DBGBUS_RB_2, 0x100, }, + { A5XX_RBBM_DBGBUS_RB_3, 0x100, }, + { A5XX_RBBM_DBGBUS_CCU_0, 0x100, }, + { A5XX_RBBM_DBGBUS_CCU_1, 0x100, }, + { A5XX_RBBM_DBGBUS_CCU_2, 0x100, }, + { A5XX_RBBM_DBGBUS_CCU_3, 0x100, }, + { A5XX_RBBM_DBGBUS_A2D_RAS_0, 0x100, }, + { A5XX_RBBM_DBGBUS_A2D_RAS_1, 0x100, }, + { A5XX_RBBM_DBGBUS_A2D_RAS_2, 0x100, }, + { A5XX_RBBM_DBGBUS_A2D_RAS_3, 0x100, }, + { A5XX_RBBM_DBGBUS_VFD_0, 0x100, }, + { A5XX_RBBM_DBGBUS_VFD_1, 0x100, }, + { A5XX_RBBM_DBGBUS_VFD_2, 0x100, }, + { A5XX_RBBM_DBGBUS_VFD_3, 0x100, }, + { A5XX_RBBM_DBGBUS_SP_0, 0x100, }, + { A5XX_RBBM_DBGBUS_SP_1, 0x100, }, + { A5XX_RBBM_DBGBUS_SP_2, 0x100, }, + { A5XX_RBBM_DBGBUS_SP_3, 0x100, }, + { A5XX_RBBM_DBGBUS_TPL1_0, 0x100, }, + { A5XX_RBBM_DBGBUS_TPL1_1, 0x100, }, + { A5XX_RBBM_DBGBUS_TPL1_2, 0x100, }, + { A5XX_RBBM_DBGBUS_TPL1_3, 0x100, }, +}; + +#define A5XX_NUM_AXI_ARB_BLOCKS 2 +#define A5XX_NUM_XIN_BLOCKS 4 + +/* Width of A5XX_CP_DRAW_STATE_ADDR is 8 bits */ +#define A5XX_CP_DRAW_STATE_ADDR_WIDTH 8 + +/* a5xx_snapshot_cp_pm4() - Dump PM4 data in snapshot */ +static size_t a5xx_snapshot_cp_pm4(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4); + size_t size = fw->size; + + if (remain < DEBUG_SECTION_SZ(size)) { + SNAPSHOT_ERR_NOMEM(device, "CP PM4 RAM DEBUG"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_CP_PM4_RAM; + header->size = size; + + memcpy(data, fw->memdesc->hostptr, size * sizeof(uint32_t)); + + return DEBUG_SECTION_SZ(size); +} + +/* a5xx_snapshot_cp_pfp() - Dump the PFP data on snapshot */ +static size_t a5xx_snapshot_cp_pfp(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); + int size = fw->size; + + if (remain < DEBUG_SECTION_SZ(size)) { + SNAPSHOT_ERR_NOMEM(device, "CP PFP RAM DEBUG"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_CP_PFP_RAM; + header->size = size; + + memcpy(data, fw->memdesc->hostptr, size * sizeof(uint32_t)); + + return DEBUG_SECTION_SZ(size); +} + +/* a5xx_rbbm_debug_bus_read() - Read data from trace bus */ +static void a5xx_rbbm_debug_bus_read(struct kgsl_device *device, + unsigned int block_id, unsigned int index, unsigned int *val) +{ + unsigned int reg; + + reg = (block_id << A5XX_RBBM_CFG_DBGBUS_SEL_PING_BLK_SEL_SHIFT) | + (index << A5XX_RBBM_CFG_DBGBUS_SEL_PING_INDEX_SHIFT); + kgsl_regwrite(device, A5XX_RBBM_CFG_DBGBUS_SEL_A, reg); + kgsl_regwrite(device, A5XX_RBBM_CFG_DBGBUS_SEL_B, reg); + kgsl_regwrite(device, A5XX_RBBM_CFG_DBGBUS_SEL_C, reg); + kgsl_regwrite(device, A5XX_RBBM_CFG_DBGBUS_SEL_D, reg); + + kgsl_regread(device, A5XX_RBBM_CFG_DBGBUS_TRACE_BUF2, val); + val++; + kgsl_regread(device, A5XX_RBBM_CFG_DBGBUS_TRACE_BUF1, val); + +} + +/* a5xx_snapshot_vbif_debugbus() - Dump the VBIF debug data */ +static size_t a5xx_snapshot_vbif_debugbus(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_debugbus *header = + (struct kgsl_snapshot_debugbus *)buf; + struct adreno_debugbus_block *block = priv; + int i, j; + /* + * Total number of VBIF data words considering 3 sections: + * 2 arbiter blocks of 16 words + * 4 AXI XIN blocks of 18 dwords each + * 4 core clock side XIN blocks of 12 dwords each + */ + unsigned int dwords = (16 * A5XX_NUM_AXI_ARB_BLOCKS) + + (18 * A5XX_NUM_XIN_BLOCKS) + (12 * A5XX_NUM_XIN_BLOCKS); + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + size_t size; + unsigned int reg_clk; + + size = (dwords * sizeof(unsigned int)) + sizeof(*header); + + if (remain < size) { + SNAPSHOT_ERR_NOMEM(device, "DEBUGBUS"); + return 0; + } + header->id = block->block_id; + header->count = dwords; + + kgsl_regread(device, A5XX_VBIF_CLKON, ®_clk); + kgsl_regwrite(device, A5XX_VBIF_CLKON, reg_clk | + (A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_MASK << + A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_SHIFT)); + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS1_CTRL0, 0); + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS_OUT_CTRL, + (A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_MASK << + A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_SHIFT)); + for (i = 0; i < A5XX_NUM_AXI_ARB_BLOCKS; i++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS2_CTRL0, + (1 << (i + 16))); + for (j = 0; j < 16; j++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS2_CTRL1, + ((j & A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK) + << A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT)); + kgsl_regread(device, A5XX_VBIF_TEST_BUS_OUT, + data); + data++; + } + } + + /* XIN blocks AXI side */ + for (i = 0; i < A5XX_NUM_XIN_BLOCKS; i++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS2_CTRL0, 1 << i); + for (j = 0; j < 18; j++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS2_CTRL1, + ((j & A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK) + << A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT)); + kgsl_regread(device, A5XX_VBIF_TEST_BUS_OUT, + data); + data++; + } + } + + /* XIN blocks core clock side */ + for (i = 0; i < A5XX_NUM_XIN_BLOCKS; i++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS1_CTRL0, 1 << i); + for (j = 0; j < 12; j++) { + kgsl_regwrite(device, A5XX_VBIF_TEST_BUS1_CTRL1, + ((j & A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_MASK) + << A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_SHIFT)); + kgsl_regread(device, A5XX_VBIF_TEST_BUS_OUT, + data); + data++; + } + } + /* restore the clock of VBIF */ + kgsl_regwrite(device, A5XX_VBIF_CLKON, reg_clk); + return size; +} + +/* a5xx_snapshot_debugbus_block() - Capture debug data for a gpu block */ +static size_t a5xx_snapshot_debugbus_block(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_debugbus *header = + (struct kgsl_snapshot_debugbus *)buf; + struct adreno_debugbus_block *block = priv; + int i; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + unsigned int dwords; + size_t size; + + dwords = block->dwords; + + /* For a5xx each debug bus data unit is 2 DWRODS */ + size = (dwords * sizeof(unsigned int) * 2) + sizeof(*header); + + if (remain < size) { + SNAPSHOT_ERR_NOMEM(device, "DEBUGBUS"); + return 0; + } + + header->id = block->block_id; + header->count = dwords * 2; + + for (i = 0; i < dwords; i++) + a5xx_rbbm_debug_bus_read(device, block->block_id, i, + &data[i*2]); + + return size; +} + +/* a5xx_snapshot_debugbus() - Capture debug bus data */ +static void a5xx_snapshot_debugbus(struct kgsl_device *device, + struct kgsl_snapshot *snapshot) +{ + int i; + + kgsl_regwrite(device, A5XX_RBBM_CFG_DBGBUS_CNTLM, + 0xf << A5XX_RBBM_CFG_DEBBUS_CTLTM_ENABLE_SHIFT); + + for (i = 0; i < ARRAY_SIZE(a5xx_debugbus_blocks); i++) { + if (a5xx_debugbus_blocks[i].block_id == A5XX_RBBM_DBGBUS_VBIF) + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_DEBUGBUS, + snapshot, a5xx_snapshot_vbif_debugbus, + (void *) &a5xx_debugbus_blocks[i]); + else + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_DEBUGBUS, + snapshot, a5xx_snapshot_debugbus_block, + (void *) &a5xx_debugbus_blocks[i]); + } +} + +static const unsigned int a5xx_vbif_registers[] = { + 0x3000, 0x3007, 0x300C, 0x3014, 0x3018, 0x302C, 0x3030, 0x3030, + 0x3034, 0x3036, 0x3038, 0x3038, 0x303C, 0x303D, 0x3040, 0x3040, + 0x3042, 0x3042, 0x3049, 0x3049, 0x3058, 0x3058, 0x305A, 0x3061, + 0x3064, 0x3068, 0x306C, 0x306D, 0x3080, 0x3088, 0x308C, 0x308C, + 0x3090, 0x3094, 0x3098, 0x3098, 0x309C, 0x309C, 0x30C0, 0x30C0, + 0x30C8, 0x30C8, 0x30D0, 0x30D0, 0x30D8, 0x30D8, 0x30E0, 0x30E0, + 0x3100, 0x3100, 0x3108, 0x3108, 0x3110, 0x3110, 0x3118, 0x3118, + 0x3120, 0x3120, 0x3124, 0x3125, 0x3129, 0x3129, 0x3131, 0x3131, + 0x340C, 0x340C, 0x3410, 0x3410, 0x3800, 0x3801, +}; + +/* + * Set of registers to dump for A5XX on snapshot. + * Registers in pairs - first value is the start offset, second + * is the stop offset (inclusive) + */ + +static const unsigned int a5xx_registers[] = { + /* RBBM */ + 0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B, + 0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095, + 0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3, + 0x04E0, 0x04F4, 0X04F8, 0x0529, 0x0531, 0x0533, 0x0540, 0x0555, + 0xF400, 0xF400, 0xF800, 0xF807, + /* CP */ + 0x0800, 0x0803, 0x0806, 0x081A, 0x081F, 0x0841, 0x0860, 0x0860, + 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0X0B1C, 0X0B1E, 0x0B28, + 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, + /* VSC */ + 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53, 0x0C60, 0x0C61, + /* GRAS */ + 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98, 0x0CA0, 0x0CA0, + 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585, + /* RB */ + 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8, + 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E, + 0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545, + /* PC */ + 0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0, + 0x24C0, 0x24C0, + /* VFD */ + 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57, + /* VPC */ + 0x0E60, 0x0E7C, + /* UCHE */ + 0x0E80, 0x0E8F, 0x0E90, 0x0E96, 0xEA0, 0xEA8, 0xEB0, 0xEB2, + + /* RB CTX 0 */ + 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9, 0xE1B0, 0xE1B6, + 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201, 0xE210, 0xE21C, + 0xE240, 0xE268, + /* GRAS CTX 0 */ + 0xE000, 0xE006, 0xE010, 0xE09A, 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, + 0xE100, 0xE105, + /* PC CTX 0 */ + 0xE380, 0xE38F, 0xE3B0, 0xE3B0, + /* VFD CTX 0 */ + 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0, + /* VPC CTX 0 */ + 0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, + + /* RB CTX 1 */ + 0xE940, 0xE947, 0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, + 0xE9C0, 0xE9C7, 0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, + 0xEA40, 0xEA68, + /* GRAS CTX 1 */ + 0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB, + 0xE900, 0xE905, + /* PC CTX 1 */ + 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, + /* VFD CTX 1 */ + 0xEC00, 0xEC05, 0xEC08, 0xECE9, 0xECF0, 0xECF0, + /* VPC CTX 1 */ + 0xEA80, 0xEA80, 0xEA82, 0xEAA3, 0xEAA5, 0xEAC2, +}; + +/* + * GPMU registers to dump for A5XX on snapshot. + * Registers in pairs - first value is the start offset, second + * is the stop offset (inclusive) + */ + +static const unsigned int a5xx_gpmu_registers[] = { + /* GPMU */ + 0xA800, 0xA8FF, 0xAC60, 0xAC60, +}; + +/* + * Set of registers to dump for A5XX before actually triggering crash dumper. + * Registers in pairs - first value is the start offset, second + * is the stop offset (inclusive) + */ +static const unsigned int a5xx_pre_crashdumper_registers[] = { + /* RBBM: RBBM_STATUS - RBBM_STATUS3 */ + 0x04F5, 0x04F7, 0x0530, 0x0530, + /* CP: CP_STATUS_1 */ + 0x0B1D, 0x0B1D, +}; + + +struct a5xx_hlsq_sp_tp_regs { + unsigned int statetype; + unsigned int ahbaddr; + unsigned int size; + uint64_t offset; +}; + +static struct a5xx_hlsq_sp_tp_regs a5xx_hlsq_sp_tp_registers[] = { + /* HSLQ non context. 0xe32 - 0xe3f are holes so don't include them */ + { 0x35, 0xE00, 0x32 }, + /* HLSQ CTX 0 2D */ + { 0x31, 0x2080, 0x1 }, + /* HLSQ CTX 1 2D */ + { 0x33, 0x2480, 0x1 }, + /* HLSQ CTX 0 3D. 0xe7e2 - 0xe7ff are holes so don't include them */ + { 0x32, 0xE780, 0x62 }, + /* HLSQ CTX 1 3D. 0xefe2 - 0xefff are holes so don't include them */ + { 0x34, 0xEF80, 0x62 }, + + /* SP non context */ + { 0x3f, 0x0EC0, 0x40 }, + /* SP CTX 0 2D */ + { 0x3d, 0x2040, 0x1 }, + /* SP CTX 1 2D */ + { 0x3b, 0x2440, 0x1 }, + /* SP CTX 0 3D */ + { 0x3e, 0xE580, 0x180 }, + /* SP CTX 1 3D */ + { 0x3c, 0xED80, 0x180 }, + + /* TP non context. 0x0f1c - 0x0f3f are holes so don't include them */ + { 0x3a, 0x0F00, 0x1c }, + /* TP CTX 0 2D. 0x200a - 0x200f are holes so don't include them */ + { 0x38, 0x2000, 0xa }, + /* TP CTX 1 2D. 0x240a - 0x240f are holes so don't include them */ + { 0x36, 0x2400, 0xa }, + /* TP CTX 0 3D */ + { 0x39, 0xE700, 0x80 }, + /* TP CTX 1 3D */ + { 0x37, 0xEF00, 0x80 }, +}; + + +#define A5XX_NUM_SHADER_BANKS 4 +#define A5XX_SHADER_STATETYPE_SHIFT 8 + +enum a5xx_shader_obj { + A5XX_TP_W_MEMOBJ = 1, + A5XX_TP_W_SAMPLER = 2, + A5XX_TP_W_MIPMAP_BASE = 3, + A5XX_TP_W_MEMOBJ_TAG = 4, + A5XX_TP_W_SAMPLER_TAG = 5, + A5XX_TP_S_3D_MEMOBJ = 6, + A5XX_TP_S_3D_SAMPLER = 0x7, + A5XX_TP_S_3D_MEMOBJ_TAG = 0x8, + A5XX_TP_S_3D_SAMPLER_TAG = 0x9, + A5XX_TP_S_CS_MEMOBJ = 0xA, + A5XX_TP_S_CS_SAMPLER = 0xB, + A5XX_TP_S_CS_MEMOBJ_TAG = 0xC, + A5XX_TP_S_CS_SAMPLER_TAG = 0xD, + A5XX_SP_W_INSTR = 0xE, + A5XX_SP_W_CONST = 0xF, + A5XX_SP_W_UAV_SIZE = 0x10, + A5XX_SP_W_CB_SIZE = 0x11, + A5XX_SP_W_UAV_BASE = 0x12, + A5XX_SP_W_CB_BASE = 0x13, + A5XX_SP_W_INST_TAG = 0x14, + A5XX_SP_W_STATE = 0x15, + A5XX_SP_S_3D_INSTR = 0x16, + A5XX_SP_S_3D_CONST = 0x17, + A5XX_SP_S_3D_CB_BASE = 0x18, + A5XX_SP_S_3D_CB_SIZE = 0x19, + A5XX_SP_S_3D_UAV_BASE = 0x1A, + A5XX_SP_S_3D_UAV_SIZE = 0x1B, + A5XX_SP_S_CS_INSTR = 0x1C, + A5XX_SP_S_CS_CONST = 0x1D, + A5XX_SP_S_CS_CB_BASE = 0x1E, + A5XX_SP_S_CS_CB_SIZE = 0x1F, + A5XX_SP_S_CS_UAV_BASE = 0x20, + A5XX_SP_S_CS_UAV_SIZE = 0x21, + A5XX_SP_S_3D_INSTR_DIRTY = 0x22, + A5XX_SP_S_3D_CONST_DIRTY = 0x23, + A5XX_SP_S_3D_CB_BASE_DIRTY = 0x24, + A5XX_SP_S_3D_CB_SIZE_DIRTY = 0x25, + A5XX_SP_S_3D_UAV_BASE_DIRTY = 0x26, + A5XX_SP_S_3D_UAV_SIZE_DIRTY = 0x27, + A5XX_SP_S_CS_INSTR_DIRTY = 0x28, + A5XX_SP_S_CS_CONST_DIRTY = 0x29, + A5XX_SP_S_CS_CB_BASE_DIRTY = 0x2A, + A5XX_SP_S_CS_CB_SIZE_DIRTY = 0x2B, + A5XX_SP_S_CS_UAV_BASE_DIRTY = 0x2C, + A5XX_SP_S_CS_UAV_SIZE_DIRTY = 0x2D, + A5XX_HLSQ_ICB = 0x2E, + A5XX_HLSQ_ICB_DIRTY = 0x2F, + A5XX_HLSQ_ICB_CB_BASE_DIRTY = 0x30, + A5XX_SP_POWER_RESTORE_RAM = 0x40, + A5XX_SP_POWER_RESTORE_RAM_TAG = 0x41, + A5XX_TP_POWER_RESTORE_RAM = 0x42, + A5XX_TP_POWER_RESTORE_RAM_TAG = 0x43, + +}; + +struct a5xx_shader_block { + unsigned int statetype; + unsigned int sz; + uint64_t offset; +}; + +struct a5xx_shader_block_info { + struct a5xx_shader_block *block; + unsigned int bank; + uint64_t offset; +}; + +static struct a5xx_shader_block a5xx_shader_blocks[] = { + {A5XX_TP_W_MEMOBJ, 0x200}, + {A5XX_TP_W_MIPMAP_BASE, 0x3C0}, + {A5XX_TP_W_SAMPLER_TAG, 0x40}, + {A5XX_TP_S_3D_SAMPLER, 0x80}, + {A5XX_TP_S_3D_SAMPLER_TAG, 0x20}, + {A5XX_TP_S_CS_SAMPLER, 0x40}, + {A5XX_TP_S_CS_SAMPLER_TAG, 0x10}, + {A5XX_SP_W_CONST, 0x800}, + {A5XX_SP_W_CB_SIZE, 0x30}, + {A5XX_SP_W_CB_BASE, 0xF0}, + {A5XX_SP_W_STATE, 0x1}, + {A5XX_SP_S_3D_CONST, 0x800}, + {A5XX_SP_S_3D_CB_SIZE, 0x28}, + {A5XX_SP_S_3D_UAV_SIZE, 0x80}, + {A5XX_SP_S_CS_CONST, 0x400}, + {A5XX_SP_S_CS_CB_SIZE, 0x8}, + {A5XX_SP_S_CS_UAV_SIZE, 0x80}, + {A5XX_SP_S_3D_CONST_DIRTY, 0x12}, + {A5XX_SP_S_3D_CB_SIZE_DIRTY, 0x1}, + {A5XX_SP_S_3D_UAV_SIZE_DIRTY, 0x2}, + {A5XX_SP_S_CS_CONST_DIRTY, 0xA}, + {A5XX_SP_S_CS_CB_SIZE_DIRTY, 0x1}, + {A5XX_SP_S_CS_UAV_SIZE_DIRTY, 0x2}, + {A5XX_HLSQ_ICB_DIRTY, 0xB}, + {A5XX_SP_POWER_RESTORE_RAM_TAG, 0xA}, + {A5XX_TP_POWER_RESTORE_RAM_TAG, 0xA}, + {A5XX_TP_W_SAMPLER, 0x80}, + {A5XX_TP_W_MEMOBJ_TAG, 0x40}, + {A5XX_TP_S_3D_MEMOBJ, 0x200}, + {A5XX_TP_S_3D_MEMOBJ_TAG, 0x20}, + {A5XX_TP_S_CS_MEMOBJ, 0x100}, + {A5XX_TP_S_CS_MEMOBJ_TAG, 0x10}, + {A5XX_SP_W_INSTR, 0x800}, + {A5XX_SP_W_UAV_SIZE, 0x80}, + {A5XX_SP_W_UAV_BASE, 0x80}, + {A5XX_SP_W_INST_TAG, 0x40}, + {A5XX_SP_S_3D_INSTR, 0x800}, + {A5XX_SP_S_3D_CB_BASE, 0xC8}, + {A5XX_SP_S_3D_UAV_BASE, 0x80}, + {A5XX_SP_S_CS_INSTR, 0x400}, + {A5XX_SP_S_CS_CB_BASE, 0x28}, + {A5XX_SP_S_CS_UAV_BASE, 0x80}, + {A5XX_SP_S_3D_INSTR_DIRTY, 0x1}, + {A5XX_SP_S_3D_CB_BASE_DIRTY, 0x5}, + {A5XX_SP_S_3D_UAV_BASE_DIRTY, 0x2}, + {A5XX_SP_S_CS_INSTR_DIRTY, 0x1}, + {A5XX_SP_S_CS_CB_BASE_DIRTY, 0x1}, + {A5XX_SP_S_CS_UAV_BASE_DIRTY, 0x2}, + {A5XX_HLSQ_ICB, 0x200}, + {A5XX_HLSQ_ICB_CB_BASE_DIRTY, 0x4}, + {A5XX_SP_POWER_RESTORE_RAM, 0x140}, + {A5XX_TP_POWER_RESTORE_RAM, 0x40}, +}; + +static struct kgsl_memdesc *capturescript; +static struct kgsl_memdesc *registers; +static bool crash_dump_valid; + +static size_t a5xx_snapshot_shader_memory(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_shader *header = + (struct kgsl_snapshot_shader *) buf; + struct a5xx_shader_block_info *info = + (struct a5xx_shader_block_info *) priv; + struct a5xx_shader_block *block = info->block; + unsigned int *data = (unsigned int *) (buf + sizeof(*header)); + + if (remain < SHADER_SECTION_SZ(block->sz)) { + SNAPSHOT_ERR_NOMEM(device, "SHADER MEMORY"); + return 0; + } + + header->type = block->statetype; + header->index = info->bank; + header->size = block->sz; + + memcpy(data, registers->hostptr + info->offset, + block->sz * sizeof(unsigned int)); + + return SHADER_SECTION_SZ(block->sz); +} + +static void a5xx_snapshot_shader(struct kgsl_device *device, + struct kgsl_snapshot *snapshot) +{ + unsigned int i, j; + struct a5xx_shader_block_info info; + + /* Shader blocks can only be read by the crash dumper */ + if (!crash_dump_valid) + return; + + for (i = 0; i < ARRAY_SIZE(a5xx_shader_blocks); i++) { + for (j = 0; j < A5XX_NUM_SHADER_BANKS; j++) { + info.block = &a5xx_shader_blocks[i]; + info.bank = j; + info.offset = a5xx_shader_blocks[i].offset + + (j * a5xx_shader_blocks[i].sz); + + /* Shader working/shadow memory */ + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_SHADER, + snapshot, a5xx_snapshot_shader_memory, &info); + } + } +} + +/* Dump registers which get affected by crash dumper trigger */ +static size_t a5xx_snapshot_pre_crashdump_regs(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_registers pre_cdregs = { + .regs = a5xx_pre_crashdumper_registers, + .count = ARRAY_SIZE(a5xx_pre_crashdumper_registers)/2, + }; + + return kgsl_snapshot_dump_registers(device, buf, remain, &pre_cdregs); +} + +struct registers { + const unsigned int *regs; + size_t size; +}; + +static size_t a5xx_legacy_snapshot_registers(struct kgsl_device *device, + u8 *buf, size_t remain, const unsigned int *regs, size_t size) +{ + struct kgsl_snapshot_registers snapshot_regs = { + .regs = regs, + .count = size / 2, + }; + + return kgsl_snapshot_dump_registers(device, buf, remain, + &snapshot_regs); +} + +#define REG_PAIR_COUNT(_a, _i) \ + (((_a)[(2 * (_i)) + 1] - (_a)[2 * (_i)]) + 1) + +static size_t a5xx_snapshot_registers(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct kgsl_snapshot_regs *header = (struct kgsl_snapshot_regs *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + unsigned int *src = (unsigned int *) registers->hostptr; + struct registers *regs = (struct registers *)priv; + unsigned int j, k; + unsigned int count = 0; + + if (!crash_dump_valid) + return a5xx_legacy_snapshot_registers(device, buf, remain, + regs->regs, regs->size); + + if (remain < sizeof(*header)) { + SNAPSHOT_ERR_NOMEM(device, "REGISTERS"); + return 0; + } + + remain -= sizeof(*header); + + for (j = 0; j < regs->size / 2; j++) { + unsigned int start = regs->regs[2 * j]; + unsigned int end = regs->regs[(2 * j) + 1]; + + if (remain < ((end - start) + 1) * 8) { + SNAPSHOT_ERR_NOMEM(device, "REGISTERS"); + goto out; + } + + remain -= ((end - start) + 1) * 8; + + for (k = start; k <= end; k++, count++) { + *data++ = k; + *data++ = *src++; + } + } + +out: + header->count = count; + + /* Return the size of the section */ + return (count * 8) + sizeof(*header); +} + +/* Snapshot a preemption record buffer */ +static size_t snapshot_preemption_record(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct kgsl_memdesc *memdesc = priv; + + struct kgsl_snapshot_gpu_object_v2 *header = + (struct kgsl_snapshot_gpu_object_v2 *)buf; + + u8 *ptr = buf + sizeof(*header); + + if (remain < (SZ_64K + sizeof(*header))) { + SNAPSHOT_ERR_NOMEM(device, "PREEMPTION RECORD"); + return 0; + } + + header->size = SZ_64K >> 2; + header->gpuaddr = memdesc->gpuaddr; + header->ptbase = + kgsl_mmu_pagetable_get_ttbr0(device->mmu.defaultpagetable); + header->type = SNAPSHOT_GPU_OBJECT_GLOBAL; + + memcpy(ptr, memdesc->hostptr, SZ_64K); + + return SZ_64K + sizeof(*header); +} + + +static void _a5xx_do_crashdump(struct kgsl_device *device) +{ + unsigned long wait_time; + unsigned int reg = 0; + + crash_dump_valid = false; + + if (!device->snapshot_crashdumper) + return; + + if (IS_ERR_OR_NULL(capturescript) || IS_ERR_OR_NULL(registers)) + return; + + /* IF the SMMU is stalled we cannot do a crash dump */ + if (adreno_smmu_is_stalled(ADRENO_DEVICE(device))) + return; + + /* Turn on APRIV so we can access the buffers */ + kgsl_regwrite(device, A5XX_CP_CNTL, 1); + + kgsl_regwrite(device, A5XX_CP_CRASH_SCRIPT_BASE_LO, + lower_32_bits(capturescript->gpuaddr)); + kgsl_regwrite(device, A5XX_CP_CRASH_SCRIPT_BASE_HI, + upper_32_bits(capturescript->gpuaddr)); + kgsl_regwrite(device, A5XX_CP_CRASH_DUMP_CNTL, 1); + + wait_time = jiffies + msecs_to_jiffies(CP_CRASH_DUMPER_TIMEOUT); + while (!time_after(jiffies, wait_time)) { + kgsl_regread(device, A5XX_CP_CRASH_DUMP_CNTL, ®); + if (reg & 0x4) + break; + cpu_relax(); + } + + kgsl_regwrite(device, A5XX_CP_CNTL, 0); + + if (!(reg & 0x4)) { + dev_err(device->dev, "Crash dump timed out: 0x%X\n", reg); + return; + } + + crash_dump_valid = true; +} + +static int get_hlsq_registers(struct kgsl_device *device, + const struct a5xx_hlsq_sp_tp_regs *regs, unsigned int *data) +{ + unsigned int i; + unsigned int *src = registers->hostptr + regs->offset; + + for (i = 0; i < regs->size; i++) { + *data++ = regs->ahbaddr + i; + *data++ = *(src + i); + } + + return (2 * regs->size); +} + +static size_t a5xx_snapshot_dump_hlsq_sp_tp_regs(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_regs *header = (struct kgsl_snapshot_regs *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + int count = 0, i; + + /* Figure out how many registers we are going to dump */ + for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_sp_tp_registers); i++) + count += a5xx_hlsq_sp_tp_registers[i].size; + + if (remain < (count * 8) + sizeof(*header)) { + SNAPSHOT_ERR_NOMEM(device, "REGISTERS"); + return 0; + } + + for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_sp_tp_registers); i++) + data += get_hlsq_registers(device, + &a5xx_hlsq_sp_tp_registers[i], data); + + header->count = count; + + /* Return the size of the section */ + return (count * 8) + sizeof(*header); +} + +static size_t a5xx_snapshot_cp_merciu(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + int i, size; + + if (adreno_is_a505_or_a506(adreno_dev) || adreno_is_a508(adreno_dev) || + adreno_is_a540(adreno_dev) || adreno_is_a512(adreno_dev)) + size = 1024; + else if (adreno_is_a510(adreno_dev)) + size = 32; + else + size = 64; + + /* The MERCIU data is two dwords per entry */ + size = size << 1; + + if (remain < DEBUG_SECTION_SZ(size)) { + SNAPSHOT_ERR_NOMEM(device, "CP MERCIU DEBUG"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_CP_MERCIU; + header->size = size; + + kgsl_regwrite(device, A5XX_CP_MERCIU_DBG_ADDR, 0); + + for (i = 0; i < size; i++) { + kgsl_regread(device, A5XX_CP_MERCIU_DBG_DATA_1, + &data[(i * 2)]); + kgsl_regread(device, A5XX_CP_MERCIU_DBG_DATA_2, + &data[(i * 2) + 1]); + } + + return DEBUG_SECTION_SZ(size); +} + +static size_t a5xx_snapshot_cp_roq(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *) buf; + u32 size, *data = (u32 *) (buf + sizeof(*header)); + + if (adreno_is_a505_or_a506(adreno_dev) || adreno_is_a508(adreno_dev) || + adreno_is_a510(adreno_dev)) + size = 256; + else + size = 512; + + if (remain < DEBUG_SECTION_SZ(size)) { + SNAPSHOT_ERR_NOMEM(device, "CP ROQ DEBUG"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_CP_ROQ; + header->size = size; + + kgsl_regmap_read_indexed(&device->regmap, A5XX_CP_ROQ_DBG_ADDR, + A5XX_CP_ROQ_DBG_DATA, data, size); + + return DEBUG_SECTION_SZ(size); +} + +static size_t a5xx_snapshot_cp_meq(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *) buf; + u32 size, *data = (u32 *) (buf + sizeof(*header)); + + if (adreno_is_a505_or_a506(adreno_dev) || adreno_is_a508(adreno_dev) || + adreno_is_a510(adreno_dev)) + size = 32; + else + size = 64; + + if (remain < DEBUG_SECTION_SZ(size)) { + SNAPSHOT_ERR_NOMEM(device, "CP MEQ DEBUG"); + return 0; + } + + header->type = SNAPSHOT_DEBUG_CP_MEQ; + header->size = size; + + kgsl_regmap_read_indexed(&device->regmap, A5XX_CP_MEQ_DBG_ADDR, + A5XX_CP_MEQ_DBG_DATA, data, size); + + return DEBUG_SECTION_SZ(size); +} + +/* + * a5xx_snapshot() - A5XX GPU snapshot function + * @adreno_dev: Device being snapshotted + * @snapshot: Pointer to the snapshot instance + * + * This is where all of the A5XX specific bits and pieces are grabbed + * into the snapshot memory + */ +void a5xx_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int i; + u32 hi, lo; + struct adreno_ringbuffer *rb; + struct registers regs; + + /* Disable Clock gating temporarily for the debug bus to work */ + a5xx_hwcg_set(adreno_dev, false); + + /* Save some CP information that the generic snapshot uses */ + kgsl_regread(device, A5XX_CP_IB1_BASE, &lo); + kgsl_regread(device, A5XX_CP_IB1_BASE_HI, &hi); + + snapshot->ib1base = (((u64) hi) << 32) | lo; + + kgsl_regread(device, A5XX_CP_IB2_BASE, &lo); + kgsl_regread(device, A5XX_CP_IB2_BASE_HI, &hi); + + snapshot->ib2base = (((u64) hi) << 32) | lo; + + kgsl_regread(device, A5XX_CP_IB1_BUFSZ, &snapshot->ib1size); + kgsl_regread(device, A5XX_CP_IB2_BUFSZ, &snapshot->ib2size); + + /* Dump the registers which get affected by crash dumper trigger */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, + snapshot, a5xx_snapshot_pre_crashdump_regs, NULL); + + /* Dump vbif registers as well which get affected by crash dumper */ + SNAPSHOT_REGISTERS(device, snapshot, a5xx_vbif_registers); + + /* Try to run the crash dumper */ + _a5xx_do_crashdump(device); + + regs.regs = a5xx_registers; + regs.size = ARRAY_SIZE(a5xx_registers); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, snapshot, + a5xx_snapshot_registers, ®s); + + if (a5xx_has_gpmu(adreno_dev)) { + regs.regs = a5xx_gpmu_registers; + regs.size = ARRAY_SIZE(a5xx_gpmu_registers); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, + snapshot, a5xx_snapshot_registers, ®s); + } + + + /* Dump SP TP HLSQ registers */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, snapshot, + a5xx_snapshot_dump_hlsq_sp_tp_regs, NULL); + + /* CP_PFP indexed registers */ + kgsl_snapshot_indexed_registers(device, snapshot, + A5XX_CP_PFP_STAT_ADDR, A5XX_CP_PFP_STAT_DATA, 0, 36); + + /* CP_ME indexed registers */ + kgsl_snapshot_indexed_registers(device, snapshot, + A5XX_CP_ME_STAT_ADDR, A5XX_CP_ME_STAT_DATA, 0, 29); + + /* CP_DRAW_STATE */ + kgsl_snapshot_indexed_registers(device, snapshot, + A5XX_CP_DRAW_STATE_ADDR, A5XX_CP_DRAW_STATE_DATA, + 0, 1 << A5XX_CP_DRAW_STATE_ADDR_WIDTH); + + /* ME_UCODE Cache */ + kgsl_snapshot_indexed_registers(device, snapshot, + A5XX_CP_ME_UCODE_DBG_ADDR, A5XX_CP_ME_UCODE_DBG_DATA, + 0, 0x53F); + + /* PFP_UCODE Cache */ + kgsl_snapshot_indexed_registers(device, snapshot, + A5XX_CP_PFP_UCODE_DBG_ADDR, A5XX_CP_PFP_UCODE_DBG_DATA, + 0, 0x53F); + + /* CP MEQ */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a5xx_snapshot_cp_meq, NULL); + + /* CP ROQ */ + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a5xx_snapshot_cp_roq, NULL); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a5xx_snapshot_cp_merciu, NULL); + + /* CP PFP and PM4 */ + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a5xx_snapshot_cp_pfp, NULL); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a5xx_snapshot_cp_pm4, NULL); + + /* Debug bus */ + a5xx_snapshot_debugbus(device, snapshot); + + /* Shader memory */ + a5xx_snapshot_shader(device, snapshot); + + /* Preemption record */ + if (adreno_is_preemption_enabled(adreno_dev)) { + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2, + snapshot, snapshot_preemption_record, + rb->preemption_desc); + } + } + +} + +static int _a5xx_crashdump_init_shader(struct a5xx_shader_block *block, + uint64_t *ptr, uint64_t *offset) +{ + int qwords = 0; + unsigned int j; + + /* Capture each bank in the block */ + for (j = 0; j < A5XX_NUM_SHADER_BANKS; j++) { + /* Program the aperture */ + ptr[qwords++] = + (block->statetype << A5XX_SHADER_STATETYPE_SHIFT) | j; + ptr[qwords++] = (((uint64_t) A5XX_HLSQ_DBG_READ_SEL << 44)) | + (1 << 21) | 1; + + /* Read all the data in one chunk */ + ptr[qwords++] = registers->gpuaddr + *offset; + ptr[qwords++] = + (((uint64_t) A5XX_HLSQ_DBG_AHB_READ_APERTURE << 44)) | + block->sz; + + /* Remember the offset of the first bank for easy access */ + if (j == 0) + block->offset = *offset; + + *offset += block->sz * sizeof(unsigned int); + } + + return qwords; +} + +static int _a5xx_crashdump_init_hlsq(struct a5xx_hlsq_sp_tp_regs *regs, + uint64_t *ptr, uint64_t *offset) +{ + int qwords = 0; + + /* Program the aperture */ + ptr[qwords++] = + (regs->statetype << A5XX_SHADER_STATETYPE_SHIFT); + ptr[qwords++] = (((uint64_t) A5XX_HLSQ_DBG_READ_SEL << 44)) | + (1 << 21) | 1; + + /* Read all the data in one chunk */ + ptr[qwords++] = registers->gpuaddr + *offset; + ptr[qwords++] = + (((uint64_t) A5XX_HLSQ_DBG_AHB_READ_APERTURE << 44)) | + regs->size; + + /* Remember the offset of the first bank for easy access */ + regs->offset = *offset; + + *offset += regs->size * sizeof(unsigned int); + + return qwords; +} + +void a5xx_crashdump_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int script_size = 0; + unsigned int data_size = 0; + unsigned int i, j; + uint64_t *ptr; + uint64_t offset = 0; + + if (!IS_ERR_OR_NULL(capturescript) && !IS_ERR_OR_NULL(registers)) + return; + + /* + * We need to allocate two buffers: + * 1 - the buffer to hold the draw script + * 2 - the buffer to hold the data + */ + + /* + * To save the registers, we need 16 bytes per register pair for the + * script and a dword for each register int the data + */ + + /* Each pair needs 16 bytes (2 qwords) */ + script_size += (ARRAY_SIZE(a5xx_registers) / 2) * 16; + + /* Each register needs a dword in the data */ + for (j = 0; j < ARRAY_SIZE(a5xx_registers) / 2; j++) + data_size += REG_PAIR_COUNT(a5xx_registers, j) * + sizeof(unsigned int); + + if (a5xx_has_gpmu(adreno_dev)) { + /* Each pair needs 16 bytes (2 qwords) */ + script_size += (ARRAY_SIZE(a5xx_gpmu_registers) / 2) * 16; + + /* Each register needs a dword in the data */ + for (j = 0; j < ARRAY_SIZE(a5xx_gpmu_registers) / 2; j++) + data_size += REG_PAIR_COUNT(a5xx_gpmu_registers, j) * + sizeof(unsigned int); + } + + /* + * To save the shader blocks for each block in each type we need 32 + * bytes for the script (16 bytes to program the aperture and 16 to + * read the data) and then a block specific number of bytes to hold + * the data + */ + for (i = 0; i < ARRAY_SIZE(a5xx_shader_blocks); i++) { + script_size += 32 * A5XX_NUM_SHADER_BANKS; + data_size += a5xx_shader_blocks[i].sz * sizeof(unsigned int) * + A5XX_NUM_SHADER_BANKS; + } + for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_sp_tp_registers); i++) { + script_size += 32; + data_size += + a5xx_hlsq_sp_tp_registers[i].size * sizeof(unsigned int); + } + + /* Now allocate the script and data buffers */ + + /* The script buffers needs 2 extra qwords on the end */ + if (!IS_ERR_OR_NULL(capturescript)) + capturescript = kgsl_allocate_global(device, + script_size + 16, 0, KGSL_MEMFLAGS_GPUREADONLY, + KGSL_MEMDESC_PRIVILEGED, "capturescript"); + + if (IS_ERR(capturescript)) + return; + + if (!IS_ERR_OR_NULL(registers)) + registers = kgsl_allocate_global(device, data_size, 0, 0, + KGSL_MEMDESC_PRIVILEGED, "capturescript_regs"); + + if (IS_ERR(registers)) + return; + + /* Build the crash script */ + + ptr = (uint64_t *) capturescript->hostptr; + + /* For the registers, program a read command for each pair */ + + for (j = 0; j < ARRAY_SIZE(a5xx_registers) / 2; j++) { + unsigned int r = REG_PAIR_COUNT(a5xx_registers, j); + *ptr++ = registers->gpuaddr + offset; + *ptr++ = (((uint64_t) a5xx_registers[2 * j]) << 44) + | r; + offset += r * sizeof(unsigned int); + } + + if (a5xx_has_gpmu(adreno_dev)) { + for (j = 0; j < ARRAY_SIZE(a5xx_gpmu_registers) / 2; j++) { + unsigned int r = REG_PAIR_COUNT(a5xx_gpmu_registers, j); + *ptr++ = registers->gpuaddr + offset; + *ptr++ = (((uint64_t) a5xx_gpmu_registers[2 * j]) << 44) + | r; + offset += r * sizeof(unsigned int); + } + } + + /* Program each shader block */ + for (i = 0; i < ARRAY_SIZE(a5xx_shader_blocks); i++) { + ptr += _a5xx_crashdump_init_shader(&a5xx_shader_blocks[i], ptr, + &offset); + } + /* Program the hlsq sp tp register sets */ + for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_sp_tp_registers); i++) + ptr += _a5xx_crashdump_init_hlsq(&a5xx_hlsq_sp_tp_registers[i], + ptr, &offset); + + *ptr++ = 0; + *ptr++ = 0; +} diff --git a/qcom/opensource/graphics-kernel/adreno_a6xx.c b/qcom/opensource/graphics-kernel/adreno_a6xx.c new file mode 100644 index 0000000000..430affa035 --- /dev/null +++ b/qcom/opensource/graphics-kernel/adreno_a6xx.c @@ -0,0 +1,2486 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2017-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "adreno.h" +#include "adreno_a6xx.h" +#include "adreno_a6xx_hwsched.h" +#include "adreno_pm4types.h" +#include "adreno_trace.h" +#include "kgsl_trace.h" +#include "kgsl_util.h" + +/* IFPC & Preemption static powerup restore list */ +static u32 a6xx_pwrup_reglist[] = { + A6XX_VSC_ADDR_MODE_CNTL, + A6XX_GRAS_ADDR_MODE_CNTL, + A6XX_RB_ADDR_MODE_CNTL, + A6XX_PC_ADDR_MODE_CNTL, + A6XX_HLSQ_ADDR_MODE_CNTL, + A6XX_VFD_ADDR_MODE_CNTL, + A6XX_VPC_ADDR_MODE_CNTL, + A6XX_UCHE_ADDR_MODE_CNTL, + A6XX_SP_ADDR_MODE_CNTL, + A6XX_TPL1_ADDR_MODE_CNTL, + A6XX_UCHE_WRITE_RANGE_MAX_LO, + A6XX_UCHE_WRITE_RANGE_MAX_HI, + A6XX_UCHE_TRAP_BASE_LO, + A6XX_UCHE_TRAP_BASE_HI, + A6XX_UCHE_WRITE_THRU_BASE_LO, + A6XX_UCHE_WRITE_THRU_BASE_HI, + A6XX_UCHE_GMEM_RANGE_MIN_LO, + A6XX_UCHE_GMEM_RANGE_MIN_HI, + A6XX_UCHE_GMEM_RANGE_MAX_LO, + A6XX_UCHE_GMEM_RANGE_MAX_HI, + A6XX_UCHE_FILTER_CNTL, + A6XX_UCHE_CACHE_WAYS, + A6XX_UCHE_MODE_CNTL, + A6XX_RB_NC_MODE_CNTL, + A6XX_TPL1_NC_MODE_CNTL, + A6XX_SP_NC_MODE_CNTL, + A6XX_PC_DBG_ECO_CNTL, + A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE, + A6XX_UCHE_GBIF_GX_CONFIG, + A6XX_UCHE_CLIENT_PF, +}; + +/* IFPC only static powerup restore list */ +static u32 a6xx_ifpc_pwrup_reglist[] = { + A6XX_CP_CHICKEN_DBG, + A6XX_CP_DBG_ECO_CNTL, + A6XX_CP_PROTECT_CNTL, + A6XX_CP_PROTECT_REG, + A6XX_CP_PROTECT_REG+1, + A6XX_CP_PROTECT_REG+2, + A6XX_CP_PROTECT_REG+3, + A6XX_CP_PROTECT_REG+4, + A6XX_CP_PROTECT_REG+5, + A6XX_CP_PROTECT_REG+6, + A6XX_CP_PROTECT_REG+7, + A6XX_CP_PROTECT_REG+8, + A6XX_CP_PROTECT_REG+9, + A6XX_CP_PROTECT_REG+10, + A6XX_CP_PROTECT_REG+11, + A6XX_CP_PROTECT_REG+12, + A6XX_CP_PROTECT_REG+13, + A6XX_CP_PROTECT_REG+14, + A6XX_CP_PROTECT_REG+15, + A6XX_CP_PROTECT_REG+16, + A6XX_CP_PROTECT_REG+17, + A6XX_CP_PROTECT_REG+18, + A6XX_CP_PROTECT_REG+19, + A6XX_CP_PROTECT_REG+20, + A6XX_CP_PROTECT_REG+21, + A6XX_CP_PROTECT_REG+22, + A6XX_CP_PROTECT_REG+23, + A6XX_CP_PROTECT_REG+24, + A6XX_CP_PROTECT_REG+25, + A6XX_CP_PROTECT_REG+26, + A6XX_CP_PROTECT_REG+27, + A6XX_CP_PROTECT_REG+28, + A6XX_CP_PROTECT_REG+29, + A6XX_CP_PROTECT_REG+30, + A6XX_CP_PROTECT_REG+31, + A6XX_CP_AHB_CNTL, +}; + +/* Applicable to a620, a621, a635, a650 and a660 */ +static u32 a650_ifpc_pwrup_reglist[] = { + A6XX_CP_PROTECT_REG+32, + A6XX_CP_PROTECT_REG+33, + A6XX_CP_PROTECT_REG+34, + A6XX_CP_PROTECT_REG+35, + A6XX_CP_PROTECT_REG+36, + A6XX_CP_PROTECT_REG+37, + A6XX_CP_PROTECT_REG+38, + A6XX_CP_PROTECT_REG+39, + A6XX_CP_PROTECT_REG+40, + A6XX_CP_PROTECT_REG+41, + A6XX_CP_PROTECT_REG+42, + A6XX_CP_PROTECT_REG+43, + A6XX_CP_PROTECT_REG+44, + A6XX_CP_PROTECT_REG+45, + A6XX_CP_PROTECT_REG+46, + A6XX_CP_PROTECT_REG+47, +}; + +/* Applicable to a620, a621, a635, a650 and a660 */ +static u32 a650_pwrup_reglist[] = { + A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0, + A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1, + A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2, + A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3, + A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4, + A6XX_UCHE_CMDQ_CONFIG, +}; + +static u32 a615_pwrup_reglist[] = { + A6XX_UCHE_GBIF_GX_CONFIG, +}; + +int a6xx_fenced_write(struct adreno_device *adreno_dev, u32 offset, + u32 value, u32 mask) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int status, i; + u64 ts1, ts2; + + kgsl_regwrite(device, offset, value); + + if (!gmu_core_isenabled(device)) + return 0; + + ts1 = a6xx_read_alwayson(adreno_dev); + for (i = 0; i < GMU_CORE_LONG_WAKEUP_RETRY_LIMIT; i++) { + /* + * Make sure the previous register write is posted before + * checking the fence status + */ + mb(); + + kgsl_regread(device, A6XX_GMU_AHB_FENCE_STATUS, &status); + + /* + * If !writedropped0/1, then the write to fenced register + * was successful + */ + if (!(status & mask)) + break; + + /* Wait a small amount of time before trying again */ + udelay(GMU_CORE_WAKEUP_DELAY_US); + + /* Try to write the fenced register again */ + kgsl_regwrite(device, offset, value); + } + + if (i < GMU_CORE_SHORT_WAKEUP_RETRY_LIMIT) + return 0; + + if (i == GMU_CORE_LONG_WAKEUP_RETRY_LIMIT) { + ts2 = a6xx_read_alwayson(adreno_dev); + dev_err(adreno_dev->dev.dev, + "Timed out waiting %d usecs to write fenced register 0x%x, timestamps: %llx %llx\n", + i * GMU_CORE_WAKEUP_DELAY_US, offset, ts1, ts2); + return -ETIMEDOUT; + } + + dev_err(adreno_dev->dev.dev, + "Waited %d usecs to write fenced register 0x%x\n", + i * GMU_CORE_WAKEUP_DELAY_US, offset); + + return 0; +} + +int a6xx_init(struct adreno_device *adreno_dev) +{ + const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); + u64 freq = a6xx_core->gmu_hub_clk_freq; + + adreno_dev->highest_bank_bit = a6xx_core->highest_bank_bit; + + adreno_dev->gmu_hub_clk_freq = freq ? freq : 150000000; + + adreno_dev->cooperative_reset = ADRENO_FEATURE(adreno_dev, + ADRENO_COOP_RESET); + + /* If the memory type is DDR 4, override the existing configuration */ + if (of_fdt_get_ddrtype() == 0x7) { + if (adreno_is_a660_shima(adreno_dev) || + adreno_is_a635(adreno_dev) || + adreno_is_a662(adreno_dev)) + adreno_dev->highest_bank_bit = 14; + else if ((adreno_is_a650(adreno_dev) || + adreno_is_a660(adreno_dev))) + adreno_dev->highest_bank_bit = 15; + } + + a6xx_crashdump_init(adreno_dev); + + return adreno_allocate_global(KGSL_DEVICE(adreno_dev), + &adreno_dev->pwrup_reglist, + PAGE_SIZE, 0, 0, KGSL_MEMDESC_PRIVILEGED, + "powerup_register_list"); +} + +static int a6xx_nogmu_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + ret = a6xx_ringbuffer_init(adreno_dev); + if (ret) + return ret; + + ret = a6xx_microcode_read(adreno_dev); + if (ret) + return ret; + + /* Try to map the GMU wrapper region if applicable */ + ret = kgsl_regmap_add_region(&device->regmap, device->pdev, + "gmu_wrapper", NULL, NULL); + if (ret && ret != -ENODEV) + dev_err(device->dev, "Couldn't map the GMU wrapper registers\n"); + + adreno_create_profile_buffer(adreno_dev); + + return a6xx_init(adreno_dev); +} + +static void a6xx_protect_init(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); + const struct adreno_protected_regs *regs = a6xx_core->protected_regs; + int i; + + /* + * Enable access protection to privileged registers, fault on an access + * protect violation and select the last span to protect from the start + * address all the way to the end of the register address space + */ + kgsl_regwrite(device, A6XX_CP_PROTECT_CNTL, + (1 << 0) | (1 << 1) | (1 << 3)); + + /* Program each register defined by the core definition */ + for (i = 0; regs[i].reg; i++) { + u32 count; + + /* + * This is the offset of the end register as counted from the + * start, i.e. # of registers in the range - 1 + */ + count = regs[i].end - regs[i].start; + + kgsl_regwrite(device, regs[i].reg, + (regs[i].start & 0x3ffff) | ((count & 0x1fff) << 18) | + (regs[i].noaccess << 31)); + } +} + +static inline unsigned int +__get_rbbm_clock_cntl_on(struct adreno_device *adreno_dev) +{ + if (adreno_is_a630(adreno_dev)) + return 0x8AA8AA02; + else if (adreno_is_a612(adreno_dev) || adreno_is_a610_family(adreno_dev)) + return 0xAAA8AA82; + else if (adreno_is_a702(adreno_dev)) + return 0xAAAAAA82; + else + return 0x8AA8AA82; +} + +static inline unsigned int +__get_gmu_ao_cgc_mode_cntl(struct adreno_device *adreno_dev) +{ + if (adreno_is_a612(adreno_dev)) + return 0x00000022; + else if (adreno_is_a615_family(adreno_dev)) + return 0x00000222; + /* a662 should be checked before a660 */ + else if (adreno_is_a662(adreno_dev) || adreno_is_a621(adreno_dev)) + return 0x00020200; + else if (adreno_is_a660(adreno_dev)) + return 0x00020000; + else + return 0x00020202; +} + +static inline unsigned int +__get_gmu_ao_cgc_delay_cntl(struct adreno_device *adreno_dev) +{ + if (adreno_is_a612(adreno_dev)) + return 0x00000011; + else if (adreno_is_a615_family(adreno_dev)) + return 0x00000111; + else + return 0x00010111; +} + +static inline unsigned int +__get_gmu_ao_cgc_hyst_cntl(struct adreno_device *adreno_dev) +{ + if (adreno_is_a612(adreno_dev)) + return 0x00000055; + else if (adreno_is_a615_family(adreno_dev)) + return 0x00000555; + else + return 0x00005555; +} + +static unsigned int __get_gmu_wfi_config(struct adreno_device *adreno_dev) +{ + unsigned int rev = ADRENO_GPUREV(adreno_dev); + + if ((rev == ADRENO_REV_A620) || adreno_is_a640(adreno_dev) || + adreno_is_a650(adreno_dev)) + return 0x00000002; + + return 0x00000000; +} + +static void set_holi_sptprac_clock(struct kgsl_device *device, bool enable) +{ + u32 val = 0; + + kgsl_regread(device, A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, &val); + val &= ~1; + kgsl_regwrite(device, A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, + val | (enable ? 1 : 0)); +} + +static void a6xx_hwcg_set(struct adreno_device *adreno_dev, bool on) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); + unsigned int value; + int i; + + if (!adreno_dev->hwcg_enabled) + on = false; + + if (gmu_core_isenabled(device)) { + gmu_core_regwrite(device, A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL, + on ? __get_gmu_ao_cgc_mode_cntl(adreno_dev) : 0); + gmu_core_regwrite(device, A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, + on ? __get_gmu_ao_cgc_delay_cntl(adreno_dev) : 0); + gmu_core_regwrite(device, A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL, + on ? __get_gmu_ao_cgc_hyst_cntl(adreno_dev) : 0); + gmu_core_regwrite(device, A6XX_GMU_CX_GMU_WFI_CONFIG, + on ? __get_gmu_wfi_config(adreno_dev) : 0); + } + + kgsl_regread(device, A6XX_RBBM_CLOCK_CNTL, &value); + + if (value == __get_rbbm_clock_cntl_on(adreno_dev) && on) + return; + + if (value == 0 && !on) + return; + + /* + * Disable SP clock before programming HWCG registers. + * A612 and A610 GPU is not having the GX power domain. + * Hence skip GMU_GX registers for A12 and A610. + */ + + if (gmu_core_isenabled(device) && !adreno_is_a612(adreno_dev) && + !adreno_is_a610_family(adreno_dev) && !adreno_is_a702(adreno_dev)) + gmu_core_regrmw(device, + A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0); + else if (adreno_is_a619_holi(adreno_dev)) + set_holi_sptprac_clock(device, false); + + for (i = 0; i < a6xx_core->hwcg_count; i++) + kgsl_regwrite(device, a6xx_core->hwcg[i].offset, + on ? a6xx_core->hwcg[i].val : 0); + + /* GBIF L2 CGC control is not part of the UCHE */ + kgsl_regrmw(device, A6XX_UCHE_GBIF_GX_CONFIG, 0x70000, + FIELD_PREP(GENMASK(18, 16), on ? 2 : 0)); + + /* + * Enable SP clock after programming HWCG registers. + * A612 and A610 GPU is not having the GX power domain. + * Hence skip GMU_GX registers for A612. + */ + if (gmu_core_isenabled(device) && !adreno_is_a612(adreno_dev) && + !adreno_is_a610_family(adreno_dev) && !adreno_is_a702(adreno_dev)) + gmu_core_regrmw(device, + A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1); + else if (adreno_is_a619_holi(adreno_dev)) + set_holi_sptprac_clock(device, true); + + /* enable top level HWCG */ + kgsl_regwrite(device, A6XX_RBBM_CLOCK_CNTL, + on ? __get_rbbm_clock_cntl_on(adreno_dev) : 0); +} + +struct a6xx_reglist_list { + u32 *regs; + u32 count; +}; + +#define REGLIST(_a) \ + ((struct a6xx_reglist_list) { .regs = _a, .count = ARRAY_SIZE(_a), }) + +static void a6xx_patch_pwrup_reglist(struct adreno_device *adreno_dev) +{ + struct a6xx_reglist_list reglist[4]; + void *ptr = adreno_dev->pwrup_reglist->hostptr; + struct cpu_gpu_lock *lock = ptr; + int items = 0, i, j; + u32 *dest = ptr + sizeof(*lock); + u16 list_offset = 0; + + /* Static IFPC-only registers */ + reglist[items] = REGLIST(a6xx_ifpc_pwrup_reglist); + list_offset += reglist[items++].count * 2; + + if (adreno_is_a650_family(adreno_dev)) { + reglist[items] = REGLIST(a650_ifpc_pwrup_reglist); + list_offset += reglist[items++].count * 2; + } + + /* Static IFPC + preemption registers */ + reglist[items++] = REGLIST(a6xx_pwrup_reglist); + + /* Add target specific registers */ + if (adreno_is_a615_family(adreno_dev)) + reglist[items++] = REGLIST(a615_pwrup_reglist); + else if (adreno_is_a650_family(adreno_dev)) + reglist[items++] = REGLIST(a650_pwrup_reglist); + + /* + * For each entry in each of the lists, write the offset and the current + * register value into the GPU buffer + */ + for (i = 0; i < items; i++) { + u32 *r = reglist[i].regs; + + for (j = 0; j < reglist[i].count; j++) { + *dest++ = r[j]; + kgsl_regread(KGSL_DEVICE(adreno_dev), r[j], dest++); + } + + lock->list_length += reglist[i].count * 2; + } + + if (adreno_is_a630(adreno_dev)) { + *dest++ = A6XX_RBBM_VBIF_CLIENT_QOS_CNTL; + kgsl_regread(KGSL_DEVICE(adreno_dev), + A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, dest++); + } else { + *dest++ = A6XX_RBBM_GBIF_CLIENT_QOS_CNTL; + kgsl_regread(KGSL_DEVICE(adreno_dev), + A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, dest++); + } + + lock->list_length += 2; + + *dest++ = A6XX_RBBM_PERFCTR_CNTL; + *dest++ = 1; + lock->list_length += 2; + + /* + * The overall register list is composed of + * 1. Static IFPC-only registers + * 2. Static IFPC + preemption registers + * 3. Dynamic IFPC + preemption registers (ex: perfcounter selects) + * + * The CP views the second and third entries as one dynamic list + * starting from list_offset. list_length should be the total dwords in + * all the lists and list_offset should be specified as the size in + * dwords of the first entry in the list. + */ + lock->list_offset = list_offset; +} + + +static void a6xx_llc_configure_gpu_scid(struct adreno_device *adreno_dev); +static void a6xx_llc_configure_gpuhtw_scid(struct adreno_device *adreno_dev); +static void a6xx_llc_enable_overrides(struct adreno_device *adreno_dev); + +static void a6xx_set_secvid(struct kgsl_device *device) +{ + static bool set; + + if (set || !device->mmu.secured) + return; + + kgsl_regwrite(device, A6XX_RBBM_SECVID_TSB_CNTL, 0x0); + kgsl_regwrite(device, A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, + lower_32_bits(KGSL_IOMMU_SECURE_BASE32)); + kgsl_regwrite(device, A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, + upper_32_bits(KGSL_IOMMU_SECURE_BASE32)); + kgsl_regwrite(device, A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, + FIELD_PREP(GENMASK(31, 12), + (KGSL_IOMMU_SECURE_SIZE(&device->mmu) / SZ_4K))); + + if (ADRENO_QUIRK(ADRENO_DEVICE(device), ADRENO_QUIRK_SECVID_SET_ONCE)) + set = true; +} + +static void a6xx_deassert_gbif_halt(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + kgsl_regwrite(device, A6XX_GBIF_HALT, 0x0); + + if (adreno_is_a619_holi(adreno_dev)) + kgsl_regwrite(device, A6XX_RBBM_GPR0_CNTL, 0x0); + else + kgsl_regwrite(device, A6XX_RBBM_GBIF_HALT, 0x0); +} + +bool a6xx_gx_is_on(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + bool gdsc_on, clk_on; + + clk_on = __clk_is_enabled(pwr->grp_clks[0]); + + gdsc_on = regulator_is_enabled(pwr->gx_gdsc); + + return (gdsc_on & clk_on); +} + +/* + * Some targets support marking certain transactions as always privileged which + * allows us to mark more memory as privileged without having to explicitly set + * the APRIV bit. For those targets, choose the following transactions to be + * privileged by default: + * CDWRITE [6:6] - Crashdumper writes + * CDREAD [5:5] - Crashdumper reads + * RBRPWB [3:3] - RPTR shadow writes + * RBPRIVLEVEL [2:2] - Memory accesses from PM4 packets in the ringbuffer + * RBFETCH [1:1] - Ringbuffer reads + */ +#define A6XX_APRIV_DEFAULT \ + ((1 << 6) | (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1)) + +void a6xx_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); + unsigned int mal, mode, hbb_hi = 0, hbb_lo = 0; + unsigned int uavflagprd_inv; + unsigned int amsbc = 0; + unsigned int rgb565_predicator = 0; + unsigned int level2_swizzling_dis = 0; + + /* Enable 64 bit addressing */ + kgsl_regwrite(device, A6XX_CP_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_VSC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_GRAS_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_RB_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_PC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_HLSQ_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_VFD_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_VPC_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_UCHE_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_SP_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_TPL1_ADDR_MODE_CNTL, 0x1); + kgsl_regwrite(device, A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1); + + /* Set up VBIF registers from the GPU core definition */ + kgsl_regmap_multi_write(&device->regmap, a6xx_core->vbif, + a6xx_core->vbif_count); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_LIMIT_UCHE_GBIF_RW)) + kgsl_regwrite(device, A6XX_UCHE_GBIF_GX_CONFIG, 0x10200F9); + + /* Make all blocks contribute to the GPU BUSY perf counter */ + kgsl_regwrite(device, A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF); + + /* + * Set UCHE_WRITE_THRU_BASE to the UCHE_TRAP_BASE effectively + * disabling L2 bypass + */ + kgsl_regwrite(device, A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0); + kgsl_regwrite(device, A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff); + kgsl_regwrite(device, A6XX_UCHE_TRAP_BASE_LO, 0xfffff000); + kgsl_regwrite(device, A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff); + kgsl_regwrite(device, A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000); + kgsl_regwrite(device, A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff); + + /* + * Some A6xx targets no longer use a programmed UCHE GMEM base + * address, so only write the registers if this address is + * non-zero. + */ + if (adreno_dev->uche_gmem_base) { + kgsl_regwrite(device, A6XX_UCHE_GMEM_RANGE_MIN_LO, + adreno_dev->uche_gmem_base); + kgsl_regwrite(device, A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x0); + kgsl_regwrite(device, A6XX_UCHE_GMEM_RANGE_MAX_LO, + adreno_dev->uche_gmem_base + + adreno_dev->gpucore->gmem_size - 1); + kgsl_regwrite(device, A6XX_UCHE_GMEM_RANGE_MAX_HI, 0x0); + } + + kgsl_regwrite(device, A6XX_UCHE_FILTER_CNTL, 0x804); + kgsl_regwrite(device, A6XX_UCHE_CACHE_WAYS, 0x4); + + /* ROQ sizes are twice as big on a640/a680 than on a630 */ + if ((ADRENO_GPUREV(adreno_dev) >= ADRENO_REV_A640) && + !adreno_is_a702(adreno_dev)) { + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140); + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362C); + } else if (adreno_is_a612(adreno_dev) || adreno_is_a610_family(adreno_dev) || + adreno_is_a702(adreno_dev)) { + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_2, 0x00800060); + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_1, 0x40201b16); + } else { + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_2, 0x010000C0); + kgsl_regwrite(device, A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362C); + } + + if (adreno_is_a660(adreno_dev)) + kgsl_regwrite(device, A6XX_CP_LPAC_PROG_FIFO_SIZE, 0x00000020); + + if (adreno_is_a663(adreno_dev)) { + kgsl_regwrite(device, A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x0); + kgsl_regwrite(device, A6XX_RBBM_LPAC_GBIF_CLIENT_QOS_CNTL, 0x0); + kgsl_regwrite(device, A6XX_CP_LPAC_PROG_FIFO_SIZE, 0x00000020); + } + + if (adreno_is_a612(adreno_dev) || adreno_is_a610_family(adreno_dev)) { + /* For A612 and A610 Mem pool size is reduced to 48 */ + kgsl_regwrite(device, A6XX_CP_MEM_POOL_SIZE, 48); + kgsl_regwrite(device, A6XX_CP_MEM_POOL_DBG_ADDR, 47); + } else if (adreno_is_a702(adreno_dev)) { + kgsl_regwrite(device, A6XX_CP_MEM_POOL_SIZE, 64); + kgsl_regwrite(device, A6XX_CP_MEM_POOL_DBG_ADDR, 63); + } else { + kgsl_regwrite(device, A6XX_CP_MEM_POOL_SIZE, 128); + } + + /* Setting the primFifo thresholds values */ + kgsl_regwrite(device, A6XX_PC_DBG_ECO_CNTL, + a6xx_core->prim_fifo_threshold); + + /* Set the AHB default slave response to "ERROR" */ + kgsl_regwrite(device, A6XX_CP_AHB_CNTL, 0x1); + + /* Turn on performance counters */ + kgsl_regwrite(device, A6XX_RBBM_PERFCTR_CNTL, 0x1); + + /* Turn on the IFPC counter (countable 4 on XOCLK4) */ + if (gmu_core_isenabled(device)) + gmu_core_regrmw(device, A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1, + 0xff, 0x4); + + /* Turn on GX_MEM retention */ + if (gmu_core_isenabled(device) && adreno_is_a612(adreno_dev)) { + kgsl_regwrite(device, A6XX_RBBM_BLOCK_GX_RETENTION_CNTL, 0x7FB); + /* For CP IPC interrupt */ + kgsl_regwrite(device, A6XX_RBBM_INT_2_MASK, 0x00000010); + } + + if (of_property_read_u32(device->pdev->dev.of_node, + "qcom,min-access-length", &mal)) + mal = 32; + + if (of_property_read_u32(device->pdev->dev.of_node, + "qcom,ubwc-mode", &mode)) + mode = 0; + + switch (mode) { + case KGSL_UBWC_1_0: + mode = 1; + break; + case KGSL_UBWC_2_0: + mode = 0; + break; + case KGSL_UBWC_3_0: + mode = 0; + amsbc = 1; /* Only valid for A640 and A680 */ + break; + case KGSL_UBWC_4_0: + mode = 0; + rgb565_predicator = 1; + amsbc = 1; + if (adreno_is_a663(adreno_dev)) + level2_swizzling_dis = 1; + break; + default: + break; + } + + /* macrotilingmode 0: 4 channels (default) + * overwrite to 1: 8 channels for A680 + */ + if (adreno_is_a680(adreno_dev) || + adreno_is_a663(adreno_dev)) + kgsl_regwrite(device, A6XX_RBBM_NC_MODE_CNTL, 1); + + if (!WARN_ON(!adreno_dev->highest_bank_bit)) { + hbb_lo = (adreno_dev->highest_bank_bit - 13) & 3; + hbb_hi = ((adreno_dev->highest_bank_bit - 13) >> 2) & 1; + } + + mal = (mal == 64) ? 1 : 0; + + uavflagprd_inv = (adreno_is_a650_family(adreno_dev)) ? 2 : 0; + + kgsl_regwrite(device, A6XX_RB_NC_MODE_CNTL, + (level2_swizzling_dis << 12) | (rgb565_predicator << 11)| + (hbb_hi << 10) | (amsbc << 4) | (mal << 3) | + (hbb_lo << 1) | mode); + + kgsl_regwrite(device, A6XX_TPL1_NC_MODE_CNTL, + (level2_swizzling_dis << 6) | (hbb_hi << 4) | + (mal << 3) | (hbb_lo << 1) | mode); + + kgsl_regwrite(device, A6XX_SP_NC_MODE_CNTL, + (level2_swizzling_dis << 12) | (hbb_hi << 10) | + (mal << 3) | (uavflagprd_inv << 4) | + (hbb_lo << 1) | mode); + + kgsl_regwrite(device, A6XX_UCHE_MODE_CNTL, (mal << 23) | + (hbb_lo << 21)); + + kgsl_regwrite(device, A6XX_RBBM_INTERFACE_HANG_INT_CNTL, + (1 << 30) | a6xx_core->hang_detect_cycles); + + kgsl_regwrite(device, A6XX_UCHE_CLIENT_PF, BIT(7) | + FIELD_PREP(GENMASK(3, 0), adreno_dev->uche_client_pf)); + + /* Set weights for bicubic filtering */ + if (adreno_is_a650_family(adreno_dev)) { + kgsl_regwrite(device, A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0, 0); + kgsl_regwrite(device, A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1, + 0x3FE05FF4); + kgsl_regwrite(device, A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2, + 0x3FA0EBEE); + kgsl_regwrite(device, A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3, + 0x3F5193ED); + kgsl_regwrite(device, A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4, + 0x3F0243F0); + } + + /* Set TWOPASSUSEWFI in A6XX_PC_DBG_ECO_CNTL if requested */ + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_TWO_PASS_USE_WFI)) + kgsl_regrmw(device, A6XX_PC_DBG_ECO_CNTL, 0, (1 << 8)); + + /* Set the bit vccCacheSkipDis=1 to get rid of TSEskip logic */ + if (a6xx_core->disable_tseskip) + kgsl_regrmw(device, A6XX_PC_DBG_ECO_CNTL, 0, (1 << 9)); + + /* Set the bit in HLSQ Cluster for A702 */ + if (adreno_is_a702(adreno_dev)) + kgsl_regwrite(device, A6XX_CP_CHICKEN_DBG, (1 << 24)); + + /* Enable the GMEM save/restore feature for preemption */ + if (adreno_is_preemption_enabled(adreno_dev)) + kgsl_regwrite(device, A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE, + 0x1); + + /* + * Enable GMU power counter 0 to count GPU busy. This is applicable to + * all a6xx targets + */ + kgsl_regwrite(device, A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, 0xff000000); + kgsl_regrmw(device, A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, 0xff, 0x20); + kgsl_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0x1); + + a6xx_protect_init(adreno_dev); + /* + * We start LM here because we want all the following to be up + * 1. GX HS + * 2. SPTPRAC + * 3. HFI + * At this point, we are guaranteed all. + */ + + /* Configure LLCC */ + a6xx_llc_configure_gpu_scid(adreno_dev); + a6xx_llc_configure_gpuhtw_scid(adreno_dev); + + a6xx_llc_enable_overrides(adreno_dev); + + if (adreno_is_a662(adreno_dev)) + kgsl_regrmw(device, A6XX_GBIF_CX_CONFIG, 0x3c0, + FIELD_PREP(GENMASK(7, 6), 0x1) | + FIELD_PREP(GENMASK(9, 8), 0x1)); + + if (adreno_is_a660(adreno_dev)) { + kgsl_regwrite(device, A6XX_CP_CHICKEN_DBG, 0x1); + kgsl_regwrite(device, A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x0); + + /* Set dualQ + disable afull for A660 GPU but not for A635 */ + if (!adreno_is_a635(adreno_dev)) + kgsl_regwrite(device, A6XX_UCHE_CMDQ_CONFIG, 0x66906); + } + + if (ADRENO_FEATURE(adreno_dev, ADRENO_APRIV)) + kgsl_regwrite(device, A6XX_CP_APRIV_CNTL, A6XX_APRIV_DEFAULT); + + a6xx_set_secvid(device); + + /* + * Enable hardware clock gating here to prevent any register access + * issue due to internal clock gating. + */ + a6xx_hwcg_set(adreno_dev, true); + + /* + * All registers must be written before this point so that we don't + * miss any register programming when we patch the power up register + * list. + */ + if (!adreno_dev->patch_reglist && + (adreno_dev->pwrup_reglist->gpuaddr != 0)) { + a6xx_patch_pwrup_reglist(adreno_dev); + adreno_dev->patch_reglist = true; + } + + /* + * During adreno_stop, GBIF halt is asserted to ensure + * no further transaction can go through GPU before GPU + * headswitch is turned off. + * + * This halt is deasserted once headswitch goes off but + * incase headswitch doesn't goes off clear GBIF halt + * here to ensure GPU wake-up doesn't fail because of + * halted GPU transactions. + */ + a6xx_deassert_gbif_halt(adreno_dev); + +} + +/* Offsets into the MX/CX mapped register regions */ +#define RDPM_MX_OFFSET 0xf00 +#define RDPM_CX_OFFSET 0xf18 + +void a6xx_rdpm_mx_freq_update(struct a6xx_gmu_device *gmu, + u32 freq) +{ + if (gmu->rdpm_mx_virt) { + writel_relaxed(freq/1000, + (gmu->rdpm_mx_virt + RDPM_MX_OFFSET)); + + /* + * ensure previous writes post before this one, + * i.e. act like normal writel() + */ + wmb(); + } +} + +void a6xx_rdpm_cx_freq_update(struct a6xx_gmu_device *gmu, + u32 freq) +{ + if (gmu->rdpm_cx_virt) { + writel_relaxed(freq/1000, + (gmu->rdpm_cx_virt + RDPM_CX_OFFSET)); + + /* + * ensure previous writes post before this one, + * i.e. act like normal writel() + */ + wmb(); + } +} + +/* This is the start point for non GMU/RGMU targets */ +static int a6xx_nogmu_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + /* + * During adreno_stop() GBIF halt is asserted to ensure that + * no further transactions go through the GPU before the + * GPU headswitch is turned off. + * + * The halt is supposed to be deasserted when the headswitch goes off + * but clear it again during start to be sure + */ + kgsl_regwrite(device, A6XX_GBIF_HALT, 0x0); + kgsl_regwrite(device, A6XX_RBBM_GBIF_HALT, 0x0); + + ret = kgsl_mmu_start(device); + if (ret) + return ret; + + adreno_get_bus_counters(adreno_dev); + adreno_perfcounter_restore(adreno_dev); + + a6xx_start(adreno_dev); + return 0; +} + +/* + * CP_INIT_MAX_CONTEXT bit tells if the multiple hardware contexts can + * be used at once of if they should be serialized + */ +#define CP_INIT_MAX_CONTEXT BIT(0) + +/* Enables register protection mode */ +#define CP_INIT_ERROR_DETECTION_CONTROL BIT(1) + +/* Header dump information */ +#define CP_INIT_HEADER_DUMP BIT(2) /* Reserved */ + +/* Default Reset states enabled for PFP and ME */ +#define CP_INIT_DEFAULT_RESET_STATE BIT(3) + +/* Drawcall filter range */ +#define CP_INIT_DRAWCALL_FILTER_RANGE BIT(4) + +/* Ucode workaround masks */ +#define CP_INIT_UCODE_WORKAROUND_MASK BIT(5) + +/* + * Operation mode mask + * + * This ordinal provides the option to disable the + * save/restore of performance counters across preemption. + */ +#define CP_INIT_OPERATION_MODE_MASK BIT(6) + +/* Register initialization list */ +#define CP_INIT_REGISTER_INIT_LIST BIT(7) + +/* Register initialization list with spinlock */ +#define CP_INIT_REGISTER_INIT_LIST_WITH_SPINLOCK BIT(8) + +#define CP_INIT_MASK (CP_INIT_MAX_CONTEXT | \ + CP_INIT_ERROR_DETECTION_CONTROL | \ + CP_INIT_HEADER_DUMP | \ + CP_INIT_DEFAULT_RESET_STATE | \ + CP_INIT_UCODE_WORKAROUND_MASK | \ + CP_INIT_OPERATION_MODE_MASK | \ + CP_INIT_REGISTER_INIT_LIST_WITH_SPINLOCK) + +void a6xx_cp_init_cmds(struct adreno_device *adreno_dev, u32 *cmds) +{ + int i = 0; + + cmds[i++] = cp_type7_packet(CP_ME_INIT, A6XX_CP_INIT_DWORDS - 1); + + /* Enabled ordinal mask */ + cmds[i++] = CP_INIT_MASK; + + if (CP_INIT_MASK & CP_INIT_MAX_CONTEXT) + cmds[i++] = 0x00000003; + + if (CP_INIT_MASK & CP_INIT_ERROR_DETECTION_CONTROL) + cmds[i++] = 0x20000000; + + if (CP_INIT_MASK & CP_INIT_HEADER_DUMP) { + /* Header dump address */ + cmds[i++] = 0x00000000; + /* Header dump enable and dump size */ + cmds[i++] = 0x00000000; + } + + if (CP_INIT_MASK & CP_INIT_UCODE_WORKAROUND_MASK) + cmds[i++] = 0x00000000; + + if (CP_INIT_MASK & CP_INIT_OPERATION_MODE_MASK) + cmds[i++] = 0x00000002; + + if (CP_INIT_MASK & CP_INIT_REGISTER_INIT_LIST_WITH_SPINLOCK) { + uint64_t gpuaddr = adreno_dev->pwrup_reglist->gpuaddr; + + cmds[i++] = lower_32_bits(gpuaddr); + cmds[i++] = upper_32_bits(gpuaddr); + cmds[i++] = 0; + } +} + +void a6xx_spin_idle_debug(struct adreno_device *adreno_dev, + const char *str) +{ + struct kgsl_device *device = &adreno_dev->dev; + unsigned int rptr, wptr; + unsigned int status, status3, intstatus; + unsigned int hwfault; + + dev_err(device->dev, str); + + kgsl_regread(device, A6XX_CP_RB_RPTR, &rptr); + kgsl_regread(device, A6XX_CP_RB_WPTR, &wptr); + + kgsl_regread(device, A6XX_RBBM_STATUS, &status); + kgsl_regread(device, A6XX_RBBM_STATUS3, &status3); + kgsl_regread(device, A6XX_RBBM_INT_0_STATUS, &intstatus); + kgsl_regread(device, A6XX_CP_HW_FAULT, &hwfault); + + + dev_err(device->dev, + "rb=%d pos=%X/%X rbbm_status=%8.8X/%8.8X int_0_status=%8.8X\n", + adreno_dev->cur_rb ? adreno_dev->cur_rb->id : -1, rptr, wptr, + status, status3, intstatus); + + dev_err(device->dev, " hwfault=%8.8X\n", hwfault); + + kgsl_device_snapshot(device, NULL, NULL, false); + +} + +/* + * a6xx_send_cp_init() - Initialize ringbuffer + * @adreno_dev: Pointer to adreno device + * @rb: Pointer to the ringbuffer of device + * + * Submit commands for ME initialization, + */ +static int a6xx_send_cp_init(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int *cmds; + int ret; + + cmds = adreno_ringbuffer_allocspace(rb, A6XX_CP_INIT_DWORDS); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + a6xx_cp_init_cmds(adreno_dev, cmds); + + ret = a6xx_ringbuffer_submit(rb, NULL, true); + if (!ret) { + ret = adreno_spin_idle(adreno_dev, 2000); + if (ret) { + a6xx_spin_idle_debug(adreno_dev, + "CP initialization failed to idle\n"); + + kgsl_sharedmem_writel(device->scratch, + SCRATCH_RB_OFFSET(rb->id, rptr), 0); + rb->wptr = 0; + rb->_wptr = 0; + } + } + + return ret; +} + +/* + * Follow the ME_INIT sequence with a preemption yield to allow the GPU to move + * to a different ringbuffer, if desired + */ +static int _preemption_init(struct adreno_device *adreno_dev, + struct adreno_ringbuffer *rb, unsigned int *cmds, + struct kgsl_context *context) +{ + unsigned int *cmds_orig = cmds; + + /* Turn CP protection OFF on legacy targets */ + if (!ADRENO_FEATURE(adreno_dev, ADRENO_APRIV)) + cmds += cp_protected_mode(adreno_dev, cmds, 0); + + *cmds++ = cp_type7_packet(CP_SET_PSEUDO_REGISTER, 6); + *cmds++ = SET_PSEUDO_PRIV_NON_SECURE_SAVE_ADDR; + cmds += cp_gpuaddr(adreno_dev, cmds, + rb->preemption_desc->gpuaddr); + + *cmds++ = SET_PSEUDO_PRIV_SECURE_SAVE_ADDR; + cmds += cp_gpuaddr(adreno_dev, cmds, + rb->secure_preemption_desc->gpuaddr); + + /* Turn CP protection back ON */ + if (!ADRENO_FEATURE(adreno_dev, ADRENO_APRIV)) + cmds += cp_protected_mode(adreno_dev, cmds, 1); + + *cmds++ = cp_type7_packet(CP_CONTEXT_SWITCH_YIELD, 4); + cmds += cp_gpuaddr(adreno_dev, cmds, 0x0); + *cmds++ = 0; + /* generate interrupt on preemption completion */ + *cmds++ = 0; + + return cmds - cmds_orig; +} + +static int a6xx_post_start(struct adreno_device *adreno_dev) +{ + int ret; + unsigned int *cmds, *start; + struct adreno_ringbuffer *rb = adreno_dev->cur_rb; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!adreno_is_preemption_enabled(adreno_dev)) + return 0; + + cmds = adreno_ringbuffer_allocspace(rb, 42); + if (IS_ERR(cmds)) { + dev_err(device->dev, + "error allocating preemption init cmds\n"); + return PTR_ERR(cmds); + } + start = cmds; + + cmds += _preemption_init(adreno_dev, rb, cmds, NULL); + + rb->_wptr = rb->_wptr - (42 - (cmds - start)); + + ret = a6xx_ringbuffer_submit(rb, NULL, false); + if (!ret) { + ret = adreno_spin_idle(adreno_dev, 2000); + if (ret) + a6xx_spin_idle_debug(adreno_dev, + "hw preemption initialization failed to idle\n"); + } + + return ret; +} + +int a6xx_rb_start(struct adreno_device *adreno_dev) +{ + const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 cp_rb_cntl = A6XX_CP_RB_CNTL_DEFAULT | + (ADRENO_FEATURE(adreno_dev, ADRENO_APRIV) ? 0 : (1 << 27)); + struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_SQE); + struct adreno_ringbuffer *rb; + uint64_t addr; + int ret, i; + unsigned int *cmds; + + /* Clear all the ringbuffers */ + FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { + memset(rb->buffer_desc->hostptr, 0xaa, KGSL_RB_SIZE); + kgsl_sharedmem_writel(device->scratch, + SCRATCH_RB_OFFSET(rb->id, rptr), 0); + + rb->wptr = 0; + rb->_wptr = 0; + rb->wptr_preempt_end = ~0; + } + + a6xx_preemption_start(adreno_dev); + + /* Set up the current ringbuffer */ + rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev); + addr = SCRATCH_RB_GPU_ADDR(device, rb->id, rptr); + + kgsl_regwrite(device, A6XX_CP_RB_RPTR_ADDR_LO, lower_32_bits(addr)); + kgsl_regwrite(device, A6XX_CP_RB_RPTR_ADDR_HI, upper_32_bits(addr)); + + /* + * The size of the ringbuffer in the hardware is the log2 + * representation of the size in quadwords (sizedwords / 2). + */ + kgsl_regwrite(device, A6XX_CP_RB_CNTL, cp_rb_cntl); + + kgsl_regwrite(device, A6XX_CP_RB_BASE, + lower_32_bits(rb->buffer_desc->gpuaddr)); + + kgsl_regwrite(device, A6XX_CP_RB_BASE_HI, + upper_32_bits(rb->buffer_desc->gpuaddr)); + + /* Program the ucode base for CP */ + kgsl_regwrite(device, A6XX_CP_SQE_INSTR_BASE_LO, + lower_32_bits(fw->memdesc->gpuaddr)); + kgsl_regwrite(device, A6XX_CP_SQE_INSTR_BASE_HI, + upper_32_bits(fw->memdesc->gpuaddr)); + + /* Clear the SQE_HALT to start the CP engine */ + kgsl_regwrite(device, A6XX_CP_SQE_CNTL, 1); + + ret = a6xx_send_cp_init(adreno_dev, rb); + if (ret) + return ret; + + ret = adreno_zap_shader_load(adreno_dev, a6xx_core->zap_name); + if (ret) + return ret; + + /* + * Take the GPU out of secure mode. Try the zap shader if it is loaded, + * otherwise just try to write directly to the secure control register + */ + if (!adreno_dev->zap_loaded) + kgsl_regwrite(device, A6XX_RBBM_SECVID_TRUST_CNTL, 0); + else { + cmds = adreno_ringbuffer_allocspace(rb, 2); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + *cmds++ = cp_packet(adreno_dev, CP_SET_SECURE_MODE, 1); + *cmds++ = 0; + + ret = a6xx_ringbuffer_submit(rb, NULL, true); + if (!ret) { + ret = adreno_spin_idle(adreno_dev, 2000); + if (ret) { + a6xx_spin_idle_debug(adreno_dev, + "Switch to unsecure failed to idle\n"); + return ret; + } + } + } + + return a6xx_post_start(adreno_dev); +} + +/* + * a6xx_sptprac_enable() - Power on SPTPRAC + * @adreno_dev: Pointer to Adreno device + */ +static int a6xx_sptprac_enable(struct adreno_device *adreno_dev) +{ + return a6xx_gmu_sptprac_enable(adreno_dev); +} + +/* + * a6xx_sptprac_disable() - Power off SPTPRAC + * @adreno_dev: Pointer to Adreno device + */ +static void a6xx_sptprac_disable(struct adreno_device *adreno_dev) +{ + a6xx_gmu_sptprac_disable(adreno_dev); +} + +/* + * a6xx_prepare_for_regulator_disable() - Prepare for regulator disable + * @adreno_dev: Pointer to Adreno device + */ +static void a6xx_prepare_for_regulator_disable(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!adreno_is_a611(adreno_dev)) + return; + + /* This sequence is only required for A611 */ + kgsl_regwrite(device, A6XX_RBBM_SW_RESET_CMD, 0x1); + /* Make sure software reset is triggered and completed */ + wmb(); + udelay(100); +} + +/* + * a6xx_gpu_keepalive() - GMU reg write to request GPU stays on + * @adreno_dev: Pointer to the adreno device that has the GMU + * @state: State to set: true is ON, false is OFF + */ +static void a6xx_gpu_keepalive(struct adreno_device *adreno_dev, + bool state) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!gmu_core_isenabled(device)) + return; + + gmu_core_regwrite(device, A6XX_GMU_GMU_PWR_COL_KEEPALIVE, state); +} + +bool a6xx_irq_pending(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 status; + + kgsl_regread(device, A6XX_RBBM_INT_0_STATUS, &status); + + /* Return busy if a interrupt is pending */ + return ((status & adreno_dev->irq_mask) || + atomic_read(&adreno_dev->pending_irq_refcnt)); +} + +static bool a619_holi_hw_isidle(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg; + + kgsl_regread(device, A6XX_RBBM_STATUS, ®); + if (reg & 0xfffffffe) + return false; + + return a6xx_irq_pending(adreno_dev) ? false : true; +} + +bool a6xx_hw_isidle(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int reg; + + /* Non GMU devices monitor the RBBM status */ + if (!gmu_core_isenabled(device)) { + kgsl_regread(device, A6XX_RBBM_STATUS, ®); + if (reg & 0xfffffffe) + return false; + + return a6xx_irq_pending(adreno_dev) ? false : true; + } + + gmu_core_regread(device, A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, ®); + + /* Bit 23 is GPUBUSYIGNAHB */ + return (reg & BIT(23)) ? false : true; +} + +int a6xx_microcode_read(struct adreno_device *adreno_dev) +{ + struct adreno_firmware *sqe_fw = ADRENO_FW(adreno_dev, ADRENO_FW_SQE); + const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); + + return adreno_get_firmware(adreno_dev, a6xx_core->sqefw_name, sqe_fw); +} + +static int64_t a6xx_read_throttling_counters(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int64_t adj = -1; + u32 a, b, c; + struct adreno_busy_data *busy = &adreno_dev->busy_data; + + if (!(adreno_dev->lm_enabled || adreno_dev->bcl_enabled)) + return 0; + + a = counter_delta(device, A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_L, + &busy->throttle_cycles[0]); + + b = counter_delta(device, A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_L, + &busy->throttle_cycles[1]); + + c = counter_delta(device, A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_L, + &busy->throttle_cycles[2]); + + /* + * Currently there are no a6xx targets with both LM and BCL enabled. + * So if BCL is enabled, we can log bcl counters and return. + */ + if (adreno_dev->bcl_enabled) { + trace_kgsl_bcl_clock_throttling(a, b, c); + return 0; + } + + /* + * The adjustment is the number of cycles lost to throttling, which + * is calculated as a weighted average of the cycles throttled + * at different levels. The adjustment is negative because in A6XX, + * the busy count includes the throttled cycles. Therefore, we want + * to remove them to prevent appearing to be busier than + * we actually are. + */ + if (adreno_is_a620(adreno_dev) || adreno_is_a650(adreno_dev)) + /* + * With the newer generations, CRC throttle from SIDs of 0x14 + * and above cannot be observed in power counters. Since 90% + * throttle uses SID 0x16 the adjustment calculation needs + * correction. The throttling is in increments of 4.2%, and the + * 91.7% counter does a weighted count by the value of sid used + * which are taken into consideration for the final formula. + */ + adj *= div_s64((a * 42) + (b * 500) + + (div_s64((int64_t)c - a - b * 12, 22) * 917), 1000); + else + adj *= ((a * 5) + (b * 50) + (c * 90)) / 100; + + trace_kgsl_clock_throttling(0, b, c, a, adj); + + return adj; +} +#define GPU_CPR_FSM_CTL_OFFSET 0x4 +static void a6xx_gx_cpr_toggle(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); + static void __iomem *gx_cpr_virt; + struct resource *res; + u32 val = 0; + + if (!a6xx_core->gx_cpr_toggle) + return; + + if (!gx_cpr_virt) { + res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM, + "gx_cpr"); + if (res == NULL) + return; + + gx_cpr_virt = devm_ioremap_resource(&device->pdev->dev, res); + if (!gx_cpr_virt) { + dev_err(device->dev, "Failed to map GX CPR\n"); + return; + } + } + + /* + * Toggle(disable -> enable) closed loop functionality to recover + * CPR measurements stall happened under certain conditions. + */ + + val = readl_relaxed(gx_cpr_virt + GPU_CPR_FSM_CTL_OFFSET); + /* Make sure memory is updated before access */ + rmb(); + + writel_relaxed(val & 0xfffffff0, gx_cpr_virt + GPU_CPR_FSM_CTL_OFFSET); + /* make sure register write committed */ + wmb(); + + /* Wait for small time before we enable GX CPR */ + udelay(5); + + writel_relaxed(val | 0x00000001, gx_cpr_virt + GPU_CPR_FSM_CTL_OFFSET); + /* make sure register write committed */ + wmb(); +} + +/* This is only defined for non-GMU and non-RGMU targets */ +static int a6xx_clear_pending_transactions(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + if (adreno_is_a619_holi(adreno_dev)) { + kgsl_regwrite(device, A6XX_RBBM_GPR0_CNTL, 0x1e0); + ret = adreno_wait_for_halt_ack(device, + A6XX_RBBM_VBIF_GX_RESET_STATUS, 0xf0); + } else { + kgsl_regwrite(device, A6XX_RBBM_GBIF_HALT, + A6XX_GBIF_GX_HALT_MASK); + ret = adreno_wait_for_halt_ack(device, A6XX_RBBM_GBIF_HALT_ACK, + A6XX_GBIF_GX_HALT_MASK); + } + + if (ret) + return ret; + + return a6xx_halt_gbif(adreno_dev); +} + +/** + * a6xx_reset() - Helper function to reset the GPU + * @adreno_dev: Pointer to the adreno device structure for the GPU + * + * Try to reset the GPU to recover from a fault for targets without + * a GMU. + */ +static int a6xx_reset(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + unsigned long flags = device->pwrctrl.ctrl_flags; + + ret = a6xx_clear_pending_transactions(adreno_dev); + if (ret) + return ret; + + /* Clear ctrl_flags to ensure clocks and regulators are turned off */ + device->pwrctrl.ctrl_flags = 0; + + kgsl_pwrctrl_change_state(device, KGSL_STATE_INIT); + + /* since device is officially off now clear start bit */ + clear_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); + + a6xx_reset_preempt_records(adreno_dev); + + ret = adreno_start(device, 0); + if (ret) + return ret; + + kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); + + device->pwrctrl.ctrl_flags = flags; + + /* Toggle GX CPR on demand */ + a6xx_gx_cpr_toggle(device); + + /* + * If active_cnt is zero, there is no need to keep the GPU active. So, + * we should transition to SLUMBER. + */ + if (!atomic_read(&device->active_cnt)) + kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); + + return 0; +} + +static void a6xx_cp_hw_err_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int status1, status2; + + kgsl_regread(device, A6XX_CP_INTERRUPT_STATUS, &status1); + + if (status1 & BIT(A6XX_CP_OPCODE_ERROR)) { + unsigned int opcode; + + kgsl_regwrite(device, A6XX_CP_SQE_STAT_ADDR, 1); + kgsl_regread(device, A6XX_CP_SQE_STAT_DATA, &opcode); + dev_crit_ratelimited(device->dev, + "CP opcode error interrupt | opcode=0x%8.8x\n", opcode); + } + if (status1 & BIT(A6XX_CP_UCODE_ERROR)) + dev_crit_ratelimited(device->dev, "CP ucode error interrupt\n"); + if (status1 & BIT(A6XX_CP_HW_FAULT_ERROR)) { + kgsl_regread(device, A6XX_CP_HW_FAULT, &status2); + dev_crit_ratelimited(device->dev, + "CP | Ringbuffer HW fault | status=%x\n", status2); + } + if (status1 & BIT(A6XX_CP_REGISTER_PROTECTION_ERROR)) { + kgsl_regread(device, A6XX_CP_PROTECT_STATUS, &status2); + dev_crit_ratelimited(device->dev, + "CP | Protected mode error | %s | addr=%x | status=%x\n", + status2 & (1 << 20) ? "READ" : "WRITE", + status2 & 0x3FFFF, status2); + } + if (status1 & BIT(A6XX_CP_AHB_ERROR)) + dev_crit_ratelimited(device->dev, + "CP AHB error interrupt\n"); + if (status1 & BIT(A6XX_CP_VSD_PARITY_ERROR)) + dev_crit_ratelimited(device->dev, + "CP VSD decoder parity error\n"); + if (status1 & BIT(A6XX_CP_ILLEGAL_INSTR_ERROR)) + dev_crit_ratelimited(device->dev, + "CP Illegal instruction error\n"); + +} + +static void a6xx_err_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + switch (bit) { + case A6XX_INT_CP_AHB_ERROR: + dev_crit_ratelimited(device->dev, "CP: AHB bus error\n"); + break; + case A6XX_INT_ATB_ASYNCFIFO_OVERFLOW: + dev_crit_ratelimited(device->dev, + "RBBM: ATB ASYNC overflow\n"); + break; + case A6XX_INT_RBBM_ATB_BUS_OVERFLOW: + dev_crit_ratelimited(device->dev, + "RBBM: ATB bus overflow\n"); + break; + case A6XX_INT_UCHE_OOB_ACCESS: + dev_crit_ratelimited(device->dev, + "UCHE: Out of bounds access\n"); + break; + case A6XX_INT_UCHE_TRAP_INTR: + dev_crit_ratelimited(device->dev, "UCHE: Trap interrupt\n"); + break; + case A6XX_INT_TSB_WRITE_ERROR: + dev_crit_ratelimited(device->dev, "TSB: Write error interrupt\n"); + break; + default: + dev_crit_ratelimited(device->dev, "Unknown interrupt %d\n", + bit); + } +} + +/* + * a6xx_llc_configure_gpu_scid() - Program the sub-cache ID for all GPU blocks + * @adreno_dev: The adreno device pointer + */ +static void a6xx_llc_configure_gpu_scid(struct adreno_device *adreno_dev) +{ + uint32_t gpu_scid; + uint32_t gpu_cntl1_val = 0; + int i; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_mmu *mmu = &device->mmu; + + if (IS_ERR_OR_NULL(adreno_dev->gpu_llc_slice) || + !adreno_dev->gpu_llc_slice_enable) + return; + + if (llcc_slice_activate(adreno_dev->gpu_llc_slice)) + return; + + gpu_scid = llcc_get_slice_id(adreno_dev->gpu_llc_slice); + for (i = 0; i < A6XX_LLC_NUM_GPU_SCIDS; i++) + gpu_cntl1_val = (gpu_cntl1_val << A6XX_GPU_LLC_SCID_NUM_BITS) + | gpu_scid; + + if (mmu->subtype == KGSL_IOMMU_SMMU_V500) + kgsl_regrmw(device, A6XX_GBIF_SCACHE_CNTL1, + A6XX_GPU_LLC_SCID_MASK, gpu_cntl1_val); + else + adreno_cx_misc_regrmw(adreno_dev, + A6XX_GPU_CX_MISC_SYSTEM_CACHE_CNTL_1, + A6XX_GPU_LLC_SCID_MASK, gpu_cntl1_val); + + /* + * On A660, the SCID programming for UCHE traffic is done in + * A6XX_GBIF_SCACHE_CNTL0[14:10] + * GFO ENABLE BIT(8) : LLC uses a 64 byte cache line size enabling + * GFO allows it allocate partial cache lines + */ + if (adreno_is_a660(adreno_dev) || + adreno_is_a663(adreno_dev)) + kgsl_regrmw(device, A6XX_GBIF_SCACHE_CNTL0, (0x1f << 10) | + BIT(8), (gpu_scid << 10) | BIT(8)); +} + +/* + * a6xx_llc_configure_gpuhtw_scid() - Program the SCID for GPU pagetables + * @adreno_dev: The adreno device pointer + */ +static void a6xx_llc_configure_gpuhtw_scid(struct adreno_device *adreno_dev) +{ + uint32_t gpuhtw_scid; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_mmu *mmu = &device->mmu; + + if (IS_ERR_OR_NULL(adreno_dev->gpuhtw_llc_slice) || + !adreno_dev->gpuhtw_llc_slice_enable) + return; + + if (llcc_slice_activate(adreno_dev->gpuhtw_llc_slice)) + return; + + /* + * On SMMU-v500, the GPUHTW SCID is configured via a NoC override in + * the XBL image. + */ + if (mmu->subtype == KGSL_IOMMU_SMMU_V500) + return; + + gpuhtw_scid = llcc_get_slice_id(adreno_dev->gpuhtw_llc_slice); + + adreno_cx_misc_regrmw(adreno_dev, + A6XX_GPU_CX_MISC_SYSTEM_CACHE_CNTL_1, + A6XX_GPUHTW_LLC_SCID_MASK, + gpuhtw_scid << A6XX_GPUHTW_LLC_SCID_SHIFT); +} + +/* + * a6xx_llc_enable_overrides() - Override the page attributes + * @adreno_dev: The adreno device pointer + */ +static void a6xx_llc_enable_overrides(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_mmu *mmu = &device->mmu; + + /* + * Attributes override through GBIF is not supported with MMU-500. + * Attributes are used as configured through SMMU pagetable entries. + */ + if (mmu->subtype == KGSL_IOMMU_SMMU_V500) + return; + + /* + * 0x3: readnoallocoverrideen=0 + * read-no-alloc=0 - Allocate lines on read miss + * writenoallocoverrideen=1 + * write-no-alloc=1 - Do not allocates lines on write miss + */ + adreno_cx_misc_regwrite(adreno_dev, + A6XX_GPU_CX_MISC_SYSTEM_CACHE_CNTL_0, 0x3); +} + +static const char *uche_client[7][3] = { + {"SP | VSC | VPC | HLSQ | PC | LRZ", "TP", "VFD"}, + {"VSC | VPC | HLSQ | PC | LRZ", "TP | VFD", "SP"}, + {"SP | VPC | HLSQ | PC | LRZ", "TP | VFD", "VSC"}, + {"SP | VSC | HLSQ | PC | LRZ", "TP | VFD", "VPC"}, + {"SP | VSC | VPC | PC | LRZ", "TP | VFD", "HLSQ"}, + {"SP | VSC | VPC | HLSQ | LRZ", "TP | VFD", "PC"}, + {"SP | VSC | VPC | HLSQ | PC", "TP | VFD", "LRZ"}, +}; + +static const char *const uche_client_a660[] = { "VFD", "SP", "VSC", "VPC", + "HLSQ", "PC", "LRZ", "TP" }; + +#define SCOOBYDOO 0x5c00bd00 + +static const char *a6xx_fault_block_uche(struct kgsl_device *device, + unsigned int mid) +{ + unsigned int uche_client_id = 0; + static char str[40]; + + /* + * Smmu driver takes a vote on CX gdsc before calling the kgsl + * pagefault handler. If there is contention for device mutex in this + * path and the dispatcher fault handler is holding this lock, trying + * to turn off CX gdsc will fail during the reset. So to avoid blocking + * here, try to lock device mutex and return if it fails. + */ + if (!mutex_trylock(&device->mutex)) + return "UCHE: unknown"; + + if (!kgsl_state_is_awake(device)) { + mutex_unlock(&device->mutex); + return "UCHE: unknown"; + } + + kgsl_regread(device, A6XX_UCHE_CLIENT_PF, &uche_client_id); + mutex_unlock(&device->mutex); + + /* Ignore the value if the gpu is in IFPC */ + if (uche_client_id == SCOOBYDOO) + return "UCHE: unknown"; + + if (adreno_is_a660(ADRENO_DEVICE(device))) { + + /* Mask is 7 bits for A660 */ + uche_client_id &= 0x7F; + if (uche_client_id >= ARRAY_SIZE(uche_client_a660) || + (mid == 2)) + return "UCHE: Unknown"; + + if (mid == 1) + snprintf(str, sizeof(str), "UCHE: Not %s", + uche_client_a660[uche_client_id]); + else if (mid == 3) + snprintf(str, sizeof(str), "UCHE: %s", + uche_client_a660[uche_client_id]); + } else { + uche_client_id &= A6XX_UCHE_CLIENT_PF_CLIENT_ID_MASK; + if (uche_client_id >= ARRAY_SIZE(uche_client)) + return "UCHE: Unknown"; + + snprintf(str, sizeof(str), "UCHE: %s", + uche_client[uche_client_id][mid - 1]); + } + + return str; +} + +static const char *a6xx_iommu_fault_block(struct kgsl_device *device, + unsigned int fsynr1) +{ + unsigned int mid = fsynr1 & 0xff; + + switch (mid) { + case 0: + return "CP"; + case 1: + case 2: + case 3: + return a6xx_fault_block_uche(device, mid); + case 4: + return "CCU"; + case 6: + return "CDP Prefetch"; + case 7: + return "GPMU"; + } + + return "Unknown"; +} + +static void a6xx_cp_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (adreno_is_preemption_enabled(adreno_dev)) + a6xx_preemption_trigger(adreno_dev, true); + + adreno_dispatcher_schedule(device); +} + +/* + * a6xx_gpc_err_int_callback() - Isr for GPC error interrupts + * @adreno_dev: Pointer to device + * @bit: Interrupt bit + */ +static void a6xx_gpc_err_int_callback(struct adreno_device *adreno_dev, int bit) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* + * GPC error is typically the result of mistake SW programming. + * Force GPU fault for this interrupt so that we can debug it + * with help of register dump. + */ + + dev_crit(device->dev, "RBBM: GPC error\n"); + adreno_irqctrl(adreno_dev, 0); + + /* Trigger a fault in the dispatcher - this will effect a restart */ + adreno_dispatcher_fault(adreno_dev, ADRENO_SOFT_FAULT); +} + +static const struct adreno_irq_funcs a6xx_irq_funcs[32] = { + ADRENO_IRQ_CALLBACK(NULL), /* 0 - RBBM_GPU_IDLE */ + ADRENO_IRQ_CALLBACK(a6xx_err_callback), /* 1 - RBBM_AHB_ERROR */ + ADRENO_IRQ_CALLBACK(NULL), /* 2 - UNUSED */ + ADRENO_IRQ_CALLBACK(NULL), /* 3 - UNUSED */ + ADRENO_IRQ_CALLBACK(NULL), /* 4 - UNUSED */ + ADRENO_IRQ_CALLBACK(NULL), /* 5 - UNUSED */ + /* 6 - RBBM_ATB_ASYNC_OVERFLOW */ + ADRENO_IRQ_CALLBACK(a6xx_err_callback), + ADRENO_IRQ_CALLBACK(a6xx_gpc_err_int_callback), /* 7 - GPC_ERR */ + ADRENO_IRQ_CALLBACK(a6xx_preemption_callback),/* 8 - CP_SW */ + ADRENO_IRQ_CALLBACK(a6xx_cp_hw_err_callback), /* 9 - CP_HW_ERROR */ + ADRENO_IRQ_CALLBACK(NULL), /* 10 - CP_CCU_FLUSH_DEPTH_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 11 - CP_CCU_FLUSH_COLOR_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 12 - CP_CCU_RESOLVE_TS */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 13 - CP_IB2_INT */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 14 - CP_IB1_INT */ + ADRENO_IRQ_CALLBACK(adreno_cp_callback), /* 15 - CP_RB_INT */ + ADRENO_IRQ_CALLBACK(NULL), /* 16 - UNUSED */ + ADRENO_IRQ_CALLBACK(NULL), /* 17 - CP_RB_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 18 - CP_WT_DONE_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 19 - UNUSED */ + ADRENO_IRQ_CALLBACK(a6xx_cp_callback), /* 20 - CP_CACHE_FLUSH_TS */ + ADRENO_IRQ_CALLBACK(NULL), /* 21 - UNUSED */ + ADRENO_IRQ_CALLBACK(a6xx_err_callback), /* 22 - RBBM_ATB_BUS_OVERFLOW */ + /* 23 - MISC_HANG_DETECT */ + ADRENO_IRQ_CALLBACK(adreno_hang_int_callback), + ADRENO_IRQ_CALLBACK(a6xx_err_callback), /* 24 - UCHE_OOB_ACCESS */ + ADRENO_IRQ_CALLBACK(a6xx_err_callback), /* 25 - UCHE_TRAP_INTR */ + ADRENO_IRQ_CALLBACK(NULL), /* 26 - DEBBUS_INTR_0 */ + ADRENO_IRQ_CALLBACK(NULL), /* 27 - DEBBUS_INTR_1 */ + ADRENO_IRQ_CALLBACK(a6xx_err_callback), /* 28 - TSBWRITEERROR */ + ADRENO_IRQ_CALLBACK(NULL), /* 29 - UNUSED */ + ADRENO_IRQ_CALLBACK(NULL), /* 30 - ISDB_CPU_IRQ */ + ADRENO_IRQ_CALLBACK(NULL), /* 31 - ISDB_UNDER_DEBUG */ +}; + +/* + * If the AHB fence is not in ALLOW mode when we receive an RBBM + * interrupt, something went wrong. This means that we cannot proceed + * since the IRQ status and clear registers are not accessible. + * This is usually harmless because the GMU will abort power collapse + * and change the fence back to ALLOW. Poll so that this can happen. + */ +static int a6xx_irq_poll_fence(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 status, fence, fence_retries = 0; + u64 a, b, c; + + if (!gmu_core_isenabled(device)) + return 0; + + a = a6xx_read_alwayson(adreno_dev); + + kgsl_regread(device, A6XX_GMU_AO_AHB_FENCE_CTRL, &fence); + + while (fence != 0) { + b = a6xx_read_alwayson(adreno_dev); + + /* Wait for small time before trying again */ + udelay(1); + kgsl_regread(device, A6XX_GMU_AO_AHB_FENCE_CTRL, &fence); + + if (fence_retries == 100 && fence != 0) { + c = a6xx_read_alwayson(adreno_dev); + + kgsl_regread(device, A6XX_GMU_RBBM_INT_UNMASKED_STATUS, + &status); + + dev_crit_ratelimited(device->dev, + "status=0x%x Unmasked status=0x%x Mask=0x%x timestamps: %llx %llx %llx\n", + status & adreno_dev->irq_mask, status, + adreno_dev->irq_mask, a, b, c); + return -ETIMEDOUT; + } + + fence_retries++; + } + + return 0; +} + +static irqreturn_t a6xx_irq_handler(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + irqreturn_t ret = IRQ_NONE; + u32 status; + + /* + * On A6xx, the GPU can power down once the INT_0_STATUS is read + * below. But there still might be some register reads required + * so force the GMU/GPU into KEEPALIVE mode until done with the ISR. + */ + a6xx_gpu_keepalive(adreno_dev, true); + + if (a6xx_irq_poll_fence(adreno_dev)) { + adreno_dispatcher_fault(adreno_dev, ADRENO_GMU_FAULT); + goto done; + } + + kgsl_regread(device, A6XX_RBBM_INT_0_STATUS, &status); + + kgsl_regwrite(device, A6XX_RBBM_INT_CLEAR_CMD, status); + + ret = adreno_irq_callbacks(adreno_dev, a6xx_irq_funcs, status); + + trace_kgsl_a5xx_irq_status(adreno_dev, status); + +done: + /* If hard fault, then let snapshot turn off the keepalive */ + if (!(adreno_gpu_fault(adreno_dev) & ADRENO_HARD_FAULT)) + a6xx_gpu_keepalive(adreno_dev, false); + + return ret; +} + +int a6xx_probe_common(struct platform_device *pdev, + struct adreno_device *adreno_dev, u32 chipid, + const struct adreno_gpu_core *gpucore) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_gpudev *gpudev = gpucore->gpudev; + int ret; + + adreno_dev->gpucore = gpucore; + adreno_dev->chipid = chipid; + + adreno_reg_offset_init(gpudev->reg_offsets); + + if (gmu_core_isenabled(device) && (gpudev != &adreno_a6xx_rgmu_gpudev)) + device->pwrctrl.cx_cfg_gdsc_offset = (adreno_is_a662(adreno_dev) || + adreno_is_a621(adreno_dev)) ? A662_GPU_CC_CX_CFG_GDSCR : + A6XX_GPU_CC_CX_CFG_GDSCR; + + adreno_dev->hwcg_enabled = true; + adreno_dev->uche_client_pf = 1; + + adreno_dev->preempt.preempt_level = 1; + adreno_dev->preempt.skipsaverestore = true; + adreno_dev->preempt.usesgmem = true; + + ret = adreno_device_probe(pdev, adreno_dev); + + if (ret) + return ret; + + a6xx_coresight_init(adreno_dev); + return 0; +} + +static int a6xx_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore) +{ + struct adreno_device *adreno_dev; + struct kgsl_device *device; + int ret; + + adreno_dev = (struct adreno_device *) + of_device_get_match_data(&pdev->dev); + + memset(adreno_dev, 0, sizeof(*adreno_dev)); + + adreno_dev->irq_mask = A6XX_INT_MASK; + + ret = a6xx_probe_common(pdev, adreno_dev, chipid, gpucore); + if (ret) + return ret; + + ret = adreno_dispatcher_init(adreno_dev); + if (ret) + return ret; + + device = KGSL_DEVICE(adreno_dev); + + timer_setup(&device->idle_timer, kgsl_timer, 0); + + INIT_WORK(&device->idle_check_ws, kgsl_idle_check); + + return 0; +} + +/* Register offset defines for A6XX, in order of enum adreno_regs */ +static unsigned int a6xx_register_offsets[ADRENO_REG_REGISTER_MAX] = { + + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE, A6XX_CP_RB_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE_HI, A6XX_CP_RB_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_LO, + A6XX_CP_RB_RPTR_ADDR_LO), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_HI, + A6XX_CP_RB_RPTR_ADDR_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR, A6XX_CP_RB_RPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_WPTR, A6XX_CP_RB_WPTR), + ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_CNTL, A6XX_CP_RB_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_CNTL, A6XX_CP_SQE_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE, A6XX_CP_IB1_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE_HI, A6XX_CP_IB1_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BUFSZ, A6XX_CP_IB1_REM_SIZE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE, A6XX_CP_IB2_BASE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE_HI, A6XX_CP_IB2_BASE_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BUFSZ, A6XX_CP_IB2_REM_SIZE), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT, A6XX_CP_CONTEXT_SWITCH_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_LO, + A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_HI, + A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI), + ADRENO_REG_DEFINE( + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO, + A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO), + ADRENO_REG_DEFINE( + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI, + A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI), + ADRENO_REG_DEFINE( + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO, + A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO), + ADRENO_REG_DEFINE( + ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI, + A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO, + A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO), + ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI, + A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI), + ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT_LEVEL_STATUS, + A6XX_CP_CONTEXT_SWITCH_LEVEL_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS, A6XX_RBBM_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS3, A6XX_RBBM_STATUS3), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_MASK, A6XX_RBBM_INT_0_MASK), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_CLOCK_CTL, A6XX_RBBM_CLOCK_CNTL), + ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SW_RESET_CMD, A6XX_RBBM_SW_RESET_CMD), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_AO_HOST_INTERRUPT_MASK, + A6XX_GMU_AO_HOST_INTERRUPT_MASK), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_AHB_FENCE_STATUS, + A6XX_GMU_AHB_FENCE_STATUS), + ADRENO_REG_DEFINE(ADRENO_REG_GMU_GMU2HOST_INTR_MASK, + A6XX_GMU_GMU2HOST_INTR_MASK), +}; + +int a6xx_perfcounter_update(struct adreno_device *adreno_dev, + struct adreno_perfcount_register *reg, bool update_reg) +{ + void *ptr = adreno_dev->pwrup_reglist->hostptr; + struct cpu_gpu_lock *lock = ptr; + u32 *data = ptr + sizeof(*lock); + int i, offset = 0; + bool select_reg_present = false; + + for (i = 0; i < lock->list_length >> 1; i++) { + if (data[offset] == reg->select) { + select_reg_present = true; + break; + } + + if (data[offset] == A6XX_RBBM_PERFCTR_CNTL) + break; + + offset += 2; + } + + if (kgsl_hwlock(lock)) { + kgsl_hwunlock(lock); + return -EBUSY; + } + + /* + * If the perfcounter select register is already present in reglist + * update it, otherwise append the