/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2021, The Linux Foundation. All rights reserved. * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _ADRENO_GEN7_HWSCHED_HFI_H_ #define _ADRENO_GEN7_HWSCHED_HFI_H_ /* Maximum number of IBs in a submission */ #define HWSCHED_MAX_NUMIBS \ ((HFI_MAX_MSG_SIZE - offsetof(struct hfi_issue_cmd_cmd, ibs)) \ / sizeof(struct hfi_issue_ib)) /* * This is used to put userspace threads to sleep when hardware fence unack count reaches a * threshold. This bit is cleared in two scenarios: * 1. If the hardware fence unack count drops to a desired threshold. * 2. If there is a GMU/GPU fault. Because we don't want the threads to keep sleeping through fault * recovery, which can easily take 100s of milliseconds to complete. */ #define GEN7_HWSCHED_HW_FENCE_SLEEP_BIT 0x0 /* * This is used to avoid creating any more hardware fences until the hardware fence unack count * drops to a desired threshold. This bit is required in cases where GEN7_HWSCHED_HW_FENCE_SLEEP_BIT * will be cleared, but we still want to avoid creating any more hardware fences. For example, if * hardware fence unack count reaches a maximum threshold, both GEN7_HWSCHED_HW_FENCE_SLEEP_BIT and * GEN7_HWSCHED_HW_FENCE_MAX_BIT will be set. Say, a GMU/GPU fault happens and * GEN7_HWSCHED_HW_FENCE_SLEEP_BIT will be cleared to wake up any sleeping threads. But, * GEN7_HWSCHED_HW_FENCE_MAX_BIT will remain set to avoid creating any new hardware fences until * recovery is complete and deferred drawctxt (if any) is handled. */ #define GEN7_HWSCHED_HW_FENCE_MAX_BIT 0x1 /* * This is used to avoid creating any more hardware fences until concurrent reset/recovery completes */ #define GEN7_HWSCHED_HW_FENCE_ABORT_BIT 0x2 struct gen7_hwsched_hfi { struct hfi_mem_alloc_entry mem_alloc_table[32]; u32 mem_alloc_entries; /** @irq_mask: Store the hfi interrupt mask */ u32 irq_mask; /** @msglock: To protect the list of un-ACKed hfi packets */ rwlock_t msglock; /** @msglist: List of un-ACKed hfi packets */ struct list_head msglist; /** @f2h_task: Task for processing gmu fw to host packets */ struct task_struct *f2h_task; /** @f2h_wq: Waitqueue for the f2h_task */ wait_queue_head_t f2h_wq; /** @big_ib: GMU buffer to hold big IBs */ struct kgsl_memdesc *big_ib; /** @big_ib_recurring: GMU buffer to hold big recurring IBs */ struct kgsl_memdesc *big_ib_recurring; /** @perfctr_scratch: Buffer to hold perfcounter PM4 commands */ struct kgsl_memdesc *perfctr_scratch; /** @msg_mutex: Mutex for accessing the msgq */ struct mutex msgq_mutex; struct { /** @lock: Spinlock for managing hardware fences */ spinlock_t lock; /** * @unack_count: Number of hardware fences sent to GMU but haven't yet been ack'd * by GMU */ u32 unack_count; /** * @unack_wq: Waitqueue to wait on till number of unacked hardware fences drops to * a desired threshold */ wait_queue_head_t unack_wq; /** * @defer_drawctxt: Drawctxt to send hardware fences from as soon as unacked * hardware fences drops to a desired threshold */ struct adreno_context *defer_drawctxt; /** * @defer_ts: The timestamp of the hardware fence which got deferred */ u32 defer_ts; /** * @flags: Flags to control the creation of new hardware fences */ unsigned long flags; /** @seqnum: Sequence number for hardware fence packet header */ atomic_t seqnum; } hw_fence; /** * @hw_fence_timer: Timer to trigger fault if unack'd hardware fence count does'nt drop * to a desired threshold in given amount of time */ struct timer_list hw_fence_timer; /** * @hw_fence_ws: Work struct that gets scheduled when hw_fence_timer expires */ struct work_struct hw_fence_ws; /** @detached_hw_fences_list: List of hardware fences belonging to detached contexts */ struct list_head detached_hw_fence_list; /** @defer_hw_fence_work: The work structure to send deferred hardware fences to GMU */ struct kthread_work defer_hw_fence_work; }; struct kgsl_drawobj_cmd; /** * gen7_hwsched_hfi_probe - Probe hwsched hfi resources * @adreno_dev: Pointer to adreno device structure * * Return: 0 on success and negative error on failure. */ int gen7_hwsched_hfi_probe(struct adreno_device *adreno_dev); /** * gen7_hwsched_hfi_remove - Release hwsched hfi resources * @adreno_dev: Pointer to adreno device structure */ void gen7_hwsched_hfi_remove(struct adreno_device *adreno_dev); /** * gen7_hwsched_hfi_init - Initialize hfi resources * @adreno_dev: Pointer to adreno device structure * * This function is used to initialize hfi resources * once before the very first gmu boot * * Return: 0 on success and negative error on failure. */ int gen7_hwsched_hfi_init(struct adreno_device *adreno_dev); /** * gen7_hwsched_hfi_start - Start hfi resources * @adreno_dev: Pointer to adreno device structure * * Send the various hfi packets before booting the gpu * * Return: 0 on success and negative error on failure. */ int gen7_hwsched_hfi_start(struct adreno_device *adreno_dev); /** * gen7_hwsched_hfi_stop - Stop the hfi resources * @adreno_dev: Pointer to the adreno device * * This function does the hfi cleanup when powering down the gmu */ void gen7_hwsched_hfi_stop(struct adreno_device *adreno_dev); /** * gen7_hwched_cp_init - Send CP_INIT via HFI * @adreno_dev: Pointer to adreno device structure * * This function is used to send CP INIT packet and bring * GPU out of secure mode using hfi raw packets. * * Return: 0 on success and negative error on failure. */ int gen7_hwsched_cp_init(struct adreno_device *adreno_dev); /** * gen7_hwsched_counter_inline_enable - Configure a performance counter for a countable * @adreno_dev - Adreno device to configure * @group - Desired performance counter group * @counter - Desired performance counter in the group * @countable - Desired countable * * Physically set up a counter within a group with the desired countable * Return 0 on success or negative error on failure. */ int gen7_hwsched_counter_inline_enable(struct adreno_device *adreno_dev, const struct adreno_perfcount_group *group, u32 counter, u32 countable); /** * gen7_hfi_send_cmd_async - Send an hfi packet * @adreno_dev: Pointer to adreno device structure * @data: Data to be sent in the hfi packet * @size_bytes: Size of the packet in bytes * * Send data in the form of an HFI packet to gmu and wait for * it's ack asynchronously * * Return: 0 on success and negative error on failure. */ int gen7_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data, u32 size_bytes); /** * gen7_hwsched_submit_drawobj - Dispatch IBs to dispatch queues * @adreno_dev: Pointer to adreno device structure * @drawobj: The command draw object which needs to be submitted * * This function is used to register the context if needed and submit * IBs to the hfi dispatch queues. * Return: 0 on success and negative error on failure */ int gen7_hwsched_submit_drawobj(struct adreno_device *adreno_dev, struct kgsl_drawobj *drawobj); /** * gen7_hwsched_context_detach - Unregister a context with GMU * @drawctxt: Pointer to the adreno context * * This function sends context unregister HFI and waits for the ack * to ensure all submissions from this context have retired */ void gen7_hwsched_context_detach(struct adreno_context *drawctxt); /* Helper function to get to gen7 hwsched hfi device from adreno device */ struct gen7_hwsched_hfi *to_gen7_hwsched_hfi(struct adreno_device *adreno_dev); /** * gen7_hwsched_preempt_count_get - Get preemption count from GMU * @adreno_dev: Pointer to adreno device * * This function sends a GET_VALUE HFI packet to get the number of * preemptions completed since last SLUMBER exit. * * Return: Preemption count */ u32 gen7_hwsched_preempt_count_get(struct adreno_device *adreno_dev); /** * gen7_hwsched_lpac_cp_init - Send CP_INIT to LPAC via HFI * @adreno_dev: Pointer to adreno device structure * * This function is used to send CP INIT packet to LPAC and * enable submission to LPAC queue. * * Return: 0 on success and negative error on failure. */ int gen7_hwsched_lpac_cp_init(struct adreno_device *adreno_dev); /** * gen7_hfi_send_lpac_feature_ctrl - Send the lpac feature hfi packet * @adreno_dev: Pointer to the adreno device * * Return: 0 on success or negative error on failure */ int gen7_hfi_send_lpac_feature_ctrl(struct adreno_device *adreno_dev); /** * gen7_hwsched_context_destroy - Destroy any hwsched related resources during context destruction * @adreno_dev: Pointer to adreno device * @drawctxt: Pointer to the adreno context * * This functions destroys any hwsched related resources when this context is destroyed */ void gen7_hwsched_context_destroy(struct adreno_device *adreno_dev, struct adreno_context *drawctxt); /** * gen7_hwsched_hfi_get_value - Send GET_VALUE packet to GMU to get the value of a property * @adreno_dev: Pointer to adreno device * @prop: property to get from GMU * * This functions sends GET_VALUE HFI packet to query value of a property * * Return: On success, return the value in the GMU response. On failure, return 0 */ u32 gen7_hwsched_hfi_get_value(struct adreno_device *adreno_dev, u32 prop); /** * gen7_send_hw_fence_hfi_wait_ack - Send hardware fence info to GMU * @adreno_dev: Pointer to adreno device * @entry: Pointer to the adreno hardware fence entry * @flags: Flags for this hardware fence * * Send the hardware fence info to the GMU and wait for the ack * * Return: 0 on success or negative error on failure */ int gen7_send_hw_fence_hfi_wait_ack(struct adreno_device *adreno_dev, struct adreno_hw_fence_entry *entry, u64 flags); /** * gen7_hwsched_create_hw_fence - Create a hardware fence * @adreno_dev: Pointer to adreno device * @kfence: Pointer to the kgsl fence * * Create a hardware fence, set up hardware fence info and send it to GMU if required */ void gen7_hwsched_create_hw_fence(struct adreno_device *adreno_dev, struct kgsl_sync_fence *kfence); /** * gen7_hwsched_drain_context_hw_fences - Drain context's hardware fences via GMU * @adreno_dev: Pointer to adreno device * @drawctxt: Pointer to the adreno context which is to be flushed * * Trigger hardware fences that were never dispatched to GMU * * Return: Zero on success or negative error on failure */ int gen7_hwsched_drain_context_hw_fences(struct adreno_device *adreno_dev, struct adreno_context *drawctxt); /** * gen7_hwsched_check_context_inflight_hw_fences - Check whether all hardware fences * from a context have been sent to the TxQueue or not * @adreno_dev: Pointer to adreno device * @drawctxt: Pointer to the adreno context which is to be flushed * * Check if all hardware fences from this context have been sent to the * TxQueue. If not, log an error and return error code. * * Return: Zero on success or negative error on failure */ int gen7_hwsched_check_context_inflight_hw_fences(struct adreno_device *adreno_dev, struct adreno_context *drawctxt); /** * gen7_remove_hw_fence_entry - Remove hardware fence entry * @adreno_dev: pointer to the adreno device * @entry: Pointer to the hardware fence entry */ void gen7_remove_hw_fence_entry(struct adreno_device *adreno_dev, struct adreno_hw_fence_entry *entry); /** * gen7_trigger_hw_fence_cpu - Trigger hardware fence from cpu * @adreno_dev: pointer to the adreno device * @fence: hardware fence entry to be triggered * * Trigger the hardware fence by sending it to GMU's TxQueue and raise the * interrupt from GMU to APPS */ void gen7_trigger_hw_fence_cpu(struct adreno_device *adreno_dev, struct adreno_hw_fence_entry *fence); /** * gen7_hwsched_disable_hw_fence_throttle - Disable hardware fence throttling after reset * @adreno_dev: pointer to the adreno device * * After device reset, clear hardware fence related data structures and send any hardware fences * that got deferred (prior to reset) and re-open the gates for hardware fence creation * * Return: Zero on success or negative error on failure */ int gen7_hwsched_disable_hw_fence_throttle(struct adreno_device *adreno_dev); /** * gen7_hwsched_process_msgq - Process msgq * @adreno_dev: pointer to the adreno device * * This function grabs the msgq mutex and processes msgq for any outstanding hfi packets */ void gen7_hwsched_process_msgq(struct adreno_device *adreno_dev); /** * gen7_hwsched_boot_gpu - Send the command to boot GPU * @adreno_dev: Pointer to adreno device * * Send the hfi to boot GPU, and check the ack, incase of a failure * get a snapshot and capture registers of interest. * * Return: Zero on success or negative error on failure */ int gen7_hwsched_boot_gpu(struct adreno_device *adreno_dev); #endif