android_kernel_samsung_sm8650-modules/qcom/opensource/graphics-kernel/adreno_gen7_hwsched_hfi.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2021, The Linux Foundation. All rights reserved.
 * Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved.
 */

#include <dt-bindings/soc/qcom,ipcc.h>
#include <linux/dma-fence-array.h>
#include <linux/iommu.h>
#include <linux/sched/clock.h>
#include <linux/soc/qcom/msm_hw_fence.h>
#include <soc/qcom/msm_performance.h>

#include "adreno.h"
#include "adreno_gen7.h"
#include "adreno_gen7_hwsched.h"
#include "adreno_hfi.h"
#include "adreno_pm4types.h"
#include "adreno_trace.h"
#include "kgsl_device.h"
#include "kgsl_eventlog.h"
#include "kgsl_pwrctrl.h"
#include "kgsl_trace.h"
#include "kgsl_util.h"

#define HFI_QUEUE_MAX (HFI_QUEUE_DEFAULT_CNT + HFI_QUEUE_DISPATCH_MAX_CNT)

#define DEFINE_QHDR(gmuaddr, id, prio) \
	{\
		.status = 1, \
		.start_addr = GMU_QUEUE_START_ADDR(gmuaddr, id), \
		.type = QUEUE_HDR_TYPE(id, prio, 0, 0), \
		.queue_size = SZ_4K >> 2, \
		.msg_size = 0, \
		.unused0 = 0, \
		.unused1 = 0, \
		.unused2 = 0, \
		.unused3 = 0, \
		.unused4 = 0, \
		.read_index = 0, \
		.write_index = 0, \
}

static struct dq_info {
	/** @max_dq: Maximum number of dispatch queues per RB level */
	u32 max_dq;
	/** @base_dq_id: Base dqid for level */
	u32 base_dq_id;
	/** @offset: Next dqid to use for roundrobin context assignment */
	u32 offset;
} gen7_hfi_dqs[KGSL_PRIORITY_MAX_RB_LEVELS] = {
	{ 4, 0, }, /* RB0 */
	{ 4, 4, }, /* RB1 */
	{ 3, 8, }, /* RB2 */
	{ 3, 11, }, /* RB3 */
}, gen7_hfi_dqs_lpac[KGSL_PRIORITY_MAX_RB_LEVELS + 1] = {
	{ 4, 0, }, /* RB0 */
	{ 4, 4, }, /* RB1 */
	{ 3, 8, }, /* RB2 */
	{ 2, 11, }, /* RB3 */
	{ 1, 13, }, /* RB LPAC */
};

struct pending_cmd hw_fence_ack;

struct gen7_hwsched_hfi *to_gen7_hwsched_hfi(
	struct adreno_device *adreno_dev)
{
	struct gen7_device *gen7_dev = container_of(adreno_dev,
					struct gen7_device, adreno_dev);
	struct gen7_hwsched_device *gen7_hwsched = container_of(gen7_dev,
					struct gen7_hwsched_device, gen7_dev);

	return &gen7_hwsched->hwsched_hfi;
}

int gen7_hfi_send_lpac_feature_ctrl(struct adreno_device *adreno_dev)
{
	if (!adreno_dev->lpac_enabled)
		return 0;

	return gen7_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_LPAC, 1, 0);
}

static void add_waiter(struct gen7_hwsched_hfi *hfi, u32 hdr,
	struct pending_cmd *ack)
{
	memset(ack, 0x0, sizeof(*ack));

	init_completion(&ack->complete);
	write_lock_irq(&hfi->msglock);
	list_add_tail(&ack->node, &hfi->msglist);
	write_unlock_irq(&hfi->msglock);

	ack->sent_hdr = hdr;
}

static void del_waiter(struct gen7_hwsched_hfi *hfi, struct pending_cmd *ack)
{
	write_lock_irq(&hfi->msglock);
	list_del(&ack->node);
	write_unlock_irq(&hfi->msglock);
}

static void gen7_receive_ack_async(struct adreno_device *adreno_dev, void *rcvd)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct pending_cmd *cmd = NULL;
	u32 waiters[64], num_waiters = 0, i;
	u32 *ack = rcvd;
	u32 hdr = ack[0];
	u32 req_hdr = ack[1];
	u32 size_bytes = MSG_HDR_GET_SIZE(hdr) << 2;

	if (size_bytes > sizeof(cmd->results))
		dev_err_ratelimited(&gmu->pdev->dev,
			"Ack result too big: %d Truncating to: %ld\n",
			size_bytes, sizeof(cmd->results));

	read_lock(&hfi->msglock);

	list_for_each_entry(cmd, &hfi->msglist, node) {
		if (CMP_HFI_ACK_HDR(cmd->sent_hdr, req_hdr)) {
			memcpy(cmd->results, ack,
				min_t(u32, size_bytes,
					sizeof(cmd->results)));
			complete(&cmd->complete);
			read_unlock(&hfi->msglock);
			return;
		}

		if (num_waiters < ARRAY_SIZE(waiters))
			waiters[num_waiters++] = cmd->sent_hdr;
	}

	read_unlock(&hfi->msglock);

	/* Didn't find the sender, list the waiter */
	dev_err_ratelimited(&gmu->pdev->dev,
		"Unexpectedly got id %d seqnum %d. Total waiters: %d Top %d Waiters:\n",
		MSG_HDR_GET_ID(req_hdr), MSG_HDR_GET_SEQNUM(req_hdr),
		num_waiters, min_t(u32, num_waiters, 5));

	for (i = 0; i < num_waiters && i < 5; i++)
		dev_err_ratelimited(&gmu->pdev->dev,
			" id %d seqnum %d\n",
			MSG_HDR_GET_ID(waiters[i]),
			MSG_HDR_GET_SEQNUM(waiters[i]));
}

/* This function is called while holding the drawctxt spinlock */
void gen7_remove_hw_fence_entry(struct adreno_device *adreno_dev,
	struct adreno_hw_fence_entry *entry)
{
	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
	struct adreno_context *drawctxt = entry->drawctxt;

	atomic_dec(&hwsched->hw_fence_count);
	drawctxt->hw_fence_count--;

	dma_fence_put(&entry->kfence->fence);
	list_del_init(&entry->node);
	kmem_cache_free(hwsched->hw_fence_cache, entry);
}

static void _retire_inflight_hw_fences(struct adreno_device *adreno_dev,
	struct kgsl_context *context)
{
	struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
	struct adreno_hw_fence_entry *entry, *tmp;

	if (!test_bit(ADRENO_HWSCHED_HW_FENCE, &adreno_dev->hwsched.flags))
		return;

	spin_lock(&drawctxt->lock);

	list_for_each_entry_safe(entry, tmp, &drawctxt->hw_fence_inflight_list, node) {
		struct gmu_context_queue_header *hdr = drawctxt->gmu_context_queue.hostptr;

		/*
		 * Since this list is sorted by timestamp, abort on the first fence that hasn't
		 * yet been sent to TxQueue
		 */
		if (timestamp_cmp((u32)entry->cmd.ts, hdr->out_fence_ts) > 0)
			break;

		gen7_remove_hw_fence_entry(adreno_dev, entry);
	}
	spin_unlock(&drawctxt->lock);
}

static void log_profiling_info(struct adreno_device *adreno_dev, u32 *rcvd)
{
	struct hfi_ts_retire_cmd *cmd = (struct hfi_ts_retire_cmd *)rcvd;
	struct kgsl_context *context;
	struct retire_info info = {0};
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

	context = kgsl_context_get(device, cmd->ctxt_id);
	if (context == NULL)
		return;

	info.timestamp = cmd->ts;
	info.rb_id = adreno_get_level(context);
	info.gmu_dispatch_queue = context->gmu_dispatch_queue;
	info.submitted_to_rb = cmd->submitted_to_rb;
	info.sop = cmd->sop;
	info.eop = cmd->eop;
	if (GMU_VER_MINOR(gmu->ver.hfi) < 4)
		info.active = cmd->eop - cmd->sop;
	else
		info.active = cmd->active;
	info.retired_on_gmu = cmd->retired_on_gmu;

	/* protected GPU work must not be reported */
	if  (!(context->flags & KGSL_CONTEXT_SECURE))
		kgsl_work_period_update(device, context->proc_priv->period,
					     info.active);

	trace_adreno_cmdbatch_retired(context, &info, 0, 0, 0);

	log_kgsl_cmdbatch_retired_event(context->id, cmd->ts,
		context->priority, 0, cmd->sop, cmd->eop);

	_retire_inflight_hw_fences(adreno_dev, context);

	kgsl_context_put(context);
}

/* Look up a particular key's value for a given type of payload */
static u32 gen7_hwsched_lookup_key_value_legacy(struct adreno_device *adreno_dev,
	u32 type, u32 key)
{
	struct hfi_context_bad_cmd_legacy *cmd = adreno_dev->hwsched.ctxt_bad;
	u32 i = 0, payload_bytes;
	void *start;

	if (!cmd->hdr)
		return 0;

	payload_bytes = (MSG_HDR_GET_SIZE(cmd->hdr) << 2) -
			offsetof(struct hfi_context_bad_cmd_legacy, payload);

	start = &cmd->payload[0];

	while (i < payload_bytes) {
		struct payload_section *payload = start + i;

		if (payload->type == type)
			return adreno_hwsched_parse_payload(payload, key);

		i += struct_size(payload, data, payload->dwords);
	}

	return 0;
}

static u32 get_payload_rb_key_legacy(struct adreno_device *adreno_dev,
	u32 rb_id, u32 key)
{
	struct hfi_context_bad_cmd_legacy *cmd = adreno_dev->hwsched.ctxt_bad;
	u32 i = 0, payload_bytes;
	void *start;

	if (!cmd->hdr)
		return 0;

	payload_bytes = (MSG_HDR_GET_SIZE(cmd->hdr) << 2) -
			offsetof(struct hfi_context_bad_cmd_legacy, payload);

	start = &cmd->payload[0];

	while (i < payload_bytes) {
		struct payload_section *payload = start + i;

		if (payload->type == PAYLOAD_RB) {
			u32 id = adreno_hwsched_parse_payload(payload, KEY_RB_ID);

			if (id == rb_id)
				return adreno_hwsched_parse_payload(payload, key);
		}

		i += struct_size(payload, data, payload->dwords);
	}

	return 0;
}

struct syncobj_flags {
	unsigned long mask;
	const char *name;
};

static void _get_syncobj_string(char *str, u32 max_size, struct hfi_syncobj *syncobj, u32 index)
{
	u32 count = scnprintf(str, max_size, "syncobj[%d] ctxt_id:%llu seqno:%llu flags:", index,
			syncobj->ctxt_id, syncobj->seq_no);
	u32 i;
	bool first = true;
	static const struct syncobj_flags _flags[] = {
		GMU_SYNCOBJ_FLAGS, { -1, NULL }};

	for (i = 0; _flags[i].name; i++) {
		if (!(syncobj->flags & _flags[i].mask))
			continue;

		if (first) {
			count += scnprintf(str + count, max_size - count, "%s", _flags[i].name);
			first = false;
		} else {
			count += scnprintf(str + count, max_size - count, "|%s", _flags[i].name);
		}
	}
}

static void log_syncobj(struct gen7_gmu_device *gmu, struct hfi_submit_syncobj *cmd)
{
	struct hfi_syncobj *syncobj = (struct hfi_syncobj *)&cmd[1];
	char str[128];
	u32 i = 0;

	for (i = 0; i < cmd->num_syncobj; i++) {
		_get_syncobj_string(str, sizeof(str), syncobj, i);
		dev_err(&gmu->pdev->dev, "%s\n", str);
		syncobj++;
	}
}

static void find_timeout_syncobj(struct adreno_device *adreno_dev, u32 ctxt_id, u32 ts)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct kgsl_context *context = NULL;
	struct adreno_context *drawctxt;
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	struct gmu_context_queue_header *hdr;
	struct hfi_submit_syncobj *cmd;
	u32 *queue, i;
	int ret;

	/* We want to get the context even if it is detached */
	read_lock(&device->context_lock);
	context = idr_find(&device->context_idr, ctxt_id);
	ret = _kgsl_context_get(context);
	read_unlock(&device->context_lock);

	if (!ret)
		return;

	drawctxt = ADRENO_CONTEXT(context);

	hdr = drawctxt->gmu_context_queue.hostptr;
	queue = (u32 *)(drawctxt->gmu_context_queue.hostptr + sizeof(*hdr));

	for (i = hdr->read_index; i != hdr->write_index;) {
		if (MSG_HDR_GET_ID(queue[i]) != H2F_MSG_ISSUE_SYNCOBJ) {
			i = (i + MSG_HDR_GET_SIZE(queue[i])) % hdr->queue_size;
			continue;
		}

		cmd = (struct hfi_submit_syncobj *)&queue[i];

		if (cmd->timestamp == ts) {
			log_syncobj(gmu, cmd);
			break;
		}
		i = (i + MSG_HDR_GET_SIZE(queue[i])) % hdr->queue_size;
	}

	if (i == hdr->write_index)
		dev_err(&gmu->pdev->dev, "Couldn't find unsignaled syncobj ctx:%d ts:%d\n",
			ctxt_id, ts);

	kgsl_context_put(context);
}

static void log_gpu_fault_legacy(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct device *dev = &gmu->pdev->dev;
	struct hfi_context_bad_cmd_legacy *cmd = adreno_dev->hwsched.ctxt_bad;

	switch (cmd->error) {
	case GMU_GPU_HW_HANG:
		dev_crit_ratelimited(dev, "MISC: GPU hang detected\n");
		break;
	case GMU_GPU_SW_HANG:
		dev_crit_ratelimited(dev, "gpu timeout ctx %d ts %u\n",
			cmd->ctxt_id, cmd->ts);
		break;
	case GMU_CP_OPCODE_ERROR:
		dev_crit_ratelimited(dev,
			"CP opcode error interrupt | opcode=0x%8.8x\n",
			gen7_hwsched_lookup_key_value_legacy(adreno_dev, PAYLOAD_FAULT_REGS,
			KEY_CP_OPCODE_ERROR));
		break;
	case GMU_CP_PROTECTED_ERROR: {
		u32 status = gen7_hwsched_lookup_key_value_legacy(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_CP_PROTECTED_ERROR);

		dev_crit_ratelimited(dev,
			"CP | Protected mode error | %s | addr=0x%5.5x | status=0x%8.8x\n",
			status & (1 << 20) ? "READ" : "WRITE",
			status & 0x3FFFF, status);
		}
		break;
	case GMU_CP_ILLEGAL_INST_ERROR:
		dev_crit_ratelimited(dev, "CP Illegal instruction error\n");
		break;
	case GMU_CP_UCODE_ERROR:
		dev_crit_ratelimited(dev, "CP ucode error interrupt\n");
		break;
	case GMU_CP_HW_FAULT_ERROR:
		dev_crit_ratelimited(dev,
			"CP | Ringbuffer HW fault | status=0x%8.8x\n",
			gen7_hwsched_lookup_key_value_legacy(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_CP_HW_FAULT));
		break;
	case GMU_GPU_PREEMPT_TIMEOUT: {
		u32 cur, next, cur_rptr, cur_wptr, next_rptr, next_wptr;

		cur = gen7_hwsched_lookup_key_value_legacy(adreno_dev,
			PAYLOAD_PREEMPT_TIMEOUT, KEY_PREEMPT_TIMEOUT_CUR_RB_ID);
		next = gen7_hwsched_lookup_key_value_legacy(adreno_dev,
			PAYLOAD_PREEMPT_TIMEOUT,
			KEY_PREEMPT_TIMEOUT_NEXT_RB_ID);
		cur_rptr = get_payload_rb_key_legacy(adreno_dev, cur, KEY_RB_RPTR);
		cur_wptr = get_payload_rb_key_legacy(adreno_dev, cur, KEY_RB_WPTR);
		next_rptr = get_payload_rb_key_legacy(adreno_dev, next, KEY_RB_RPTR);
		next_wptr = get_payload_rb_key_legacy(adreno_dev, next, KEY_RB_WPTR);

		dev_crit_ratelimited(dev,
			"Preemption Fault: cur=%d R/W=0x%x/0x%x, next=%d R/W=0x%x/0x%x\n",
			cur, cur_rptr, cur_wptr, next, next_rptr, next_wptr);
		}
		break;
	case GMU_CP_GPC_ERROR:
		dev_crit_ratelimited(dev, "RBBM: GPC error\n");
		break;
	case GMU_CP_BV_OPCODE_ERROR:
		dev_crit_ratelimited(dev,
			"CP BV opcode error | opcode=0x%8.8x\n",
			gen7_hwsched_lookup_key_value_legacy(adreno_dev, PAYLOAD_FAULT_REGS,
			KEY_CP_BV_OPCODE_ERROR));
		break;
	case GMU_CP_BV_PROTECTED_ERROR: {
		u32 status = gen7_hwsched_lookup_key_value_legacy(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_CP_BV_PROTECTED_ERROR);

		dev_crit_ratelimited(dev,
			"CP BV | Protected mode error | %s | addr=0x%5.5x | status=0x%8.8x\n",
			status & (1 << 20) ? "READ" : "WRITE",
			status & 0x3FFFF, status);
		}
		break;
	case GMU_CP_BV_HW_FAULT_ERROR:
		dev_crit_ratelimited(dev,
			"CP BV | Ringbuffer HW fault | status=0x%8.8x\n",
			gen7_hwsched_lookup_key_value_legacy(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_CP_HW_FAULT));
		break;
	case GMU_CP_BV_ILLEGAL_INST_ERROR:
		dev_crit_ratelimited(dev, "CP BV Illegal instruction error\n");
		break;
	case GMU_CP_BV_UCODE_ERROR:
		dev_crit_ratelimited(dev, "CP BV ucode error interrupt\n");
		break;
	case GMU_GPU_SW_FUSE_VIOLATION:
		dev_crit_ratelimited(dev, "RBBM: SW Feature Fuse violation status=0x%8.8x\n",
			gen7_hwsched_lookup_key_value_legacy(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_SWFUSE_VIOLATION_FAULT));
		break;
	case GMU_GPU_AQE0_OPCODE_ERRROR:
		dev_crit_ratelimited(dev, "AQE0 opcode error | opcode=0x%8.8x\n",
			gen7_hwsched_lookup_key_value_legacy(adreno_dev,
				PAYLOAD_FAULT_REGS, KEY_AQE0_OPCODE_ERROR));
		break;
	case GMU_GPU_AQE0_UCODE_ERROR:
		dev_crit_ratelimited(dev, "AQE0 ucode error interrupt\n");
		break;
	case GMU_GPU_AQE0_HW_FAULT_ERROR:
		dev_crit_ratelimited(dev, "AQE0 HW fault | status=0x%8.8x\n",
			gen7_hwsched_lookup_key_value_legacy(adreno_dev,
				PAYLOAD_FAULT_REGS, KEY_AQE0_HW_FAULT));
		break;
	case GMU_GPU_AQE0_ILLEGAL_INST_ERROR:
		dev_crit_ratelimited(dev, "AQE0 Illegal instruction error\n");
		break;
	case GMU_GPU_AQE1_OPCODE_ERRROR:
		dev_crit_ratelimited(dev, "AQE1 opcode error | opcode=0x%8.8x\n",
			gen7_hwsched_lookup_key_value_legacy(adreno_dev,
				PAYLOAD_FAULT_REGS, KEY_AQE1_OPCODE_ERROR));
		break;
	case GMU_GPU_AQE1_UCODE_ERROR:
		dev_crit_ratelimited(dev, "AQE1 ucode error interrupt\n");
		break;
	case GMU_GPU_AQE1_HW_FAULT_ERROR:
		dev_crit_ratelimited(dev, "AQE1 HW fault | status=0x%8.8x\n",
			gen7_hwsched_lookup_key_value_legacy(adreno_dev,
				PAYLOAD_FAULT_REGS, KEY_AQE1_HW_FAULT));
		break;
	case GMU_GPU_AQE1_ILLEGAL_INST_ERROR:
		dev_crit_ratelimited(dev, "AQE1 Illegal instruction error\n");
		break;
	case GMU_SYNCOBJ_TIMEOUT_ERROR:
		dev_crit_ratelimited(dev, "syncobj timeout ctx %d ts %u\n",
			cmd->ctxt_id, cmd->ts);
		find_timeout_syncobj(adreno_dev, cmd->ctxt_id, cmd->ts);
		break;
	case GMU_CP_UNKNOWN_ERROR:
		fallthrough;
	default:
		dev_crit_ratelimited(dev, "Unknown GPU fault: %u\n",
			cmd->error);
		break;
	}
}

/* Look up a particular key's value for a given type of payload */
static u32 gen7_hwsched_lookup_key_value(struct adreno_device *adreno_dev,
	u32 type, u32 key)
{
	struct hfi_context_bad_cmd *cmd = adreno_dev->hwsched.ctxt_bad;
	u32 i = 0, payload_bytes;
	void *start;

	if (!cmd->hdr)
		return 0;

	payload_bytes = (MSG_HDR_GET_SIZE(cmd->hdr) << 2) -
			offsetof(struct hfi_context_bad_cmd, payload);

	start = &cmd->payload[0];

	while (i < payload_bytes) {
		struct payload_section *payload = start + i;

		if (payload->type == type)
			return adreno_hwsched_parse_payload(payload, key);

		i += struct_size(payload, data, payload->dwords);
	}

	return 0;
}

static u32 get_payload_rb_key(struct adreno_device *adreno_dev,
	u32 rb_id, u32 key)
{
	struct hfi_context_bad_cmd *cmd = adreno_dev->hwsched.ctxt_bad;
	u32 i = 0, payload_bytes;
	void *start;

	if (!cmd->hdr)
		return 0;

	payload_bytes = (MSG_HDR_GET_SIZE(cmd->hdr) << 2) -
			offsetof(struct hfi_context_bad_cmd, payload);

	start = &cmd->payload[0];

	while (i < payload_bytes) {
		struct payload_section *payload = start + i;

		if (payload->type == PAYLOAD_RB) {
			u32 id = adreno_hwsched_parse_payload(payload, KEY_RB_ID);

			if (id == rb_id)
				return adreno_hwsched_parse_payload(payload, key);
		}

		i += struct_size(payload, data, payload->dwords);
	}

	return 0;
}

static bool log_gpu_fault(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct device *dev = &gmu->pdev->dev;
	struct hfi_context_bad_cmd *cmd = adreno_dev->hwsched.ctxt_bad;

	/* Return false for non fatal errors */
	if (adreno_hwsched_log_nonfatal_gpu_fault(adreno_dev, dev, cmd->error))
		return false;

	switch (cmd->error) {
	case GMU_GPU_HW_HANG:
		dev_crit_ratelimited(dev, "MISC: GPU hang detected\n");
		break;
	case GMU_GPU_SW_HANG:
		dev_crit_ratelimited(dev, "gpu timeout ctx %d ts %d\n",
			cmd->gc.ctxt_id, cmd->gc.ts);
		break;
	case GMU_CP_OPCODE_ERROR:
		dev_crit_ratelimited(dev,
			"CP opcode error interrupt | opcode=0x%8.8x\n",
			gen7_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
			KEY_CP_OPCODE_ERROR));
		break;
	case GMU_CP_PROTECTED_ERROR: {
		u32 status = gen7_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_CP_PROTECTED_ERROR);

		dev_crit_ratelimited(dev,
			"CP | Protected mode error | %s | addr=0x%5.5x | status=0x%8.8x\n",
			status & (1 << 20) ? "READ" : "WRITE",
			status & 0x3FFFF, status);
		}
		break;
	case GMU_CP_ILLEGAL_INST_ERROR:
		dev_crit_ratelimited(dev, "CP Illegal instruction error\n");
		break;
	case GMU_CP_UCODE_ERROR:
		dev_crit_ratelimited(dev, "CP ucode error interrupt\n");
		break;
	case GMU_CP_HW_FAULT_ERROR:
		dev_crit_ratelimited(dev,
			"CP | Ringbuffer HW fault | status=0x%8.8x\n",
			gen7_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_CP_HW_FAULT));
		break;
	case GMU_GPU_PREEMPT_TIMEOUT: {
		u32 cur, next, cur_rptr, cur_wptr, next_rptr, next_wptr;

		cur = gen7_hwsched_lookup_key_value(adreno_dev,
			PAYLOAD_PREEMPT_TIMEOUT, KEY_PREEMPT_TIMEOUT_CUR_RB_ID);
		next = gen7_hwsched_lookup_key_value(adreno_dev,
			PAYLOAD_PREEMPT_TIMEOUT,
			KEY_PREEMPT_TIMEOUT_NEXT_RB_ID);
		cur_rptr = get_payload_rb_key(adreno_dev, cur, KEY_RB_RPTR);
		cur_wptr = get_payload_rb_key(adreno_dev, cur, KEY_RB_WPTR);
		next_rptr = get_payload_rb_key(adreno_dev, next, KEY_RB_RPTR);
		next_wptr = get_payload_rb_key(adreno_dev, next, KEY_RB_WPTR);

		dev_crit_ratelimited(dev,
			"Preemption Fault: cur=%d R/W=0x%x/0x%x, next=%d R/W=0x%x/0x%x\n",
			cur, cur_rptr, cur_wptr, next, next_rptr, next_wptr);
		}
		break;
	case GMU_CP_GPC_ERROR:
		dev_crit_ratelimited(dev, "RBBM: GPC error\n");
		break;
	case GMU_CP_BV_OPCODE_ERROR:
		dev_crit_ratelimited(dev,
			"CP BV opcode error | opcode=0x%8.8x\n",
			gen7_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
			KEY_CP_BV_OPCODE_ERROR));
		break;
	case GMU_CP_BV_PROTECTED_ERROR: {
		u32 status = gen7_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_CP_BV_PROTECTED_ERROR);

		dev_crit_ratelimited(dev,
			"CP BV | Protected mode error | %s | addr=0x%5.5x | status=0x%8.8x\n",
			status & (1 << 20) ? "READ" : "WRITE",
			status & 0x3FFFF, status);
		}
		break;
	case GMU_CP_BV_HW_FAULT_ERROR:
		dev_crit_ratelimited(dev,
			"CP BV | Ringbuffer HW fault | status=0x%8.8x\n",
			gen7_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_CP_HW_FAULT));
		break;
	case GMU_CP_BV_ILLEGAL_INST_ERROR:
		dev_crit_ratelimited(dev, "CP BV Illegal instruction error\n");
		break;
	case GMU_CP_BV_UCODE_ERROR:
		dev_crit_ratelimited(dev, "CP BV ucode error interrupt\n");
		break;
	case GMU_CP_LPAC_OPCODE_ERROR:
		dev_crit_ratelimited(dev,
			"CP LPAC opcode error | opcode=0x%8.8x\n",
			gen7_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
			KEY_CP_LPAC_OPCODE_ERROR));
		break;
	case GMU_CP_LPAC_PROTECTED_ERROR: {
		u32 status = gen7_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_CP_LPAC_PROTECTED_ERROR);

		dev_crit_ratelimited(dev,
			"CP LPAC | Protected mode error | %s | addr=0x%5.5x | status=0x%8.8x\n",
			status & (1 << 20) ? "READ" : "WRITE",
			status & 0x3FFFF, status);
		}
		break;
	case GMU_CP_LPAC_HW_FAULT_ERROR:
		dev_crit_ratelimited(dev,
			"CP LPAC | Ringbuffer HW fault | status=0x%8.8x\n",
			gen7_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_CP_LPAC_HW_FAULT));
		break;
	case GMU_CP_LPAC_ILLEGAL_INST_ERROR:
		dev_crit_ratelimited(dev, "CP LPAC Illegal instruction error\n");
		break;
	case GMU_CP_LPAC_UCODE_ERROR:
		dev_crit_ratelimited(dev, "CP LPAC ucode error interrupt\n");
		break;
	case GMU_GPU_LPAC_SW_HANG:
		dev_crit_ratelimited(dev, "LPAC: gpu timeout ctx %d ts %d\n",
			cmd->lpac.ctxt_id, cmd->lpac.ts);
		break;
	case GMU_GPU_SW_FUSE_VIOLATION:
		dev_crit_ratelimited(dev, "RBBM: SW Feature Fuse violation status=0x%8.8x\n",
			gen7_hwsched_lookup_key_value(adreno_dev, PAYLOAD_FAULT_REGS,
				KEY_SWFUSE_VIOLATION_FAULT));
		break;
	case GMU_GPU_AQE0_OPCODE_ERRROR:
		dev_crit_ratelimited(dev, "AQE0 opcode error | opcode=0x%8.8x\n",
			gen7_hwsched_lookup_key_value(adreno_dev,
				PAYLOAD_FAULT_REGS, KEY_AQE0_OPCODE_ERROR));
		break;
	case GMU_GPU_AQE0_UCODE_ERROR:
		dev_crit_ratelimited(dev, "AQE0 ucode error interrupt\n");
		break;
	case GMU_GPU_AQE0_HW_FAULT_ERROR:
		dev_crit_ratelimited(dev, "AQE0 HW fault | status=0x%8.8x\n",
			gen7_hwsched_lookup_key_value(adreno_dev,
				PAYLOAD_FAULT_REGS, KEY_AQE0_HW_FAULT));
		break;
	case GMU_GPU_AQE0_ILLEGAL_INST_ERROR:
		dev_crit_ratelimited(dev, "AQE0 Illegal instruction error\n");
		break;
	case GMU_GPU_AQE1_OPCODE_ERRROR:
		dev_crit_ratelimited(dev, "AQE1 opcode error | opcode=0x%8.8x\n",
			gen7_hwsched_lookup_key_value(adreno_dev,
				PAYLOAD_FAULT_REGS, KEY_AQE1_OPCODE_ERROR));
		break;
	case GMU_GPU_AQE1_UCODE_ERROR:
		dev_crit_ratelimited(dev, "AQE1 ucode error interrupt\n");
		break;
	case GMU_GPU_AQE1_HW_FAULT_ERROR:
		dev_crit_ratelimited(dev, "AQE1 HW fault | status=0x%8.8x\n",
			gen7_hwsched_lookup_key_value(adreno_dev,
				PAYLOAD_FAULT_REGS, KEY_AQE1_HW_FAULT));
		break;
	case GMU_GPU_AQE1_ILLEGAL_INST_ERROR:
		dev_crit_ratelimited(dev, "AQE1 Illegal instruction error\n");
		break;
	case GMU_SYNCOBJ_TIMEOUT_ERROR:
		dev_crit_ratelimited(dev, "syncobj timeout ctx %d ts %u\n",
			cmd->gc.ctxt_id, cmd->gc.ts);
		find_timeout_syncobj(adreno_dev, cmd->gc.ctxt_id, cmd->gc.ts);
		break;
	case GMU_CP_UNKNOWN_ERROR:
		fallthrough;
	default:
		dev_crit_ratelimited(dev, "Unknown GPU fault: %u\n",
			cmd->error);
		break;
	}

	/* Return true for fatal errors to perform recovery sequence */
	return true;
}

static u32 peek_next_header(struct gen7_gmu_device *gmu, uint32_t queue_idx)
{
	struct kgsl_memdesc *mem_addr = gmu->hfi.hfi_mem;
	struct hfi_queue_table *tbl = mem_addr->hostptr;
	struct hfi_queue_header *hdr = &tbl->qhdr[queue_idx];
	u32 *queue;

	if (hdr->status == HFI_QUEUE_STATUS_DISABLED)
		return 0;

	if (hdr->read_index == hdr->write_index)
		return 0;

	queue = HOST_QUEUE_START_ADDR(mem_addr, queue_idx);

	return queue[hdr->read_index];
}

static void process_ctx_bad(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);

	if (GMU_VER_MINOR(gmu->ver.hfi) < 2) {
		log_gpu_fault_legacy(adreno_dev);
		goto done;
	}

	/* Non fatal RBBM error interrupts don't go through reset and recovery */
	if (!log_gpu_fault(adreno_dev)) {
		memset(adreno_dev->hwsched.ctxt_bad, 0x0, HFI_MAX_MSG_SIZE);
		return;
	}

done:
	gen7_hwsched_fault(adreno_dev, ADRENO_HARD_FAULT);
}

#define GET_QUERIED_FENCE_INDEX(x) (x / BITS_PER_SYNCOBJ_QUERY)
#define GET_QUERIED_FENCE_BIT(x) (x % BITS_PER_SYNCOBJ_QUERY)

static bool fence_is_queried(struct hfi_syncobj_query_cmd *cmd, u32 fence_index)
{
	u32 index = GET_QUERIED_FENCE_INDEX(fence_index);
	u32 bit = GET_QUERIED_FENCE_BIT(fence_index);

	return (cmd->queries[index].query_bitmask & BIT(bit));
}

static void set_fence_signal_bit(struct adreno_device *adreno_dev,
	struct hfi_syncobj_query_cmd *reply, struct dma_fence *fence, u32 fence_index,
	char *name)
{
	u32 index = GET_QUERIED_FENCE_INDEX(fence_index);
	u32 bit = GET_QUERIED_FENCE_BIT(fence_index);
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	u64 flags = ADRENO_HW_FENCE_SW_STATUS_PENDING;
	char value[32] = "unknown";

	if (fence->ops->timeline_value_str)
		fence->ops->timeline_value_str(fence, value, sizeof(value));

	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
		dev_err(&gmu->pdev->dev,
			"GMU is waiting for signaled fence(ctx:%llu seqno:%llu value:%s)\n",
			fence->context, fence->seqno, value);
		reply->queries[index].query_bitmask |= BIT(bit);
		flags = ADRENO_HW_FENCE_SW_STATUS_SIGNALED;
	}
	trace_adreno_hw_fence_query(fence->context, fence->seqno, flags, name, value);
}

static void gen7_syncobj_query_reply(struct adreno_device *adreno_dev,
	struct kgsl_drawobj *drawobj, struct hfi_syncobj_query_cmd *cmd)
{
	struct hfi_syncobj_query_cmd reply = {0};
	int i, j, fence_index = 0;
	struct kgsl_drawobj_sync *syncobj = SYNCOBJ(drawobj);
	const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);

	for (i = 0; i < syncobj->numsyncs; i++) {
		struct kgsl_drawobj_sync_event *event = &syncobj->synclist[i];
		struct kgsl_sync_fence_cb *kcb = event->handle;
		struct dma_fence **fences;
		struct dma_fence_array *array;
		struct event_fence_info *info = event->priv;
		u32 num_fences;

		array = to_dma_fence_array(kcb->fence);
		if (array != NULL) {
			num_fences = array->num_fences;
			fences = array->fences;
		} else {
			num_fences = 1;
			fences = &kcb->fence;
		}

		for (j = 0; j < num_fences; j++, fence_index++) {
			if (!fence_is_queried(cmd, fence_index))
				continue;

			set_fence_signal_bit(adreno_dev, &reply, fences[j], fence_index,
				info ? info->fences[j].name : "unknown");
		}
	}

	reply.hdr = CREATE_MSG_HDR(F2H_MSG_SYNCOBJ_QUERY, HFI_MSG_CMD);
	reply.gmu_ctxt_id = cmd->gmu_ctxt_id;
	reply.sync_obj_ts = cmd->sync_obj_ts;

	trace_adreno_syncobj_query_reply(reply.gmu_ctxt_id, reply.sync_obj_ts,
		gpudev->read_alwayson(adreno_dev));

	gen7_hfi_send_cmd_async(adreno_dev, &reply, sizeof(reply));
}

struct syncobj_query_work {
	/** @cmd: The query command to be processed */
	struct hfi_syncobj_query_cmd cmd;
	/** @context: kgsl context that is waiting for this sync object */
	struct kgsl_context *context;
	/** @work: The work structure to execute syncobj query reply */
	struct kthread_work work;
};

static void gen7_process_syncobj_query_work(struct kthread_work *work)
{
	struct syncobj_query_work *query_work = container_of(work,
						struct syncobj_query_work, work);
	struct hfi_syncobj_query_cmd *cmd = (struct hfi_syncobj_query_cmd *)&query_work->cmd;
	struct kgsl_context *context = query_work->context;
	struct kgsl_device *device = context->device;
	struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
	struct cmd_list_obj *obj;
	bool missing = true;

	mutex_lock(&hwsched->mutex);
	mutex_lock(&device->mutex);

	list_for_each_entry(obj, &hwsched->cmd_list, node) {
		struct kgsl_drawobj *drawobj = obj->drawobj;

		if ((drawobj->type & SYNCOBJ_TYPE) == 0)
			continue;

		if ((drawobj->context->id == cmd->gmu_ctxt_id) &&
			(drawobj->timestamp == cmd->sync_obj_ts)) {
			gen7_syncobj_query_reply(adreno_dev, drawobj, cmd);
			missing = false;
			break;
		}
	}

	if (missing) {
		struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
		struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
		struct gmu_context_queue_header *hdr = drawctxt->gmu_context_queue.hostptr;

		/*
		 * If the sync object is not found, it can only mean that the sync object was
		 * retired by the GMU in the meanwhile. However, if that is not the case, then
		 * we have a problem.
		 */
		if (timestamp_cmp(cmd->sync_obj_ts, hdr->sync_obj_ts) > 0) {
			dev_err(&gmu->pdev->dev, "Missing sync object ctx:%d ts:%d retired:%d\n",
				context->id, cmd->sync_obj_ts, hdr->sync_obj_ts);
			gmu_core_fault_snapshot(device);
			gen7_hwsched_fault(adreno_dev, ADRENO_GMU_FAULT);
		}
	}

	mutex_unlock(&device->mutex);
	mutex_unlock(&hwsched->mutex);

	kgsl_context_put(context);
	kfree(query_work);
}

static void gen7_trigger_syncobj_query(struct adreno_device *adreno_dev,
	u32 *rcvd)
{
	struct syncobj_query_work *query_work;
	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
	struct hfi_syncobj_query_cmd *cmd = (struct hfi_syncobj_query_cmd *)rcvd;
	struct kgsl_context *context = NULL;
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
	int ret;

	trace_adreno_syncobj_query(cmd->gmu_ctxt_id, cmd->sync_obj_ts,
		gpudev->read_alwayson(adreno_dev));

	/*
	 * We need the context even if it is detached. Hence, we can't use kgsl_context_get here.
	 * We must make sure that this context id doesn't get destroyed (to avoid re-use) until GMU
	 * has ack'd the query reply.
	 */
	read_lock(&device->context_lock);
	context = idr_find(&device->context_idr, cmd->gmu_ctxt_id);
	ret = _kgsl_context_get(context);
	read_unlock(&device->context_lock);

	if (!ret)
		return;

	query_work = kzalloc(sizeof(*query_work), GFP_KERNEL);
	if (!query_work) {
		kgsl_context_put(context);
		return;
	}

	kthread_init_work(&query_work->work, gen7_process_syncobj_query_work);
	memcpy(&query_work->cmd, cmd, sizeof(*cmd));
	query_work->context = context;

	kthread_queue_work(hwsched->worker, &query_work->work);
}

/*
 * This defines the maximum unack'd hardware fences that we allow. When this limit is reached, we
 * will put all threads (that want to create a hardware fence) to sleep until the maximum unack'd
 * hardware fence count drops to MIN_HW_FENCE_UNACK_COUNT
 */
#define MAX_HW_FENCE_UNACK_COUNT 20

/*
 * Once the maximum unack'd hardware fences drops to this value, wake up all the threads (that want
 * to create hardware fences)
 */
#define MIN_HW_FENCE_UNACK_COUNT 10

/*
 * This is the maximum duration (in milliseconds) a thread (that wants to create a hardware fence)
 * is put to sleep while we wait for the maximum number of unack'd hardware fences to drop from
 * MAX_HW_FENCE_UNACK_COUNT to MIN_HW_FENCE_UNACK_COUNT. If the count doesn't drop to the desired
 * value, then log an error and trigger snapshot and recovery.
 */
#define HW_FENCE_SLEEP_MS 200

static void _enable_hw_fence_throttle(struct adreno_device *adreno_dev)
{
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);

	set_bit(GEN7_HWSCHED_HW_FENCE_SLEEP_BIT, &hfi->hw_fence.flags);
	set_bit(GEN7_HWSCHED_HW_FENCE_MAX_BIT, &hfi->hw_fence.flags);

	/* Avoid submitting new work to gpu until the unack count drops to a desired threshold */
	adreno_get_gpu_halt(adreno_dev);

	mod_timer(&hfi->hw_fence_timer, jiffies + msecs_to_jiffies(HW_FENCE_SLEEP_MS));
}

static void _increment_hw_fence_unack_count(struct adreno_device *adreno_dev)
{
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);

	if ((++hfi->hw_fence.unack_count) == MAX_HW_FENCE_UNACK_COUNT)
		_enable_hw_fence_throttle(adreno_dev);
}

/**
 * _send_hw_fence_no_ack - Send a hardware fence hfi packet to GMU without waiting for its ack.
 * Increment the unack count on success
 *
 * Return: 0 on success or negative error on failure
 */
static int _send_hw_fence_no_ack(struct adreno_device *adreno_dev,
	struct adreno_hw_fence_entry *entry)
{
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	u32 seqnum;
	int ret;


	seqnum = atomic_inc_return(&hfi->hw_fence.seqnum);
	entry->cmd.hdr = MSG_HDR_SET_SEQNUM_SIZE(entry->cmd.hdr, seqnum, sizeof(entry->cmd) >> 2);

	ret = gen7_hfi_cmdq_write(adreno_dev, (u32 *)&entry->cmd, sizeof(entry->cmd));
	if (!ret)
		_increment_hw_fence_unack_count(adreno_dev);

	return ret;
}

static struct adreno_hw_fence_entry *_get_deferred_hw_fence(struct adreno_context *drawctxt, u32 ts)
{
	struct adreno_hw_fence_entry *entry = NULL, *next, *deferred_hw_fence_entry = NULL;

	spin_lock(&drawctxt->lock);
	list_for_each_entry_safe(entry, next, &drawctxt->hw_fence_list, node) {

		if (timestamp_cmp((u32)entry->cmd.ts, ts) > 0)
			break;

		/* We found a deferred hardware fence */
		deferred_hw_fence_entry = entry;
		break;
	}
	spin_unlock(&drawctxt->lock);

	/*
	 * This path executes in isolation from any paths that may release this entry. So, it is
	 * safe to handle this entry outside of the drawctxt spinlock
	 */
	return deferred_hw_fence_entry;
}

static int _send_deferred_hw_fence(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt, struct adreno_hw_fence_entry *entry, u32 ts)
{
	bool retired = kgsl_check_timestamp(KGSL_DEVICE(adreno_dev), &drawctxt->base, ts) ||
				kgsl_context_is_bad(&drawctxt->base);
	int ret = 0;
	u32 flags = 0;

	if (retired)
		flags |= HW_FENCE_FLAG_SKIP_MEMSTORE;

	ret = gen7_send_hw_fence_hfi_wait_ack(adreno_dev, entry, flags);
	if (ret)
		return ret;

	spin_lock(&drawctxt->lock);
	if (!retired)
		list_move_tail(&entry->node, &drawctxt->hw_fence_inflight_list);
	else
		gen7_remove_hw_fence_entry(adreno_dev, entry);
	spin_unlock(&drawctxt->lock);

	return 0;
}

/**
 * process_hw_fence_deferred_ctxt - This function sends hardware fences to GMU (from the
 * deferred drawctxt) which couldn't be sent earlier
 */
static int process_hw_fence_deferred_ctxt(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt, u32 ts)
{
	struct adreno_hw_fence_entry *deferred_hw_fence_entry = NULL;
	int ret = 0;

	do {
		deferred_hw_fence_entry = _get_deferred_hw_fence(drawctxt, ts);

		if (!deferred_hw_fence_entry)
			break;

		ret = _send_deferred_hw_fence(adreno_dev, drawctxt, deferred_hw_fence_entry, ts);
		if (ret)
			break;

	} while (deferred_hw_fence_entry != NULL);

	return ret;
}

static void _disable_hw_fence_throttle(struct adreno_device *adreno_dev, bool clear_abort_bit)
{
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	bool max;

	spin_lock(&hfi->hw_fence.lock);

	hfi->hw_fence.defer_drawctxt = NULL;
	hfi->hw_fence.defer_ts = 0;
	max = test_bit(GEN7_HWSCHED_HW_FENCE_MAX_BIT, &hfi->hw_fence.flags);
	if (max) {
		clear_bit(GEN7_HWSCHED_HW_FENCE_SLEEP_BIT, &hfi->hw_fence.flags);
		clear_bit(GEN7_HWSCHED_HW_FENCE_MAX_BIT, &hfi->hw_fence.flags);
	}

	if (clear_abort_bit)
		clear_bit(GEN7_HWSCHED_HW_FENCE_ABORT_BIT, &hfi->hw_fence.flags);
	spin_unlock(&hfi->hw_fence.lock);

	/* Wake up dispatcher and any sleeping threads that want to create hardware fences */
	if (max) {
		adreno_put_gpu_halt(adreno_dev);
		adreno_hwsched_trigger(adreno_dev);
		wake_up_all(&hfi->hw_fence.unack_wq);
	}
}

static void gen7_defer_hw_fence_work(struct kthread_work *work)
{
	struct gen7_hwsched_hfi *hfi = container_of(work,
						struct gen7_hwsched_hfi, defer_hw_fence_work);
	struct adreno_context *drawctxt = NULL;
	struct kgsl_device *device;
	struct adreno_device *adreno_dev;
	u32 ts;
	int ret;

	spin_lock(&hfi->hw_fence.lock);
	drawctxt = hfi->hw_fence.defer_drawctxt;
	ts = hfi->hw_fence.defer_ts;
	spin_unlock(&hfi->hw_fence.lock);

	device = drawctxt->base.device;
	adreno_dev = ADRENO_DEVICE(device);

	/*
	 * Grab the dispatcher and device mutex as we don't want to race with concurrent fault
	 * recovery
	 */
	mutex_lock(&adreno_dev->hwsched.mutex);
	mutex_lock(&device->mutex);

	ret = process_hw_fence_deferred_ctxt(adreno_dev, drawctxt, ts);
	if (ret) {
		/* the deferred drawctxt will be handled post fault recovery */
		gen7_hwsched_fault(adreno_dev, ADRENO_GMU_FAULT);
		goto unlock;
	}

	/*
	 * Put back the context reference which was incremented when hw_fence.defer_drawctxt was set
	 */
	kgsl_context_put(&drawctxt->base);

	gen7_hwsched_active_count_put(adreno_dev);

	_disable_hw_fence_throttle(adreno_dev, false);

unlock:
	mutex_unlock(&device->mutex);
	mutex_unlock(&adreno_dev->hwsched.mutex);
}

static void process_hw_fence_ack(struct adreno_device *adreno_dev, u32 received_hdr)
{
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct adreno_context *drawctxt = NULL;
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);

	spin_lock(&hfi->hw_fence.lock);

	/* If this ack is being waited on, we don't need to touch the unack count */
	if (hw_fence_ack.sent_hdr && CMP_HFI_ACK_HDR(hw_fence_ack.sent_hdr, received_hdr)) {
		spin_unlock(&hfi->hw_fence.lock);
		complete(&hw_fence_ack.complete);
		return;
	}

	hfi->hw_fence.unack_count--;

	/* The unack count should never be greater than MAX_HW_FENCE_UNACK_COUNT */
	if (hfi->hw_fence.unack_count > MAX_HW_FENCE_UNACK_COUNT)
		dev_err(&gmu->pdev->dev, "unexpected hardware fence unack count:%d\n",
			hfi->hw_fence.unack_count);

	if (!test_bit(GEN7_HWSCHED_HW_FENCE_MAX_BIT, &hfi->hw_fence.flags) ||
		(hfi->hw_fence.unack_count != MIN_HW_FENCE_UNACK_COUNT)) {
		spin_unlock(&hfi->hw_fence.lock);
		return;
	}

	drawctxt = hfi->hw_fence.defer_drawctxt;

	spin_unlock(&hfi->hw_fence.lock);

	del_timer_sync(&hfi->hw_fence_timer);

	/*
	 * We need to handle the deferred context in another thread so that we can unblock the f2h
	 * daemon here as it will need to process the acks for the hardware fences belonging to the
	 * deferred context
	 */
	if (drawctxt) {
		kthread_init_work(&hfi->defer_hw_fence_work, gen7_defer_hw_fence_work);
		kthread_queue_work(adreno_dev->hwsched.worker, &hfi->defer_hw_fence_work);
		return;
	}

	_disable_hw_fence_throttle(adreno_dev, false);
}

void gen7_hwsched_process_msgq(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct gen7_hwsched_hfi *hw_hfi = to_gen7_hwsched_hfi(adreno_dev);
	u32 rcvd[MAX_RCVD_SIZE], next_hdr, type;

	mutex_lock(&hw_hfi->msgq_mutex);

	for (;;) {
		next_hdr = peek_next_header(gmu, HFI_MSG_ID);

		if (!next_hdr)
			break;

		if (MSG_HDR_GET_TYPE(next_hdr) == HFI_MSG_ACK)
			type = HFI_MSG_ACK;
		else
			type = MSG_HDR_GET_ID(next_hdr);

		if (type != F2H_MSG_CONTEXT_BAD)
			gen7_hfi_queue_read(gmu, HFI_MSG_ID, rcvd, sizeof(rcvd));

		switch (type) {
		case HFI_MSG_ACK:
			/*
			 * We are assuming that there is only one outstanding ack because hfi
			 * sending thread waits for completion while holding the device mutex
			 * (except when we send H2F_MSG_HW_FENCE_INFO packets)
			 */
			if (MSG_HDR_GET_ID(rcvd[1]) == H2F_MSG_HW_FENCE_INFO)
				process_hw_fence_ack(adreno_dev, rcvd[1]);
			else
				gen7_receive_ack_async(adreno_dev, rcvd);
			break;
		case F2H_MSG_CONTEXT_BAD:
			gen7_hfi_queue_read(gmu, HFI_MSG_ID, (u32 *)adreno_dev->hwsched.ctxt_bad,
						HFI_MAX_MSG_SIZE);
			process_ctx_bad(adreno_dev);
			break;
		case F2H_MSG_TS_RETIRE:
			log_profiling_info(adreno_dev, rcvd);
			adreno_hwsched_trigger(adreno_dev);
			break;
		case F2H_MSG_SYNCOBJ_QUERY:
			gen7_trigger_syncobj_query(adreno_dev, rcvd);
			break;
		case F2H_MSG_GMU_CNTR_RELEASE: {
			struct hfi_gmu_cntr_release_cmd *cmd =
				(struct hfi_gmu_cntr_release_cmd *) rcvd;

			adreno_perfcounter_put(adreno_dev,
				cmd->group_id, cmd->countable, PERFCOUNTER_FLAG_KERNEL);
			}
			break;
		}
	}
	mutex_unlock(&hw_hfi->msgq_mutex);
}

static void process_log_block(struct adreno_device *adreno_dev, void *data)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct hfi_log_block *cmd = data;
	u32 *log_event = gmu->gmu_log->hostptr;
	u32 start, end;

	start = cmd->start_index;
	end = cmd->stop_index;

	log_event += start * 4;
	while (start != end) {
		trace_gmu_event(log_event);
		log_event += 4;
		start++;
	}
}

static void gen7_hwsched_process_dbgq(struct adreno_device *adreno_dev, bool limited)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	u32 rcvd[MAX_RCVD_SIZE];
	bool recovery = false;

	while (gen7_hfi_queue_read(gmu, HFI_DBG_ID, rcvd, sizeof(rcvd)) > 0) {

		if (MSG_HDR_GET_ID(rcvd[0]) == F2H_MSG_ERR) {
			adreno_gen7_receive_err_req(gmu, rcvd);
			recovery = true;
			break;
		}

		if (MSG_HDR_GET_ID(rcvd[0]) == F2H_MSG_DEBUG)
			adreno_gen7_receive_debug_req(gmu, rcvd);

		if (MSG_HDR_GET_ID(rcvd[0]) == F2H_MSG_LOG_BLOCK)
			process_log_block(adreno_dev, rcvd);

		/* Process one debug queue message and return to not delay msgq processing */
		if (limited)
			break;
	}

	if (!recovery)
		return;

	gen7_hwsched_fault(adreno_dev, ADRENO_GMU_FAULT);
}

/* HFI interrupt handler */
static irqreturn_t gen7_hwsched_hfi_handler(int irq, void *data)
{
	struct adreno_device *adreno_dev = data;
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	u32 status = 0;

	/*
	 * GEN7_GMU_GMU2HOST_INTR_INFO may have bits set not specified in hfi->irq_mask.
	 * Read and clear only those irq bits that we are processing here.
	 */
	gmu_core_regread(device, GEN7_GMU_GMU2HOST_INTR_INFO, &status);
	gmu_core_regwrite(device, GEN7_GMU_GMU2HOST_INTR_CLR, status & hfi->irq_mask);

	/*
	 * If interrupts are not enabled on the HFI message queue,
	 * the inline message processing loop will process it,
	 * else, process it here.
	 */
	if (!(hfi->irq_mask & HFI_IRQ_MSGQ_MASK))
		status &= ~HFI_IRQ_MSGQ_MASK;

	if (status & (HFI_IRQ_MSGQ_MASK | HFI_IRQ_DBGQ_MASK)) {
		wake_up_interruptible(&hfi->f2h_wq);
		adreno_hwsched_trigger(adreno_dev);
	}
	if (status & HFI_IRQ_CM3_FAULT_MASK) {
		atomic_set(&gmu->cm3_fault, 1);

		/* make sure other CPUs see the update */
		smp_wmb();

		dev_err_ratelimited(&gmu->pdev->dev,
				"GMU CM3 fault interrupt received\n");

		gen7_hwsched_fault(adreno_dev, ADRENO_GMU_FAULT);
	}

	/* Ignore OOB bits */
	status &= GENMASK(31 - (oob_max - 1), 0);

	if (status & ~hfi->irq_mask)
		dev_err_ratelimited(&gmu->pdev->dev,
			"Unhandled HFI interrupts 0x%x\n",
			status & ~hfi->irq_mask);

	return IRQ_HANDLED;
}

#define HFI_IRQ_MSGQ_MASK BIT(0)

static int check_ack_failure(struct adreno_device *adreno_dev,
	struct pending_cmd *ack)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);

	if (ack->results[2] != 0xffffffff)
		return 0;

	dev_err(&gmu->pdev->dev,
		"ACK error: sender id %d seqnum %d\n",
		MSG_HDR_GET_ID(ack->sent_hdr),
		MSG_HDR_GET_SEQNUM(ack->sent_hdr));

	return -EINVAL;
}

int gen7_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data, u32 size_bytes)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	u32 *cmd = data;
	u32 seqnum;
	int rc;
	struct pending_cmd pending_ack;

	seqnum = atomic_inc_return(&gmu->hfi.seqnum);
	*cmd = MSG_HDR_SET_SEQNUM_SIZE(*cmd, seqnum, size_bytes >> 2);

	add_waiter(hfi, *cmd, &pending_ack);

	rc = gen7_hfi_cmdq_write(adreno_dev, cmd, size_bytes);
	if (rc)
		goto done;

	rc = adreno_hwsched_wait_ack_completion(adreno_dev, &gmu->pdev->dev, &pending_ack,
		gen7_hwsched_process_msgq);
	if (rc)
		goto done;

	rc = check_ack_failure(adreno_dev, &pending_ack);

done:
	del_waiter(hfi, &pending_ack);

	return rc;
}

static void init_queues(struct gen7_hfi *hfi)
{
	u32 gmuaddr = hfi->hfi_mem->gmuaddr;
	struct hfi_queue_table hfi_table = {
		.qtbl_hdr = {
			.version = 0,
			.size = sizeof(struct hfi_queue_table) >> 2,
			.qhdr0_offset =
				sizeof(struct hfi_queue_table_header) >> 2,
			.qhdr_size = sizeof(struct hfi_queue_header) >> 2,
			.num_q = HFI_QUEUE_MAX,
			.num_active_q = HFI_QUEUE_MAX,
		},
		.qhdr = {
			DEFINE_QHDR(gmuaddr, HFI_CMD_ID, 0),
			DEFINE_QHDR(gmuaddr, HFI_MSG_ID, 0),
			DEFINE_QHDR(gmuaddr, HFI_DBG_ID, 0),
			/* 4 DQs for RB priority 0 */
			DEFINE_QHDR(gmuaddr, 3, 0),
			DEFINE_QHDR(gmuaddr, 4, 0),
			DEFINE_QHDR(gmuaddr, 5, 0),
			DEFINE_QHDR(gmuaddr, 6, 0),
			/* 4 DQs for RB priority 1 */
			DEFINE_QHDR(gmuaddr, 7, 1),
			DEFINE_QHDR(gmuaddr, 8, 1),
			DEFINE_QHDR(gmuaddr, 9, 1),
			DEFINE_QHDR(gmuaddr, 10, 1),
			/* 3 DQs for RB priority 2 */
			DEFINE_QHDR(gmuaddr, 11, 2),
			DEFINE_QHDR(gmuaddr, 12, 2),
			DEFINE_QHDR(gmuaddr, 13, 2),
			/* 2 DQs for RB priority 3 */
			DEFINE_QHDR(gmuaddr, 14, 3),
			DEFINE_QHDR(gmuaddr, 15, 3),
			/* 1 DQ for LPAC RB priority 4 */
			DEFINE_QHDR(gmuaddr, 16, 4),
		},
	};

	memcpy(hfi->hfi_mem->hostptr, &hfi_table, sizeof(hfi_table));
}

/* Total header sizes + queue sizes + 16 for alignment */
#define HFIMEM_SIZE (sizeof(struct hfi_queue_table) + 16 + \
	(SZ_4K * HFI_QUEUE_MAX))

static int hfi_f2h_main(void *arg);

int gen7_hwsched_hfi_init(struct adreno_device *adreno_dev)
{
	struct gen7_hwsched_hfi *hw_hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct gen7_hfi *hfi = to_gen7_hfi(adreno_dev);

	if (IS_ERR_OR_NULL(hw_hfi->big_ib)) {
		hw_hfi->big_ib = gen7_reserve_gmu_kernel_block(
				to_gen7_gmu(adreno_dev), 0,
				HWSCHED_MAX_IBS * sizeof(struct hfi_issue_ib),
				GMU_NONCACHED_KERNEL, 0);
		if (IS_ERR(hw_hfi->big_ib))
			return PTR_ERR(hw_hfi->big_ib);
	}

	if (ADRENO_FEATURE(adreno_dev, ADRENO_LSR) &&
			IS_ERR_OR_NULL(hw_hfi->big_ib_recurring)) {
		hw_hfi->big_ib_recurring = gen7_reserve_gmu_kernel_block(
				to_gen7_gmu(adreno_dev), 0,
				HWSCHED_MAX_IBS * sizeof(struct hfi_issue_ib),
				GMU_NONCACHED_KERNEL, 0);
		if (IS_ERR(hw_hfi->big_ib_recurring))
			return PTR_ERR(hw_hfi->big_ib_recurring);
	}

	if (IS_ERR_OR_NULL(hfi->hfi_mem)) {
		hfi->hfi_mem = gen7_reserve_gmu_kernel_block(
				to_gen7_gmu(adreno_dev),
				0, HFIMEM_SIZE, GMU_NONCACHED_KERNEL, 0);
		if (IS_ERR(hfi->hfi_mem))
			return PTR_ERR(hfi->hfi_mem);
		init_queues(hfi);
	}

	if (IS_ERR_OR_NULL(hw_hfi->f2h_task)) {
		hw_hfi->f2h_task = kthread_run(hfi_f2h_main, adreno_dev, "gmu_f2h");
		if (!IS_ERR(hw_hfi->f2h_task))
			sched_set_fifo(hw_hfi->f2h_task);
	}

	return PTR_ERR_OR_ZERO(hw_hfi->f2h_task);
}

static int get_attrs(u32 flags)
{
	int attrs = IOMMU_READ;

	if (flags & HFI_MEMFLAG_GMU_PRIV)
		attrs |= IOMMU_PRIV;

	if (flags & HFI_MEMFLAG_GMU_WRITEABLE)
		attrs |= IOMMU_WRITE;

	return attrs;
}

static int gmu_import_buffer(struct adreno_device *adreno_dev,
	struct hfi_mem_alloc_entry *entry)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct hfi_mem_alloc_desc *desc = &entry->desc;
	u32 vma_id = (desc->flags & HFI_MEMFLAG_GMU_CACHEABLE) ? GMU_CACHE : GMU_NONCACHED_KERNEL;

	return gen7_gmu_import_buffer(gmu, vma_id, entry->md, get_attrs(desc->flags), desc->align);
}

static struct hfi_mem_alloc_entry *lookup_mem_alloc_table(
	struct adreno_device *adreno_dev, struct hfi_mem_alloc_desc *desc)
{
	struct gen7_hwsched_hfi *hw_hfi = to_gen7_hwsched_hfi(adreno_dev);
	int i;

	for (i = 0; i < hw_hfi->mem_alloc_entries; i++) {
		struct hfi_mem_alloc_entry *entry = &hw_hfi->mem_alloc_table[i];

		if ((entry->desc.mem_kind == desc->mem_kind) &&
			(entry->desc.gmu_mem_handle == desc->gmu_mem_handle))
			return entry;
	}

	return NULL;
}

static struct hfi_mem_alloc_entry *get_mem_alloc_entry(
	struct adreno_device *adreno_dev, struct hfi_mem_alloc_desc *desc)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct hfi_mem_alloc_entry *entry =
		lookup_mem_alloc_table(adreno_dev, desc);
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	u64 flags = 0;
	u32 priv = 0;
	int ret;
	const char *memkind_string = desc->mem_kind < HFI_MEMKIND_MAX ?
			hfi_memkind_strings[desc->mem_kind] : "UNKNOWN";

	if (entry)
		return entry;

	if (desc->mem_kind >= HFI_MEMKIND_MAX) {
		dev_err(&gmu->pdev->dev, "Invalid mem kind: %d\n",
			desc->mem_kind);
		return ERR_PTR(-EINVAL);
	}

	if (hfi->mem_alloc_entries == ARRAY_SIZE(hfi->mem_alloc_table)) {
		dev_err(&gmu->pdev->dev,
			"Reached max mem alloc entries\n");
		return ERR_PTR(-ENOMEM);
	}

	entry = &hfi->mem_alloc_table[hfi->mem_alloc_entries];

	memcpy(&entry->desc, desc, sizeof(*desc));

	entry->desc.host_mem_handle = desc->gmu_mem_handle;

	if (desc->flags & HFI_MEMFLAG_GFX_PRIV)
		priv |= KGSL_MEMDESC_PRIVILEGED;

	if (!(desc->flags & HFI_MEMFLAG_GFX_WRITEABLE))
		flags |= KGSL_MEMFLAGS_GPUREADONLY;

	if (desc->flags & HFI_MEMFLAG_GFX_SECURE)
		flags |= KGSL_MEMFLAGS_SECURE;

	if (!(desc->flags & HFI_MEMFLAG_GFX_ACC) &&
		(desc->mem_kind != HFI_MEMKIND_HW_FENCE)) {
		if (desc->mem_kind == HFI_MEMKIND_MMIO_IPC_CORE)
			entry->md = gen7_reserve_gmu_kernel_block_fixed(gmu, 0,
					desc->size,
					(desc->flags & HFI_MEMFLAG_GMU_CACHEABLE) ?
					GMU_CACHE : GMU_NONCACHED_KERNEL,
					"qcom,ipc-core", get_attrs(desc->flags),
					desc->align);
		else
			entry->md = gen7_reserve_gmu_kernel_block(gmu, 0,
					desc->size,
					(desc->flags & HFI_MEMFLAG_GMU_CACHEABLE) ?
					GMU_CACHE : GMU_NONCACHED_KERNEL,
					desc->align);

		if (IS_ERR(entry->md)) {
			int ret = PTR_ERR(entry->md);

			memset(entry, 0, sizeof(*entry));
			return ERR_PTR(ret);
		}
		entry->desc.size = entry->md->size;
		entry->desc.gmu_addr = entry->md->gmuaddr;

		goto done;
	}

	/*
	 * Use pre-allocated memory descriptors to map the HFI_MEMKIND_HW_FENCE and
	 * HFI_MEMKIND_MEMSTORE
	 */
	switch (desc->mem_kind) {
	case HFI_MEMKIND_HW_FENCE:
		entry->md = &adreno_dev->hwsched.hw_fence.memdesc;
		break;
	case HFI_MEMKIND_MEMSTORE:
		entry->md = device->memstore;
		break;
	default:
		entry->md = kgsl_allocate_global(device, desc->size, 0, flags,
			priv, memkind_string);
		break;
	}
	if (IS_ERR(entry->md)) {
		int ret = PTR_ERR(entry->md);

		memset(entry, 0, sizeof(*entry));
		return ERR_PTR(ret);
	}

	entry->desc.size = entry->md->size;
	entry->desc.gpu_addr = entry->md->gpuaddr;

	if (!(desc->flags & HFI_MEMFLAG_GMU_ACC))
		goto done;

	 /*
	  * If gmu mapping fails, then we have to live with
	  * leaking the gpu global buffer allocated above.
	  */
	ret = gmu_import_buffer(adreno_dev, entry);
	if (ret) {
		dev_err(&gmu->pdev->dev,
			"gpuaddr: 0x%llx size: %lld bytes lost\n",
			entry->md->gpuaddr, entry->md->size);
		memset(entry, 0, sizeof(*entry));
		return ERR_PTR(ret);
	}

	entry->desc.gmu_addr = entry->md->gmuaddr;
done:
	hfi->mem_alloc_entries++;

	return entry;
}

static int process_mem_alloc(struct adreno_device *adreno_dev,
	struct hfi_mem_alloc_desc *mad)
{
	struct hfi_mem_alloc_entry *entry;

	entry = get_mem_alloc_entry(adreno_dev, mad);
	if (IS_ERR(entry))
		return PTR_ERR(entry);

	if (entry->md) {
		mad->gpu_addr = entry->md->gpuaddr;
		mad->gmu_addr = entry->md->gmuaddr;
	}

	/*
	 * GMU uses the host_mem_handle to check if this memalloc was
	 * successful
	 */
	mad->host_mem_handle = mad->gmu_mem_handle;

	return 0;
}

static int mem_alloc_reply(struct adreno_device *adreno_dev, void *rcvd)
{
	struct hfi_mem_alloc_desc desc = {0};
	struct hfi_mem_alloc_reply_cmd out = {0};
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	u32 seqnum;
	int ret;

	hfi_get_mem_alloc_desc(rcvd, &desc);

	ret = process_mem_alloc(adreno_dev, &desc);
	if (ret)
		return ret;

	memcpy(&out.desc, &desc, sizeof(out.desc));

	out.hdr = ACK_MSG_HDR(F2H_MSG_MEM_ALLOC);

	seqnum = atomic_inc_return(&gmu->hfi.seqnum);
	out.hdr = MSG_HDR_SET_SEQNUM_SIZE(out.hdr, seqnum, sizeof(out) >> 2);

	out.req_hdr = *(u32 *)rcvd;

	return gen7_hfi_cmdq_write(adreno_dev, (u32 *)&out, sizeof(out));
}

static int gmu_cntr_register_reply(struct adreno_device *adreno_dev, void *rcvd)
{
	struct hfi_gmu_cntr_register_cmd *in = (struct hfi_gmu_cntr_register_cmd *)rcvd;
	struct hfi_gmu_cntr_register_reply_cmd out = {0};
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	u32 lo = 0, hi = 0, seqnum;

	/*
	 * Failure to allocate counter is not fatal. Sending lo = 0, hi = 0
	 * indicates to GMU that counter allocation failed.
	 */
	adreno_perfcounter_get(adreno_dev,
		in->group_id, in->countable, &lo, &hi, PERFCOUNTER_FLAG_KERNEL);

	out.hdr = ACK_MSG_HDR(F2H_MSG_GMU_CNTR_REGISTER);
	seqnum = atomic_inc_return(&gmu->hfi.seqnum);
	out.hdr = MSG_HDR_SET_SEQNUM_SIZE(out.hdr, seqnum, sizeof(out) >> 2);
	out.req_hdr = in->hdr;
	out.group_id = in->group_id;
	out.countable = in->countable;
	/* Fill in byte offset of counter */
	out.cntr_lo = lo << 2;
	out.cntr_hi = hi << 2;

	return gen7_hfi_cmdq_write(adreno_dev, (u32 *)&out, sizeof(out));
}

static int send_warmboot_start_msg(struct adreno_device *adreno_dev)
{
	int ret = 0;
	struct hfi_start_cmd cmd;

	if (!adreno_dev->warmboot_enabled)
		return ret;

	ret = CMD_MSG_HDR(cmd, H2F_MSG_START);
	if (ret)
		return ret;

	cmd.hdr = RECORD_NOP_MSG_HDR(cmd.hdr);

	return gen7_hfi_send_generic_req(adreno_dev, &cmd, sizeof(cmd));
}

static int send_start_msg(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	u32 seqnum;
	int ret, rc = 0;
	struct hfi_start_cmd cmd;
	u32 rcvd[MAX_RCVD_SIZE];
	struct pending_cmd pending_ack = {0};

	ret = CMD_MSG_HDR(cmd, H2F_MSG_START);
	if (ret)
		return ret;

	seqnum = atomic_inc_return(&gmu->hfi.seqnum);
	cmd.hdr = MSG_HDR_SET_SEQNUM_SIZE(cmd.hdr, seqnum, sizeof(cmd) >> 2);

	pending_ack.sent_hdr = cmd.hdr;

	rc = gen7_hfi_cmdq_write(adreno_dev, (u32 *)&cmd, sizeof(cmd));
	if (rc)
		return rc;

poll:
	rc = gmu_core_timed_poll_check(device, GEN7_GMU_GMU2HOST_INTR_INFO,
		HFI_IRQ_MSGQ_MASK, HFI_RSP_TIMEOUT, HFI_IRQ_MSGQ_MASK);

	if (rc) {
		dev_err(&gmu->pdev->dev,
			"Timed out processing MSG_START seqnum: %d\n",
			seqnum);
		gmu_core_fault_snapshot(device);
		return rc;
	}

	/* Clear the interrupt */
	gmu_core_regwrite(device, GEN7_GMU_GMU2HOST_INTR_CLR,
		HFI_IRQ_MSGQ_MASK);

	if (gen7_hfi_queue_read(gmu, HFI_MSG_ID, rcvd, sizeof(rcvd)) <= 0) {
		dev_err(&gmu->pdev->dev, "MSG_START: no payload\n");
		gmu_core_fault_snapshot(device);
		return -EINVAL;
	}

	if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) {
		rc = gen7_receive_ack_cmd(gmu, rcvd, &pending_ack);
		if (rc)
			return rc;

		return check_ack_failure(adreno_dev, &pending_ack);
	}

	if (MSG_HDR_GET_ID(rcvd[0]) == F2H_MSG_MEM_ALLOC) {
		rc = mem_alloc_reply(adreno_dev, rcvd);
		if (rc)
			return rc;

		goto poll;
	}

	if (MSG_HDR_GET_ID(rcvd[0]) == F2H_MSG_GMU_CNTR_REGISTER) {
		rc = gmu_cntr_register_reply(adreno_dev, rcvd);
		if (rc)
			return rc;
		goto poll;
	}

	dev_err(&gmu->pdev->dev,
		"MSG_START: unexpected response id:%d, type:%d\n",
		MSG_HDR_GET_ID(rcvd[0]),
		MSG_HDR_GET_TYPE(rcvd[0]));

	gmu_core_fault_snapshot(device);

	return rc;
}

static void reset_hfi_mem_records(struct adreno_device *adreno_dev)
{
	struct gen7_hwsched_hfi *hw_hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct kgsl_memdesc *md = NULL;
	u32 i;

	for (i = 0; i < hw_hfi->mem_alloc_entries; i++) {
		struct hfi_mem_alloc_desc *desc = &hw_hfi->mem_alloc_table[i].desc;

		if (desc->flags & HFI_MEMFLAG_HOST_INIT) {
			md = hw_hfi->mem_alloc_table[i].md;
			memset(md->hostptr, 0x0, md->size);
		}
	}
}

static void reset_hfi_queues(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct hfi_queue_table *tbl = gmu->hfi.hfi_mem->hostptr;
	u32 i;

	/* Flush HFI queues */
	for (i = 0; i < HFI_QUEUE_MAX; i++) {
		struct hfi_queue_header *hdr = &tbl->qhdr[i];

		if (hdr->status == HFI_QUEUE_STATUS_DISABLED)
			continue;

		hdr->read_index = hdr->write_index;
	}
}

void gen7_hwsched_hfi_stop(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);

	hfi->irq_mask &= ~HFI_IRQ_MSGQ_MASK;

	/*
	 * In some corner cases, it is possible that GMU put TS_RETIRE
	 * on the msgq after we have turned off gmu interrupts. Hence,
	 * drain the queue one last time before we reset HFI queues.
	 */
	gen7_hwsched_process_msgq(adreno_dev);

	/* Drain the debug queue before we reset HFI queues */
	gen7_hwsched_process_dbgq(adreno_dev, false);

	kgsl_pwrctrl_axi(KGSL_DEVICE(adreno_dev), false);

	clear_bit(GMU_PRIV_HFI_STARTED, &gmu->flags);

	/*
	 * Reset the hfi host access memory records, As GMU expects hfi memory
	 * records to be clear in bootup.
	 */
	reset_hfi_mem_records(adreno_dev);
}

static void gen7_hwsched_enable_async_hfi(struct adreno_device *adreno_dev)
{
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);

	hfi->irq_mask |= HFI_IRQ_MSGQ_MASK;

	gmu_core_regwrite(KGSL_DEVICE(adreno_dev), GEN7_GMU_GMU2HOST_INTR_MASK,
		(u32)~hfi->irq_mask);
}

static int enable_preemption(struct adreno_device *adreno_dev)
{
	const struct adreno_gen7_core *gen7_core = to_gen7_core(adreno_dev);
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	u32 data;
	int ret;

	if (!adreno_is_preemption_enabled(adreno_dev))
		return 0;

	/*
	 * Bits [0:1] contains the preemption level
	 * Bit 2 is to enable/disable gmem save/restore
	 * Bit 3 is to enable/disable skipsaverestore
	 */
	data = FIELD_PREP(GENMASK(1, 0), adreno_dev->preempt.preempt_level) |
			FIELD_PREP(BIT(2), adreno_dev->preempt.usesgmem) |
			FIELD_PREP(BIT(3), adreno_dev->preempt.skipsaverestore);

	ret = gen7_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_PREEMPTION, 1,
			data);
	if (ret)
		return ret;

	if (gen7_core->qos_value) {
		int i;

		for (i = 0; i < KGSL_PRIORITY_MAX_RB_LEVELS; i++) {
			if (!gen7_core->qos_value[i])
				continue;

			gen7_hfi_send_set_value(adreno_dev,
				HFI_VALUE_RB_GPU_QOS, i,
				gen7_core->qos_value[i]);
		}
	}

	if (device->pwrctrl.rt_bus_hint) {
		ret = gen7_hfi_send_set_value(adreno_dev, HFI_VALUE_RB_IB_RULE, 0,
			device->pwrctrl.rt_bus_hint);
		if (ret)
			device->pwrctrl.rt_bus_hint = 0;
	}

	/*
	 * Bits[3:0] contain the preemption timeout enable bit per ringbuffer
	 * Bits[31:4] contain the timeout in ms
	 */
	return gen7_hfi_send_set_value(adreno_dev, HFI_VALUE_BIN_TIME, 1,
		FIELD_PREP(GENMASK(31, 4), ADRENO_PREEMPT_TIMEOUT) |
		FIELD_PREP(GENMASK(3, 0), 0xf));

}

static int enable_gmu_stats(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	u32 data;

	if (!gmu->stats_enable)
		return 0;

	/*
	 * Bits [23:0] contains the countables mask
	 * Bits [31:24] is the sampling interval
	 */
	data = FIELD_PREP(GENMASK(23, 0), gmu->stats_mask) |
		FIELD_PREP(GENMASK(31, 24), gmu->stats_interval);

	return gen7_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_GMU_STATS, 1, data);
}

static int gen7_hfi_send_perfcounter_feature_ctrl(struct adreno_device *adreno_dev)
{
	/*
	 * Perfcounter retention is disabled by default in GMU firmware.
	 * In case perfcounter retention behaviour is overwritten by sysfs
	 * setting dynmaically, send this HFI feature with 'enable = 0' to
	 * disable this feature in GMU firmware.
	 */
	if (adreno_dev->perfcounter)
		return gen7_hfi_send_feature_ctrl(adreno_dev,
				HFI_FEATURE_PERF_NORETAIN, 0, 0);

	return 0;
}

u32 gen7_hwsched_hfi_get_value(struct adreno_device *adreno_dev, u32 prop)
{
	struct hfi_get_value_cmd cmd;
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct pending_cmd pending_ack;
	int rc;
	u32 seqnum;

	rc = CMD_MSG_HDR(cmd, H2F_MSG_GET_VALUE);
	if (rc)
		return 0;

	seqnum = atomic_inc_return(&gmu->hfi.seqnum);
	cmd.hdr = MSG_HDR_SET_SEQNUM_SIZE(cmd.hdr, seqnum, sizeof(cmd) >> 2);
	cmd.type = prop;
	cmd.subtype = 0;

	add_waiter(hfi, cmd.hdr, &pending_ack);

	rc = gen7_hfi_cmdq_write(adreno_dev, (u32 *)&cmd, sizeof(cmd));
	if (rc)
		goto done;

	rc = adreno_hwsched_wait_ack_completion(adreno_dev, &gmu->pdev->dev, &pending_ack,
		gen7_hwsched_process_msgq);

done:
	del_waiter(hfi, &pending_ack);

	if (rc || (pending_ack.results[2] == UINT_MAX))
		return 0;

	return pending_ack.results[2];
}

static void _context_queue_enable(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);

	if (GMU_VER_MINOR(gmu->ver.hfi) >= 3) {
		if (gen7_hfi_send_get_value(adreno_dev, HFI_VALUE_CONTEXT_QUEUE, 0) == 1)
			set_bit(ADRENO_HWSCHED_CONTEXT_QUEUE, &adreno_dev->hwsched.flags);
	}
}

static int gen7_hfi_send_hw_fence_feature_ctrl(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
	int ret;

	if (!test_bit(ADRENO_HWSCHED_HW_FENCE, &hwsched->flags))
		return 0;

	ret = gen7_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_HW_FENCE, 1, 0);
	if (ret && (ret == -ENOENT)) {
		dev_err(&gmu->pdev->dev, "GMU doesn't support HW_FENCE feature\n");
		adreno_hwsched_deregister_hw_fence(hwsched->hw_fence.handle);
		return 0;
	}

	return ret;
}

static int gen7_hfi_send_dms_feature_ctrl(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	int ret;

	if (!test_bit(ADRENO_DEVICE_DMS, &adreno_dev->priv))
		return 0;

	ret = gen7_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_DMS, 1, 0);
	if (ret == -ENOENT) {
		dev_err(&gmu->pdev->dev, "GMU doesn't support DMS feature\n");
		clear_bit(ADRENO_DEVICE_DMS, &adreno_dev->priv);
		adreno_dev->dms_enabled = false;
		return 0;
	}

	return ret;
}

static void gen7_spin_idle_debug_lpac(struct adreno_device *adreno_dev,
				const char *str)
{
	struct kgsl_device *device = &adreno_dev->dev;
	u32 rptr, wptr, status, status3, intstatus, hwfault;
	bool val = adreno_is_preemption_enabled(adreno_dev);

	dev_err(device->dev, str);

	kgsl_regread(device, GEN7_CP_LPAC_RB_RPTR, &rptr);
	kgsl_regread(device, GEN7_CP_LPAC_RB_WPTR, &wptr);

	kgsl_regread(device, GEN7_RBBM_STATUS, &status);
	kgsl_regread(device, GEN7_RBBM_STATUS3, &status3);
	kgsl_regread(device, GEN7_RBBM_INT_0_STATUS, &intstatus);
	kgsl_regread(device, GEN7_CP_HW_FAULT, &hwfault);

	dev_err(device->dev,
		"LPAC rb=%d pos=%X/%X rbbm_status=%8.8X/%8.8X int_0_status=%8.8X\n",
		val ? KGSL_LPAC_RB_ID : 1, rptr, wptr,
		status, status3, intstatus);

	dev_err(device->dev, " hwfault=%8.8X\n", hwfault);

	kgsl_device_snapshot(device, NULL, NULL, false);
}

static bool gen7_hwsched_warmboot_possible(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);

	if (adreno_dev->warmboot_enabled && test_bit(GMU_PRIV_WARMBOOT_GMU_INIT_DONE, &gmu->flags)
		&& test_bit(GMU_PRIV_WARMBOOT_GPU_BOOT_DONE, &gmu->flags) &&
		!test_bit(ADRENO_DEVICE_FORCE_COLDBOOT, &adreno_dev->priv))
		return true;

	return false;
}

static int gen7_hwsched_hfi_send_warmboot_cmd(struct adreno_device *adreno_dev,
		struct kgsl_memdesc *desc, u32 flag, bool async, struct pending_cmd *ack)
{
	struct hfi_warmboot_scratch_cmd cmd = {0};
	int ret;

	if (!adreno_dev->warmboot_enabled)
		return 0;

	cmd.scratch_addr = desc->gmuaddr;
	cmd.scratch_size =  desc->size;
	cmd.flags = flag;

	ret = CMD_MSG_HDR(cmd, H2F_MSG_WARMBOOT_CMD);
	if (ret)
		return ret;

	if (async)
		return gen7_hfi_send_cmd_async(adreno_dev, &cmd, sizeof(cmd));

	return gen7_hfi_send_generic_req_v5(adreno_dev, &cmd, ack, sizeof(cmd));
}

static int gen7_hwsched_hfi_warmboot_gpu_cmd(struct adreno_device *adreno_dev,
		struct pending_cmd *ret_cmd)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct hfi_warmboot_scratch_cmd cmd = {
		.scratch_addr = gmu->gpu_boot_scratch->gmuaddr,
		.scratch_size = gmu->gpu_boot_scratch->size,
		.flags = HFI_WARMBOOT_EXEC_SCRATCH,
	};
	int ret = 0;
	u32 seqnum;

	if (!adreno_dev->warmboot_enabled)
		return 0;

	ret = CMD_MSG_HDR(cmd, H2F_MSG_WARMBOOT_CMD);
	if (ret)
		return ret;

	seqnum = atomic_inc_return(&gmu->hfi.seqnum);
	cmd.hdr = MSG_HDR_SET_SEQNUM_SIZE(cmd.hdr, seqnum, sizeof(cmd) >> 2);

	add_waiter(hfi, cmd.hdr, ret_cmd);

	ret = gen7_hfi_cmdq_write(adreno_dev, (u32 *)&cmd, sizeof(cmd));
	if (ret)
		goto err;

	ret = adreno_hwsched_wait_ack_completion(adreno_dev, &gmu->pdev->dev, ret_cmd,
		gen7_hwsched_process_msgq);
err:
	del_waiter(hfi, ret_cmd);

	return ret;
}

static int gen7_hwsched_warmboot_gpu(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct pending_cmd ret_cmd = {0};
	int ret = 0;

	ret = gen7_hwsched_hfi_warmboot_gpu_cmd(adreno_dev, &ret_cmd);
	if (!ret)
		return ret;

	if (MSG_HDR_GET_TYPE(ret_cmd.results[1]) != H2F_MSG_WARMBOOT_CMD)
		goto err;

	switch (MSG_HDR_GET_TYPE(ret_cmd.results[2])) {
	case H2F_MSG_ISSUE_CMD_RAW: {
		if (ret_cmd.results[2] == gmu->cp_init_hdr)
			gen7_spin_idle_debug(adreno_dev,
				"CP initialization failed to idle\n");
		else if (ret_cmd.results[2] == gmu->switch_to_unsec_hdr)
			gen7_spin_idle_debug(adreno_dev,
				"Switch to unsecure failed to idle\n");
		}
		break;
	case H2F_MSG_ISSUE_LPAC_CMD_RAW:
		gen7_spin_idle_debug_lpac(adreno_dev,
			"LPAC CP initialization failed to idle\n");
		break;
	}
err:
	/* Clear the bit on error so that in the next slumber exit we coldboot */
	clear_bit(GMU_PRIV_WARMBOOT_GPU_BOOT_DONE, &gmu->flags);
	gen7_disable_gpu_irq(adreno_dev);
	return ret;
}

static int gen7_hwsched_coldboot_gpu(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct gen7_hfi *hfi = to_gen7_hfi(adreno_dev);
	struct pending_cmd ack = {0};
	int ret = 0;

	ret = gen7_hwsched_hfi_send_warmboot_cmd(adreno_dev, gmu->gpu_boot_scratch,
		 HFI_WARMBOOT_SET_SCRATCH, true, &ack);
	if (ret)
		goto done;

	ret = gen7_hwsched_cp_init(adreno_dev);
	if (ret)
		goto done;

	ret = gen7_hwsched_lpac_cp_init(adreno_dev);
	if (ret)
		goto done;

	ret = gen7_hwsched_hfi_send_warmboot_cmd(adreno_dev, gmu->gpu_boot_scratch,
		HFI_WARMBOOT_QUERY_SCRATCH, true, &ack);
	if (ret)
		goto done;

	if (adreno_dev->warmboot_enabled)
		set_bit(GMU_PRIV_WARMBOOT_GPU_BOOT_DONE, &gmu->flags);

done:
	/* Clear the bitmask so that we don't send record bit with future HFI messages */
	memset(hfi->wb_set_record_bitmask, 0x0, sizeof(hfi->wb_set_record_bitmask));

	if (ret)
		gen7_disable_gpu_irq(adreno_dev);

	return ret;
}

int gen7_hwsched_boot_gpu(struct adreno_device *adreno_dev)
{
	/* If warmboot is possible just send the warmboot command else coldboot */
	if (gen7_hwsched_warmboot_possible(adreno_dev))
		return gen7_hwsched_warmboot_gpu(adreno_dev);
	else
		return gen7_hwsched_coldboot_gpu(adreno_dev);
}

static int gen7_hwsched_setup_default_votes(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	int ret = 0;

	/* Request default DCVS level */
	ret = kgsl_pwrctrl_set_default_gpu_pwrlevel(device);
	if (ret)
		return ret;

	/* Request default BW vote */
	return kgsl_pwrctrl_axi(device, true);
}

int gen7_hwsched_warmboot_init_gmu(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct pending_cmd ack = {0};
	int ret = 0;

	ret = gen7_hwsched_hfi_send_warmboot_cmd(adreno_dev, gmu->gmu_init_scratch,
		 HFI_WARMBOOT_EXEC_SCRATCH, false, &ack);
	if (ret)
		goto err;

	gen7_hwsched_enable_async_hfi(adreno_dev);

	set_bit(GMU_PRIV_HFI_STARTED, &gmu->flags);

	ret = gen7_hwsched_setup_default_votes(adreno_dev);

err:
	if (ret) {
		/* Clear the bit in case of an error so next boot will be coldboot */
		clear_bit(GMU_PRIV_WARMBOOT_GMU_INIT_DONE, &gmu->flags);
		clear_bit(GMU_PRIV_WARMBOOT_GPU_BOOT_DONE, &gmu->flags);
		gen7_hwsched_hfi_stop(adreno_dev);
	}

	return ret;
}

static void warmboot_init_message_record_bitmask(struct adreno_device *adreno_dev)
{
	struct gen7_hfi *hfi = to_gen7_hfi(adreno_dev);

	if (!adreno_dev->warmboot_enabled)
		return;

	/* Set the record bit for all the messages */
	memset(hfi->wb_set_record_bitmask, 0xFF, sizeof(hfi->wb_set_record_bitmask));

	/* These messages should not be recorded */
	clear_bit(H2F_MSG_WARMBOOT_CMD, hfi->wb_set_record_bitmask);
	clear_bit(H2F_MSG_START, hfi->wb_set_record_bitmask);
	clear_bit(H2F_MSG_GET_VALUE, hfi->wb_set_record_bitmask);
	clear_bit(H2F_MSG_GX_BW_PERF_VOTE, hfi->wb_set_record_bitmask);
}

int gen7_hwsched_hfi_start(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct pending_cmd ack = {0};
	int ret;

	reset_hfi_queues(adreno_dev);

	ret = gen7_gmu_hfi_start(adreno_dev);
	if (ret)
		goto err;

	if (gen7_hwsched_warmboot_possible(adreno_dev))
		return gen7_hwsched_warmboot_init_gmu(adreno_dev);

	if (ADRENO_FEATURE(adreno_dev, ADRENO_GMU_WARMBOOT) &&
		(!test_bit(GMU_PRIV_FIRST_BOOT_DONE, &gmu->flags))) {
		if (gen7_hfi_send_get_value(adreno_dev, HFI_VALUE_GMU_WARMBOOT, 0) == 1)
			adreno_dev->warmboot_enabled = true;
	}

	warmboot_init_message_record_bitmask(adreno_dev);

	/* Reset the variable here and set it when we successfully record the scratch */
	clear_bit(GMU_PRIV_WARMBOOT_GMU_INIT_DONE, &gmu->flags);
	clear_bit(GMU_PRIV_WARMBOOT_GPU_BOOT_DONE, &gmu->flags);

	ret = gen7_hwsched_hfi_send_warmboot_cmd(adreno_dev, gmu->gmu_init_scratch,
		HFI_WARMBOOT_SET_SCRATCH, false, &ack);
	if (ret)
		goto err;

	ret = gen7_hfi_send_gpu_perf_table(adreno_dev);
	if (ret)
		goto err;

	ret = gen7_hfi_send_generic_req(adreno_dev, &gmu->hfi.bw_table, sizeof(gmu->hfi.bw_table));
	if (ret)
		goto err;

	ret = gen7_hfi_send_acd_feature_ctrl(adreno_dev);
	if (ret)
		goto err;

	ret = gen7_hfi_send_bcl_feature_ctrl(adreno_dev);
	if (ret)
		goto err;

	ret = gen7_hfi_send_clx_feature_ctrl(adreno_dev);
	if (ret)
		goto err;

	ret = gen7_hfi_send_ifpc_feature_ctrl(adreno_dev);
	if (ret)
		goto err;

	ret = gen7_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_HWSCHED, 1, 0);
	if (ret)
		goto err;

	ret = gen7_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_KPROF, 1, 0);
	if (ret)
		goto err;

	if (ADRENO_FEATURE(adreno_dev, ADRENO_LSR)) {
		ret = gen7_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_LSR,
				1, 0);
		if (ret)
			goto err;
	}

	ret = gen7_hfi_send_perfcounter_feature_ctrl(adreno_dev);
	if (ret)
		goto err;

	ret = gen7_hfi_send_dms_feature_ctrl(adreno_dev);
	if (ret)
		goto err;

	/* Enable the long ib timeout detection */
	if (adreno_long_ib_detect(adreno_dev)) {
		ret = gen7_hfi_send_feature_ctrl(adreno_dev,
			HFI_FEATURE_BAIL_OUT_TIMER, 1, 0);
		if (ret)
			goto err;
	}

	enable_gmu_stats(adreno_dev);

	if (gmu->log_stream_enable)
		gen7_hfi_send_set_value(adreno_dev,
			HFI_VALUE_LOG_STREAM_ENABLE, 0, 1);

	if (gmu->log_group_mask)
		gen7_hfi_send_set_value(adreno_dev,
			HFI_VALUE_LOG_GROUP, 0, gmu->log_group_mask);

	ret = gen7_hfi_send_core_fw_start(adreno_dev);
	if (ret)
		goto err;

	/*
	 * HFI_VALUE_CONTEXT_QUEUE can only be queried after GMU has initialized some of the
	 * required resources as part of handling gen7_hfi_send_core_fw_start()
	 */
	if (!test_bit(GMU_PRIV_FIRST_BOOT_DONE, &gmu->flags)) {
		_context_queue_enable(adreno_dev);
		adreno_hwsched_register_hw_fence(adreno_dev);
	}

	ret = gen7_hfi_send_hw_fence_feature_ctrl(adreno_dev);
	if (ret)
		goto err;

	ret = enable_preemption(adreno_dev);
	if (ret)
		goto err;

	ret = gen7_hfi_send_lpac_feature_ctrl(adreno_dev);
	if (ret)
		goto err;

	if (ADRENO_FEATURE(adreno_dev, ADRENO_AQE)) {
		ret = gen7_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_AQE, 1, 0);
		if (ret)
			goto err;
	}

	ret = send_start_msg(adreno_dev);
	if (ret)
		goto err;

	/*
	 * Send this additional start message on cold boot if warmboot is enabled.
	 * This message will be recorded and on a warmboot this will trigger the
	 * sequence to replay memory allocation requests and ECP task setup
	 */
	ret = send_warmboot_start_msg(adreno_dev);
	if (ret)
		goto err;

	gen7_hwsched_enable_async_hfi(adreno_dev);

	set_bit(GMU_PRIV_HFI_STARTED, &gmu->flags);

	/* Send this message only on cold boot */
	ret = gen7_hwsched_hfi_send_warmboot_cmd(adreno_dev, gmu->gmu_init_scratch,
		HFI_WARMBOOT_QUERY_SCRATCH, true, &ack);
	if (ret)
		goto err;

	if (adreno_dev->warmboot_enabled)
		set_bit(GMU_PRIV_WARMBOOT_GMU_INIT_DONE, &gmu->flags);

	ret = gen7_hwsched_setup_default_votes(adreno_dev);

err:
	if (ret)
		gen7_hwsched_hfi_stop(adreno_dev);

	return ret;
}

static int submit_raw_cmds(struct adreno_device *adreno_dev, void *cmds, u32 size_bytes,
	const char *str)
{
	int ret;

	ret = gen7_hfi_send_cmd_async(adreno_dev, cmds, size_bytes);
	if (ret)
		return ret;

	ret = gmu_core_timed_poll_check(KGSL_DEVICE(adreno_dev),
			GEN7_GPU_GMU_AO_GPU_CX_BUSY_STATUS, 0, 200, BIT(23));
	if (ret)
		gen7_spin_idle_debug(adreno_dev, str);

	return ret;
}

static int submit_lpac_raw_cmds(struct adreno_device *adreno_dev, void *cmds, u32 size_bytes,
	const char *str)
{
	int ret;

	ret = gen7_hfi_send_cmd_async(adreno_dev, cmds, size_bytes);
	if (ret)
		return ret;

	ret = gmu_core_timed_poll_check(KGSL_DEVICE(adreno_dev),
			GEN7_GPU_GMU_AO_GPU_LPAC_BUSY_STATUS, 0, 200, BIT(23));
	if (ret)
		gen7_spin_idle_debug_lpac(adreno_dev, str);

	return ret;
}

static int cp_init(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	u32 cmds[GEN7_CP_INIT_DWORDS + 1];
	int ret = 0;

	cmds[0] = CREATE_MSG_HDR(H2F_MSG_ISSUE_CMD_RAW, HFI_MSG_CMD);

	gen7_cp_init_cmds(adreno_dev, &cmds[1]);

	ret = submit_raw_cmds(adreno_dev, cmds, sizeof(cmds),
			"CP initialization failed to idle\n");

	/* Save the header incase we need a warmboot debug */
	gmu->cp_init_hdr = cmds[0];

	return ret;
}

static int send_switch_to_unsecure(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	u32 cmds[3];
	int ret = 0;

	cmds[0] = CREATE_MSG_HDR(H2F_MSG_ISSUE_CMD_RAW, HFI_MSG_CMD);

	cmds[1] = cp_type7_packet(CP_SET_SECURE_MODE, 1);
	cmds[2] = 0;

	ret = submit_raw_cmds(adreno_dev, cmds, sizeof(cmds),
			"Switch to unsecure failed to idle\n");

	/* Save the header incase we need a warmboot debug */
	gmu->switch_to_unsec_hdr = cmds[0];

	return ret;
}

int gen7_hwsched_cp_init(struct adreno_device *adreno_dev)
{
	const struct adreno_gen7_core *gen7_core = to_gen7_core(adreno_dev);
	int ret;

	ret = cp_init(adreno_dev);
	if (ret)
		return ret;

	ret = adreno_zap_shader_load(adreno_dev, gen7_core->zap_name);
	if (ret)
		return ret;

	if (!adreno_dev->zap_loaded)
		kgsl_regwrite(KGSL_DEVICE(adreno_dev),
			GEN7_RBBM_SECVID_TRUST_CNTL, 0x0);
	else
		ret = send_switch_to_unsecure(adreno_dev);

	return ret;
}

int gen7_hwsched_lpac_cp_init(struct adreno_device *adreno_dev)
{
	u32 cmds[GEN7_CP_INIT_DWORDS + 1];

	if (!adreno_dev->lpac_enabled)
		return 0;

	cmds[0] = CREATE_MSG_HDR(H2F_MSG_ISSUE_LPAC_CMD_RAW, HFI_MSG_CMD);

	gen7_cp_init_cmds(adreno_dev, &cmds[1]);

	return submit_lpac_raw_cmds(adreno_dev, cmds, sizeof(cmds),
			"LPAC CP initialization failed to idle\n");
}

static bool is_queue_empty(struct adreno_device *adreno_dev, u32 queue_idx)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct kgsl_memdesc *mem_addr = gmu->hfi.hfi_mem;
	struct hfi_queue_table *tbl = mem_addr->hostptr;
	struct hfi_queue_header *hdr = &tbl->qhdr[queue_idx];

	if (hdr->status == HFI_QUEUE_STATUS_DISABLED)
		return true;

	if (hdr->read_index == hdr->write_index)
		return true;

	return false;
}

static int hfi_f2h_main(void *arg)
{
	struct adreno_device *adreno_dev = arg;
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);

	while (!kthread_should_stop()) {
		wait_event_interruptible(hfi->f2h_wq, kthread_should_stop() ||
			/* If msgq irq is enabled and msgq has messages to process */
			(((hfi->irq_mask & HFI_IRQ_MSGQ_MASK) &&
			!is_queue_empty(adreno_dev, HFI_MSG_ID)) ||
			/* Trace buffer has messages to process */
			!gmu_core_is_trace_empty(gmu->trace.md->hostptr) ||
			/* Dbgq has messages to process */
			!is_queue_empty(adreno_dev, HFI_DBG_ID)));

		if (kthread_should_stop())
			break;

		gen7_hwsched_process_msgq(adreno_dev);
		gmu_core_process_trace_data(KGSL_DEVICE(adreno_dev),
					&gmu->pdev->dev, &gmu->trace);
		gen7_hwsched_process_dbgq(adreno_dev, true);
	}

	return 0;
}

static void gen7_hwsched_hw_fence_timeout(struct work_struct *work)
{
	struct gen7_hwsched_hfi *hfi = container_of(work, struct gen7_hwsched_hfi, hw_fence_ws);
	struct gen7_hwsched_device *gen7_hw_dev = container_of(hfi, struct gen7_hwsched_device,
						hwsched_hfi);
	struct adreno_device *adreno_dev = &gen7_hw_dev->gen7_dev.adreno_dev;
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	u32 unack_count, ts;
	struct adreno_context *drawctxt = NULL;
	bool fault;

	/* Check msgq one last time before recording a fault */
	gen7_hwsched_process_msgq(adreno_dev);

	spin_lock(&hfi->hw_fence.lock);

	unack_count = hfi->hw_fence.unack_count;

	fault = test_bit(GEN7_HWSCHED_HW_FENCE_SLEEP_BIT, &hfi->hw_fence.flags) &&
		test_bit(GEN7_HWSCHED_HW_FENCE_MAX_BIT, &hfi->hw_fence.flags) &&
		(unack_count > MIN_HW_FENCE_UNACK_COUNT);

	drawctxt = hfi->hw_fence.defer_drawctxt;
	ts = hfi->hw_fence.defer_ts;

	spin_unlock(&hfi->hw_fence.lock);

	if (!fault)
		return;

	dev_err(&gmu->pdev->dev, "Hardware fence unack(%d) timeout\n", unack_count);

	if (drawctxt) {
		struct kgsl_process_private *proc_priv = drawctxt->base.proc_priv;

		dev_err(&gmu->pdev->dev,
			"Hardware fence got deferred for ctx:%d ts:%d pid:%d proc:%s\n",
			drawctxt->base.id, ts, pid_nr(proc_priv->pid), proc_priv->comm);
	}
	gen7_hwsched_fault(adreno_dev, ADRENO_GMU_FAULT);
}

static void gen7_hwsched_hw_fence_timer(struct timer_list *t)
{
	struct gen7_hwsched_hfi *hfi = from_timer(hfi, t, hw_fence_timer);

	kgsl_schedule_work(&hfi->hw_fence_ws);
}

int gen7_hwsched_hfi_probe(struct adreno_device *adreno_dev)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct gen7_hwsched_hfi *hw_hfi = to_gen7_hwsched_hfi(adreno_dev);

	gmu->hfi.irq = kgsl_request_irq(gmu->pdev, "hfi",
		gen7_hwsched_hfi_handler, adreno_dev);

	if (gmu->hfi.irq < 0)
		return gmu->hfi.irq;

	hw_hfi->irq_mask = HFI_IRQ_MASK;

	rwlock_init(&hw_hfi->msglock);

	INIT_LIST_HEAD(&hw_hfi->msglist);
	INIT_LIST_HEAD(&hw_hfi->detached_hw_fence_list);

	init_waitqueue_head(&hw_hfi->f2h_wq);
	init_waitqueue_head(&hw_hfi->hw_fence.unack_wq);

	spin_lock_init(&hw_hfi->hw_fence.lock);

	mutex_init(&hw_hfi->msgq_mutex);

	INIT_WORK(&hw_hfi->hw_fence_ws, gen7_hwsched_hw_fence_timeout);

	timer_setup(&hw_hfi->hw_fence_timer, gen7_hwsched_hw_fence_timer, 0);

	return 0;
}

void gen7_hwsched_hfi_remove(struct adreno_device *adreno_dev)
{
	struct gen7_hwsched_hfi *hw_hfi = to_gen7_hwsched_hfi(adreno_dev);

	if (hw_hfi->f2h_task)
		kthread_stop(hw_hfi->f2h_task);
}

static void gen7_add_profile_events(struct adreno_device *adreno_dev,
	struct kgsl_drawobj_cmd *cmdobj, struct adreno_submit_time *time)
{
	unsigned long flags;
	u64 time_in_s;
	unsigned long time_in_ns;
	struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj);
	struct kgsl_context *context = drawobj->context;
	struct submission_info info = {0};
	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
	const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);

	if (!time)
		return;

	/*
	 * Here we are attempting to create a mapping between the
	 * GPU time domain (alwayson counter) and the CPU time domain
	 * (local_clock) by sampling both values as close together as
	 * possible. This is useful for many types of debugging and
	 * profiling. In order to make this mapping as accurate as
	 * possible, we must turn off interrupts to avoid running
	 * interrupt handlers between the two samples.
	 */

	local_irq_save(flags);

	/* Read always on registers */
	time->ticks = gpudev->read_alwayson(adreno_dev);

	/* Trace the GPU time to create a mapping to ftrace time */
	trace_adreno_cmdbatch_sync(context->id, context->priority,
		drawobj->timestamp, time->ticks);

	/* Get the kernel clock for time since boot */
	time->ktime = local_clock();

	/* Get the timeofday for the wall time (for the user) */
	ktime_get_real_ts64(&time->utime);

	local_irq_restore(flags);

	/* Return kernel clock time to the client if requested */
	time_in_s = time->ktime;
	time_in_ns = do_div(time_in_s, 1000000000);

	info.inflight = hwsched->inflight;
	info.rb_id = adreno_get_level(context);
	info.gmu_dispatch_queue = context->gmu_dispatch_queue;

	cmdobj->submit_ticks = time->ticks;

	msm_perf_events_update(MSM_PERF_GFX, MSM_PERF_SUBMIT,
		pid_nr(context->proc_priv->pid),
		context->id, drawobj->timestamp,
		!!(drawobj->flags & KGSL_DRAWOBJ_END_OF_FRAME));
	trace_adreno_cmdbatch_submitted(drawobj, &info, time->ticks,
		(unsigned long) time_in_s, time_in_ns / 1000, 0);

	log_kgsl_cmdbatch_submitted_event(context->id, drawobj->timestamp,
			context->priority, drawobj->flags);
}

static void init_gmu_context_queue(struct adreno_context *drawctxt)
{
	struct kgsl_memdesc *md = &drawctxt->gmu_context_queue;
	struct gmu_context_queue_header *hdr = md->hostptr;

	hdr->start_addr = md->gmuaddr + sizeof(*hdr);
	hdr->queue_size = (md->size - sizeof(*hdr)) >> 2;
	hdr->hw_fence_buffer_va = drawctxt->gmu_hw_fence_queue.gmuaddr;
	hdr->hw_fence_buffer_size = drawctxt->gmu_hw_fence_queue.size;
}

static u32 get_dq_id(struct adreno_device *adreno_dev, struct kgsl_context *context)
{
	struct dq_info *info;
	u32 next;
	u32 priority = adreno_get_level(context);

	if (adreno_dev->lpac_enabled)
		info = &gen7_hfi_dqs_lpac[priority];
	else
		info = &gen7_hfi_dqs[priority];

	next = info->base_dq_id + info->offset;

	info->offset = (info->offset + 1) % info->max_dq;

	return next;
}

static int allocate_context_queues(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt)
{
	int ret = 0;

	if (!adreno_hwsched_context_queue_enabled(adreno_dev))
		return 0;

	if (test_bit(ADRENO_HWSCHED_HW_FENCE, &adreno_dev->hwsched.flags) &&
		!drawctxt->gmu_hw_fence_queue.gmuaddr) {
		ret = gen7_alloc_gmu_kernel_block(
			to_gen7_gmu(adreno_dev), &drawctxt->gmu_hw_fence_queue,
			HW_FENCE_QUEUE_SIZE, GMU_NONCACHED_KERNEL,
			IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV);
		if (ret) {
			memset(&drawctxt->gmu_hw_fence_queue, 0x0,
				sizeof(drawctxt->gmu_hw_fence_queue));
			return ret;
		}
	}

	if (!drawctxt->gmu_context_queue.gmuaddr) {
		ret = gen7_alloc_gmu_kernel_block(
			to_gen7_gmu(adreno_dev), &drawctxt->gmu_context_queue,
			SZ_4K, GMU_NONCACHED_KERNEL,
			IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV);
		if (ret) {
			memset(&drawctxt->gmu_context_queue, 0x0,
				sizeof(drawctxt->gmu_context_queue));
			return ret;
		}
		init_gmu_context_queue(drawctxt);
	}

	return 0;
}

static int send_context_register(struct adreno_device *adreno_dev,
	struct kgsl_context *context)
{
	struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
	struct hfi_register_ctxt_cmd cmd;
	struct kgsl_pagetable *pt = context->proc_priv->pagetable;
	int ret, asid = kgsl_mmu_pagetable_get_asid(pt, context);

	if (asid < 0)
		return asid;

	ret = CMD_MSG_HDR(cmd, H2F_MSG_REGISTER_CONTEXT);
	if (ret)
		return ret;

	ret = allocate_context_queues(adreno_dev, drawctxt);
	if (ret)
		return ret;

	cmd.ctxt_id = context->id;
	cmd.flags = HFI_CTXT_FLAG_NOTIFY | context->flags;
	/*
	 * HLOS SMMU driver programs context bank to look up ASID from TTBR0 during a page
	 * table walk. So the TLB entries are tagged with the ASID from TTBR0. TLBIASID
	 * invalidates TLB entries whose ASID matches the value that was written to the
	 * CBn_TLBIASID register. Set ASID along with PT address.
	 */
	cmd.pt_addr = kgsl_mmu_pagetable_get_ttbr0(pt) |
		FIELD_PREP(GENMASK_ULL(63, KGSL_IOMMU_ASID_START_BIT), asid);
	cmd.ctxt_idr = context->id;
	cmd.ctxt_bank = kgsl_mmu_pagetable_get_context_bank(pt, context);

	return gen7_hfi_send_cmd_async(adreno_dev, &cmd, sizeof(cmd));
}

static int send_context_pointers(struct adreno_device *adreno_dev,
	struct kgsl_context *context)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	struct hfi_context_pointers_cmd cmd = {0};
	struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
	int ret;

	ret = CMD_MSG_HDR(cmd, H2F_MSG_CONTEXT_POINTERS);
	if (ret)
		return ret;

	cmd.ctxt_id = context->id;
	cmd.sop_addr = MEMSTORE_ID_GPU_ADDR(device, context->id, soptimestamp);
	cmd.eop_addr = MEMSTORE_ID_GPU_ADDR(device, context->id, eoptimestamp);
	if (context->user_ctxt_record)
		cmd.user_ctxt_record_addr =
			context->user_ctxt_record->memdesc.gpuaddr;

	if (adreno_hwsched_context_queue_enabled(adreno_dev))
		cmd.gmu_context_queue_addr = drawctxt->gmu_context_queue.gmuaddr;

	return gen7_hfi_send_cmd_async(adreno_dev, &cmd, sizeof(cmd));
}

static int hfi_context_register(struct adreno_device *adreno_dev,
	struct kgsl_context *context)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	int ret;

	if (context->gmu_registered)
		return 0;

	ret = send_context_register(adreno_dev, context);
	if (ret) {
		dev_err(&gmu->pdev->dev,
			"Unable to register context %u: %d\n",
			context->id, ret);

		if (device->gmu_fault)
			gen7_hwsched_fault(adreno_dev, ADRENO_GMU_FAULT);

		return ret;
	}

	ret = send_context_pointers(adreno_dev, context);
	if (ret) {
		dev_err(&gmu->pdev->dev,
			"Unable to register context %u pointers: %d\n",
			context->id, ret);

		if (device->gmu_fault)
			gen7_hwsched_fault(adreno_dev, ADRENO_GMU_FAULT);

		return ret;
	}

	context->gmu_registered = true;
	if (adreno_hwsched_context_queue_enabled(adreno_dev))
		context->gmu_dispatch_queue = UINT_MAX;
	else
		context->gmu_dispatch_queue = get_dq_id(adreno_dev, context);

	return 0;
}

static void populate_ibs(struct adreno_device *adreno_dev,
	struct hfi_submit_cmd *cmd, struct kgsl_drawobj_cmd *cmdobj)
{
	struct hfi_issue_ib *issue_ib;
	struct kgsl_memobj_node *ib;

	if (cmdobj->numibs > HWSCHED_MAX_DISPATCH_NUMIBS) {
		struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
		struct kgsl_memdesc *big_ib;

		if (test_bit(CMDOBJ_RECURRING_START, &cmdobj->priv))
			big_ib = hfi->big_ib_recurring;
		else
			big_ib = hfi->big_ib;
		/*
		 * The dispatcher ensures that there is only one big IB inflight
		 */
		cmd->big_ib_gmu_va = big_ib->gmuaddr;
		cmd->flags |= CMDBATCH_INDIRECT;
		issue_ib = big_ib->hostptr;
	} else {
		issue_ib = (struct hfi_issue_ib *)&cmd[1];
	}

	list_for_each_entry(ib, &cmdobj->cmdlist, node) {
		issue_ib->addr = ib->gpuaddr;
		issue_ib->size = ib->size;
		issue_ib++;
	}

	cmd->numibs = cmdobj->numibs;
}

#define HFI_DSP_IRQ_BASE 2

#define DISPQ_IRQ_BIT(_idx) BIT((_idx) + HFI_DSP_IRQ_BASE)

int gen7_gmu_context_queue_write(struct adreno_device *adreno_dev,
	struct kgsl_memdesc *gmu_context_queue, u32 *msg, u32 size_bytes,
	struct kgsl_drawobj *drawobj, struct adreno_submit_time *time)
{
	struct gmu_context_queue_header *hdr = gmu_context_queue->hostptr;
	const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
	u32 *queue = gmu_context_queue->hostptr + sizeof(*hdr);
	u32 i, empty_space, write_idx = hdr->write_index, read_idx = hdr->read_index;
	u32 size_dwords = size_bytes >> 2;
	u32 align_size = ALIGN(size_dwords, SZ_4);
	u32 id = MSG_HDR_GET_ID(*msg);
	struct kgsl_drawobj_cmd *cmdobj = NULL;

	empty_space = (write_idx >= read_idx) ?
			(hdr->queue_size - (write_idx - read_idx))
			: (read_idx - write_idx);

	if (empty_space <= align_size)
		return -ENOSPC;

	if (!IS_ALIGNED(size_bytes, sizeof(u32)))
		return -EINVAL;

	for (i = 0; i < size_dwords; i++) {
		queue[write_idx] = msg[i];
		write_idx = (write_idx + 1) % hdr->queue_size;
	}

	/* Cookify any non used data at the end of the write buffer */
	for (; i < align_size; i++) {
		queue[write_idx] = 0xfafafafa;
		write_idx = (write_idx + 1) % hdr->queue_size;
	}

	/* Ensure packet is written out before proceeding */
	wmb();

	if (!drawobj)
		goto done;

	if (drawobj->type & SYNCOBJ_TYPE) {
		struct kgsl_drawobj_sync *syncobj = SYNCOBJ(drawobj);

		trace_adreno_syncobj_submitted(drawobj->context->id, drawobj->timestamp,
			syncobj->numsyncs, gpudev->read_alwayson(adreno_dev));
		goto done;
	}

	cmdobj = CMDOBJ(drawobj);

	gen7_add_profile_events(adreno_dev, cmdobj, time);

	/*
	 * Put the profiling information in the user profiling buffer.
	 * The hfi_update_write_idx below has a wmb() before the actual
	 * write index update to ensure that the GMU does not see the
	 * packet before the profile data is written out.
	 */
	adreno_profile_submit_time(time);

done:
	trace_kgsl_hfi_send(id, size_dwords, MSG_HDR_GET_SEQNUM(*msg));

	hfi_update_write_idx(&hdr->write_index, write_idx);

	return 0;
}

static u32 get_irq_bit(struct adreno_device *adreno_dev, struct kgsl_drawobj *drawobj)
{
	if (!adreno_hwsched_context_queue_enabled(adreno_dev))
		return drawobj->context->gmu_dispatch_queue;

	if (adreno_is_preemption_enabled(adreno_dev))
		return adreno_get_level(drawobj->context);

	if (kgsl_context_is_lpac(drawobj->context))
		return 1;

	return 0;
}

static int add_gmu_waiter(struct adreno_device *adreno_dev,
	struct dma_fence *fence)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	int ret = msm_hw_fence_wait_update(adreno_dev->hwsched.hw_fence.handle,
			&fence, 1, true);

	if (ret)
		dev_err_ratelimited(device->dev,
			"Failed to add GMU as waiter ret:%d fence ctx:%llu ts:%llu\n",
			ret, fence->context, fence->seqno);

	return ret;
}

static void populate_kgsl_fence(struct hfi_syncobj *obj,
	struct dma_fence *fence)
{
	struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
	struct kgsl_sync_timeline *ktimeline = kfence->parent;
	unsigned long flags;

	obj->flags |= BIT(GMU_SYNCOBJ_FLAG_KGSL_FENCE_BIT);

	spin_lock_irqsave(&ktimeline->lock, flags);
	/* If the context is going away or the dma fence is signaled, mark the fence as triggered */
	if (!ktimeline->context || dma_fence_is_signaled_locked(fence)) {
		obj->flags |= BIT(GMU_SYNCOBJ_FLAG_SIGNALED_BIT);
		spin_unlock_irqrestore(&ktimeline->lock, flags);
		return;
	}
	obj->ctxt_id = ktimeline->context->id;
	spin_unlock_irqrestore(&ktimeline->lock, flags);

	obj->seq_no =  kfence->timestamp;
}

static int _submit_hw_fence(struct adreno_device *adreno_dev,
	struct kgsl_drawobj *drawobj, void *cmdbuf)
{
	struct adreno_context *drawctxt = ADRENO_CONTEXT(drawobj->context);
	int i, j;
	u32 cmd_sizebytes;
	struct kgsl_drawobj_sync *syncobj = SYNCOBJ(drawobj);
	struct hfi_submit_syncobj *cmd;
	struct hfi_syncobj *obj = NULL;
	u32 seqnum;

	/* Add hfi_syncobj struct for sync object */
	cmd_sizebytes = sizeof(*cmd) +
			(sizeof(struct hfi_syncobj) *
			syncobj->num_hw_fence);

	if (WARN_ON(cmd_sizebytes > HFI_MAX_MSG_SIZE))
		return -EMSGSIZE;

	memset(cmdbuf, 0x0, cmd_sizebytes);
	cmd = cmdbuf;
	cmd->num_syncobj = syncobj->num_hw_fence;
	obj = (struct hfi_syncobj *)&cmd[1];

	for (i = 0; i < syncobj->numsyncs; i++) {
		struct kgsl_drawobj_sync_event *event = &syncobj->synclist[i];
		struct kgsl_sync_fence_cb *kcb = event->handle;
		struct dma_fence **fences;
		struct dma_fence_array *array;
		u32 num_fences;

		if (!kcb)
			return -EINVAL;

		array = to_dma_fence_array(kcb->fence);
		if (array != NULL) {
			num_fences = array->num_fences;
			fences = array->fences;
		} else {
			num_fences = 1;
			fences = &kcb->fence;
		}

		for (j = 0; j < num_fences; j++) {

			/*
			 * If this sync object has a software only fence, make sure that it is
			 * already signaled so that we can skip sending this fence to the GMU.
			 */
			if (!test_bit(MSM_HW_FENCE_FLAG_ENABLED_BIT, &fences[j]->flags)) {
				if (WARN(!dma_fence_is_signaled(fences[j]),
					"sync object has unsignaled software fence"))
					return -EINVAL;
				continue;
			}

			if (is_kgsl_fence(fences[j])) {
				populate_kgsl_fence(obj, fences[j]);
			} else {
				int ret = add_gmu_waiter(adreno_dev, fences[j]);

				if (ret) {
					syncobj->flags &= ~KGSL_SYNCOBJ_HW;
					return ret;
				}

				if (test_bit(MSM_HW_FENCE_FLAG_SIGNALED_BIT, &fences[j]->flags) ||
					test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fences[j]->flags))
					obj->flags |= BIT(GMU_SYNCOBJ_FLAG_SIGNALED_BIT);

				obj->ctxt_id = fences[j]->context;
				obj->seq_no =  fences[j]->seqno;
			}
			trace_adreno_input_hw_fence(drawobj->context->id, obj->ctxt_id,
				obj->seq_no, obj->flags, fences[j]->ops->get_timeline_name ?
				fences[j]->ops->get_timeline_name(fences[j]) : "unknown");

			obj++;
		}
	}

	/*
	 * Attach a timestamp to this SYNCOBJ to keep track whether GMU has deemed it signaled
	 * or not.
	 */
	drawobj->timestamp = ++drawctxt->syncobj_timestamp;
	cmd->timestamp = drawobj->timestamp;

	seqnum = atomic_inc_return(&adreno_dev->hwsched.submission_seqnum);
	cmd->hdr = CREATE_MSG_HDR(H2F_MSG_ISSUE_SYNCOBJ, HFI_MSG_CMD);
	cmd->hdr = MSG_HDR_SET_SEQNUM_SIZE(cmd->hdr, seqnum, cmd_sizebytes >> 2);

	return gen7_gmu_context_queue_write(adreno_dev, &drawctxt->gmu_context_queue,
			(u32 *)cmd, cmd_sizebytes, drawobj, NULL);
}

int gen7_hwsched_check_context_inflight_hw_fences(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt)
{
	struct adreno_hw_fence_entry *entry, *tmp;
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	int ret = 0;

	spin_lock(&drawctxt->lock);
	list_for_each_entry_safe(entry, tmp, &drawctxt->hw_fence_inflight_list, node) {
		struct gmu_context_queue_header *hdr =  drawctxt->gmu_context_queue.hostptr;

		if (timestamp_cmp((u32)entry->cmd.ts, hdr->out_fence_ts) > 0) {
			dev_err(&gmu->pdev->dev,
				"detached ctx:%d has unsignaled fence ts:%d retired:%d\n",
				drawctxt->base.id, (u32)entry->cmd.ts, hdr->out_fence_ts);
			ret = -EINVAL;
			break;
		}

		gen7_remove_hw_fence_entry(adreno_dev, entry);
	}
	spin_unlock(&drawctxt->lock);

	return ret;
}

/**
 * move_detached_context_hardware_fences - Move all pending hardware fences belonging to this
 * context to the detached hardware fence list so as to send them to TxQueue after fault recovery.
 * This is needed because this context may get destroyed before fault recovery gets executed.
 */
static void move_detached_context_hardware_fences(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt)
{
	struct adreno_hw_fence_entry *entry, *tmp;
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);

	/* We don't need the drawctxt lock here because this context has already been detached */
	list_for_each_entry_safe(entry, tmp, &drawctxt->hw_fence_inflight_list, node) {
		struct gmu_context_queue_header *hdr =  drawctxt->gmu_context_queue.hostptr;

		if ((timestamp_cmp((u32)entry->cmd.ts, hdr->out_fence_ts) > 0)) {
			_kgsl_context_get(&drawctxt->base);
			list_move_tail(&entry->node, &hfi->detached_hw_fence_list);
			continue;
		}

		gen7_remove_hw_fence_entry(adreno_dev, entry);
	}

	/* Also grab all the hardware fences which were never sent to GMU */
	list_for_each_entry_safe(entry, tmp, &drawctxt->hw_fence_list, node) {
		_kgsl_context_get(&drawctxt->base);
		list_move_tail(&entry->node, &hfi->detached_hw_fence_list);
	}
}

/**
 * check_detached_context_hardware_fences - When this context has been un-registered with the GMU,
 * make sure all the hardware fences(that were sent to GMU) for this context have been sent to
 * TxQueue. Also, send any hardware fences (to GMU) that were not yet dispatched to the GMU. In case
 * of an error, move the pending hardware fences to detached hardware fence list, log the error,
 * take a snapshot and trigger recovery.
 */
static int check_detached_context_hardware_fences(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	struct adreno_hw_fence_entry *entry, *tmp;
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	int ret = 0;

	/* We don't need the drawctxt lock because this context has been detached */
	list_for_each_entry_safe(entry, tmp, &drawctxt->hw_fence_inflight_list, node) {
		struct gmu_context_queue_header *hdr =  drawctxt->gmu_context_queue.hostptr;

		if ((timestamp_cmp((u32)entry->cmd.ts, hdr->out_fence_ts) > 0)) {
			dev_err(&gmu->pdev->dev,
				"detached ctx:%d has unsignaled fence ts:%d retired:%d\n",
				drawctxt->base.id, (u32)entry->cmd.ts, hdr->out_fence_ts);
			ret = -EINVAL;
			goto fault;
		}
		gen7_remove_hw_fence_entry(adreno_dev, entry);
	}

	/* Send hardware fences (to TxQueue) that were not dispatched to GMU */
	list_for_each_entry_safe(entry, tmp, &drawctxt->hw_fence_list, node) {

		ret = gen7_send_hw_fence_hfi_wait_ack(adreno_dev, entry,
			HW_FENCE_FLAG_SKIP_MEMSTORE);
		if (ret)
			goto fault;

		gen7_remove_hw_fence_entry(adreno_dev, entry);
	}

	return 0;

fault:
	move_detached_context_hardware_fences(adreno_dev, drawctxt);
	gmu_core_fault_snapshot(device);
	gen7_hwsched_fault(adreno_dev, ADRENO_GMU_FAULT);

	return ret;
}

static inline int setup_hw_fence_info_cmd(struct adreno_device *adreno_dev,
	struct adreno_hw_fence_entry *entry)
{
	struct kgsl_sync_fence *kfence = entry->kfence;
	int ret;

	ret = CMD_MSG_HDR(entry->cmd, H2F_MSG_HW_FENCE_INFO);
	if (ret)
		return ret;

	entry->cmd.gmu_ctxt_id = entry->drawctxt->base.id;
	entry->cmd.ctxt_id = kfence->fence.context;
	entry->cmd.ts = kfence->fence.seqno;

	entry->cmd.hash_index = kfence->hw_fence_index;

	return 0;
}

/*
 * gen7_send_hw_fence_hfi_wait_ack - This function is used in cases where multiple hardware fences
 * are to be sent to GMU. Hence, we must send them one by one to avoid overwhelming the GMU with
 * mutliple fences in a short span of time.
 */
int gen7_send_hw_fence_hfi_wait_ack(struct adreno_device *adreno_dev,
	struct adreno_hw_fence_entry *entry, u64 flags)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	u32 seqnum;
	int ret = 0;

	/* Device mutex is necessary to ensure only one hardware fence ack is being waited for */
	if (WARN_ON(!mutex_is_locked(&device->mutex)))
		return -EINVAL;

	spin_lock(&hfi->hw_fence.lock);

	init_completion(&hw_fence_ack.complete);

	entry->cmd.flags |= flags;
	seqnum = atomic_inc_return(&hfi->hw_fence.seqnum);
	entry->cmd.hdr = MSG_HDR_SET_SEQNUM_SIZE(entry->cmd.hdr, seqnum, sizeof(entry->cmd) >> 2);

	hw_fence_ack.sent_hdr = entry->cmd.hdr;

	/*
	 * We don't need to increment the unack count here as we are waiting for the ack for
	 * this fence before sending another hardware fence.
	 */
	ret = gen7_hfi_cmdq_write(adreno_dev, (u32 *)&entry->cmd, sizeof(entry->cmd));

	spin_unlock(&hfi->hw_fence.lock);

	if (!ret)
		ret = adreno_hwsched_wait_ack_completion(adreno_dev, &gmu->pdev->dev, &hw_fence_ack,
			gen7_hwsched_process_msgq);

	memset(&hw_fence_ack, 0x0, sizeof(hw_fence_ack));
	return ret;
}

/**
 * drawctxt_queue_hw_fence - Add a hardware fence to draw context's hardware fence list and make
 * sure the list remains sorted (with the fence with the largest timestamp at the end)
 */
static void drawctxt_queue_hw_fence(struct adreno_context *drawctxt,
	struct adreno_hw_fence_entry *new)
{
	struct adreno_hw_fence_entry *entry = NULL;
	u32 ts = (u32)new->cmd.ts;

	/* Walk the list backwards to find the right spot for this fence */
	list_for_each_entry_reverse(entry, &drawctxt->hw_fence_list, node) {
		if (timestamp_cmp(ts, (u32)entry->cmd.ts) > 0)
			break;
	}

	list_add(&new->node, &entry->node);
}

#define DRAWCTXT_SLOT_AVAILABLE(count)  \
	((count + 1) < (HW_FENCE_QUEUE_SIZE / sizeof(struct hfi_hw_fence_info)))

/**
 * allocate_hw_fence_entry - Allocate an entry to keep track of a hardware fence. This is free'd
 * when we know GMU has sent this fence to the TxQueue
 */
static struct adreno_hw_fence_entry *allocate_hw_fence_entry(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt, struct kgsl_sync_fence *kfence)
{
	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
	struct adreno_hw_fence_entry *entry;

	if (!DRAWCTXT_SLOT_AVAILABLE(drawctxt->hw_fence_count))
		return NULL;

	entry = kmem_cache_zalloc(hwsched->hw_fence_cache, GFP_ATOMIC);
	if (!entry)
		return NULL;

	entry->kfence = kfence;
	entry->drawctxt = drawctxt;

	if (setup_hw_fence_info_cmd(adreno_dev, entry)) {
		kmem_cache_free(hwsched->hw_fence_cache, entry);
		return NULL;
	}

	dma_fence_get(&kfence->fence);

	drawctxt->hw_fence_count++;
	atomic_inc(&hwsched->hw_fence_count);

	INIT_LIST_HEAD(&entry->node);
	INIT_LIST_HEAD(&entry->reset_node);
	return entry;
}

static bool _hw_fence_end_sleep(struct adreno_device *adreno_dev)
{
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	bool ret;

	spin_lock(&hfi->hw_fence.lock);
	ret = !test_bit(GEN7_HWSCHED_HW_FENCE_SLEEP_BIT, &hfi->hw_fence.flags);
	spin_unlock(&hfi->hw_fence.lock);

	return ret;
}

/**
 * _hw_fence_sleep() - Check if the thread needs to sleep until the hardware fence unack count
 * drops to a desired threshold.
 *
 * Return: negative error code if the thread was woken up by a signal, or the context became bad in
 * the meanwhile, or the hardware fence unack count hasn't yet dropped to a desired threshold, or
 * if fault recovery is imminent.
 * Otherwise, return 0.
 */
static int _hw_fence_sleep(struct adreno_device *adreno_dev, struct adreno_context *drawctxt)
{
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	int ret = 0;

	if (!test_bit(GEN7_HWSCHED_HW_FENCE_SLEEP_BIT, &hfi->hw_fence.flags))
		return 0;

	spin_unlock(&hfi->hw_fence.lock);
	spin_unlock(&drawctxt->lock);

	ret = wait_event_interruptible(hfi->hw_fence.unack_wq,
		_hw_fence_end_sleep(adreno_dev));

	spin_lock(&drawctxt->lock);
	spin_lock(&hfi->hw_fence.lock);

	/*
	 * If the thread received a signal, or the context became bad in the meanwhile or the limit
	 * is still not settled, then return error to avoid creating this hardware fence
	 */
	if ((ret == -ERESTARTSYS) || kgsl_context_is_bad(&drawctxt->base) ||
		test_bit(GEN7_HWSCHED_HW_FENCE_MAX_BIT, &hfi->hw_fence.flags))
		return -EINVAL;

	/*
	 * If fault recovery is imminent then return error code to avoid creating new hardware
	 * fences until recovery is complete
	 */
	if (test_bit(GEN7_HWSCHED_HW_FENCE_ABORT_BIT, &hfi->hw_fence.flags))
		return -EBUSY;

	return ret;
}

void gen7_hwsched_create_hw_fence(struct adreno_device *adreno_dev,
	struct kgsl_sync_fence *kfence)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	struct kgsl_sync_timeline *ktimeline = kfence->parent;
	struct kgsl_context *context = ktimeline->context;
	struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
	struct adreno_hw_fence_entry *entry = NULL;
	struct msm_hw_fence_create_params params = {0};
	/* Only allow a single log in a second */
	static DEFINE_RATELIMIT_STATE(_rs, HZ, 1);
	struct gen7_hwsched_hfi *hw_hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	u32 retired = 0;
	int ret = 0;
	bool destroy_hw_fence = true;

	params.fence = &kfence->fence;
	params.handle = &kfence->hw_fence_index;
	kfence->hw_fence_handle = adreno_dev->hwsched.hw_fence.handle;

	ret = msm_hw_fence_create(kfence->hw_fence_handle, &params);
	if ((ret || IS_ERR_OR_NULL(params.handle))) {
		if (__ratelimit(&_rs))
			dev_err(device->dev, "Failed to create ctx:%d ts:%d hardware fence:%d\n",
				kfence->context_id, kfence->timestamp, ret);
		return;
	}

	spin_lock(&drawctxt->lock);
	spin_lock(&hw_hfi->hw_fence.lock);

	/*
	 * If we create a hardware fence and this context is going away, we may never dispatch
	 * this fence to the GMU. Hence, avoid creating a hardware fence if context is going away.
	 */
	if (kgsl_context_is_bad(context))
		goto done;

	entry = allocate_hw_fence_entry(adreno_dev, drawctxt, kfence);
	if (!entry)
		goto done;

	/* If recovery is imminent, then do not create a hardware fence */
	if (test_bit(GEN7_HWSCHED_HW_FENCE_ABORT_BIT, &hw_hfi->hw_fence.flags)) {
		destroy_hw_fence = true;
		goto done;
	}

	ret = _hw_fence_sleep(adreno_dev, drawctxt);
	if (ret)
		goto done;

	/*
	 * If this ts hasn't been submitted yet, then store it in the drawctxt hardware fence
	 * list and return. This fence will be sent to GMU when this ts is dispatched to GMU.
	 */
	if (timestamp_cmp(kfence->timestamp, drawctxt->internal_timestamp) > 0) {
		drawctxt_queue_hw_fence(drawctxt, entry);
		destroy_hw_fence = false;
		goto done;
	}

	kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_RETIRED, &retired);

	/*
	 * Check if timestamp is retired. If we are in SLUMBER at this point, the timestamp is
	 * guaranteed to be retired. This way, we don't need the device mutex to check the device
	 * state explicitly.
	 */
	if (timestamp_cmp(retired, kfence->timestamp) >= 0) {
		kgsl_sync_timeline_signal(ktimeline, kfence->timestamp);
		goto done;
	}

	/*
	 * If timestamp is not retired then GMU must already be powered up. This is because SLUMBER
	 * thread has to wait for hardware fence spinlock to make sure the hardware fence unack
	 * count is zero.
	 */
	ret = _send_hw_fence_no_ack(adreno_dev, entry);
	if (ret) {
		if (__ratelimit(&_rs))
			dev_err(&gmu->pdev->dev, "Aborting hw fence for ctx:%d ts:%d ret:%d\n",
				kfence->context_id, kfence->timestamp, ret);
		goto done;
	}

	list_add_tail(&entry->node, &drawctxt->hw_fence_inflight_list);

	destroy_hw_fence = false;

done:
	if (destroy_hw_fence) {
		msm_hw_fence_destroy(kfence->hw_fence_handle, &kfence->fence);
		if (entry)
			gen7_remove_hw_fence_entry(adreno_dev, entry);
	}

	spin_unlock(&hw_hfi->hw_fence.lock);
	spin_unlock(&drawctxt->lock);
}

/**
 * setup_hw_fence_deferred_ctxt - The hardware fence(s) from this context couldn't be sent to the
 * GMU because the hardware fence unack count reached a threshold. Hence, setup this context such
 * that these hardware fences are sent to the GMU when the unack count drops to a desired threshold.
 */
static void setup_hw_fence_deferred_ctxt(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt, u32 ts)
{
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);

	if (!_kgsl_context_get(&drawctxt->base))
		return;

	hfi->hw_fence.defer_drawctxt = drawctxt;
	hfi->hw_fence.defer_ts = ts;
	/*
	 * Increment the active count so that device doesn't get powered off until this fence has
	 * been sent to GMU
	 */
	gen7_hwsched_active_count_get(adreno_dev);
}

/**
 * process_hw_fence_queue - This function walks the draw context's list of hardware fences
 * and sends the ones which have a timestamp less than or equal to the timestamp that just
 * got submitted to the GMU.
 */
static void process_hw_fence_queue(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt, u32 ts)
{
	struct adreno_hw_fence_entry *entry = NULL, *next;
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	int ret = 0;

	/* This list is sorted with smallest timestamp at head and highest timestamp at tail */
	list_for_each_entry_safe(entry, next, &drawctxt->hw_fence_list, node) {

		if (timestamp_cmp((u32)entry->cmd.ts, ts) > 0)
			return;

		spin_lock(&hfi->hw_fence.lock);

		if (test_bit(GEN7_HWSCHED_HW_FENCE_MAX_BIT, &hfi->hw_fence.flags)) {
			setup_hw_fence_deferred_ctxt(adreno_dev, drawctxt, ts);
			spin_unlock(&hfi->hw_fence.lock);
			return;
		}

		ret = _send_hw_fence_no_ack(adreno_dev, entry);

		spin_unlock(&hfi->hw_fence.lock);

		if (ret)
			return;

		/*
		 * A fence that is sent to GMU must be added to the drawctxt->hw_fence_inflight_list
		 * so that we can keep track of when GMU sends it to the TxQueue
		 */
		list_del_init(&entry->node);
		list_add_tail(&entry->node, &drawctxt->hw_fence_inflight_list);
	}
}

/* Size in below functions are in unit of dwords */
static int gen7_hfi_dispatch_queue_write(struct adreno_device *adreno_dev, u32 queue_idx,
	u32 *msg, u32 size_bytes, struct kgsl_drawobj_cmd *cmdobj, struct adreno_submit_time *time)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct hfi_queue_table *tbl = gmu->hfi.hfi_mem->hostptr;
	struct hfi_queue_header *hdr = &tbl->qhdr[queue_idx];
	u32 *queue;
	u32 i, write, empty_space;
	u32 size_dwords = size_bytes >> 2;
	u32 align_size = ALIGN(size_dwords, SZ_4);
	u32 id = MSG_HDR_GET_ID(*msg);

	if (hdr->status == HFI_QUEUE_STATUS_DISABLED || !IS_ALIGNED(size_bytes, sizeof(u32)))
		return -EINVAL;

	queue = HOST_QUEUE_START_ADDR(gmu->hfi.hfi_mem, queue_idx);

	empty_space = (hdr->write_index >= hdr->read_index) ?
			(hdr->queue_size - (hdr->write_index - hdr->read_index))
			: (hdr->read_index - hdr->write_index);

	if (empty_space <= align_size)
		return -ENOSPC;

	write = hdr->write_index;

	for (i = 0; i < size_dwords; i++) {
		queue[write] = msg[i];
		write = (write + 1) % hdr->queue_size;
	}

	/* Cookify any non used data at the end of the write buffer */
	for (; i < align_size; i++) {
		queue[write] = 0xfafafafa;
		write = (write + 1) % hdr->queue_size;
	}

	/* Ensure packet is written out before proceeding */
	wmb();

	if (!cmdobj)
		goto done;

	gen7_add_profile_events(adreno_dev, cmdobj, time);

	/*
	 * Put the profiling information in the user profiling buffer.
	 * The hfi_update_write_idx below has a wmb() before the actual
	 * write index update to ensure that the GMU does not see the
	 * packet before the profile data is written out.
	 */
	adreno_profile_submit_time(time);

done:
	trace_kgsl_hfi_send(id, size_dwords, MSG_HDR_GET_SEQNUM(*msg));

	hfi_update_write_idx(&hdr->write_index, write);

	return 0;
}

int gen7_hwsched_submit_drawobj(struct adreno_device *adreno_dev, struct kgsl_drawobj *drawobj)
{
	int ret = 0;
	u32 cmd_sizebytes, seqnum;
	struct kgsl_drawobj_cmd *cmdobj = NULL;
	struct hfi_submit_cmd *cmd;
	struct adreno_submit_time time = {0};
	struct adreno_context *drawctxt = ADRENO_CONTEXT(drawobj->context);
	static void *cmdbuf;

	if (cmdbuf == NULL) {
		struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

		cmdbuf = devm_kzalloc(&device->pdev->dev, HFI_MAX_MSG_SIZE,
				GFP_KERNEL);
		if (!cmdbuf)
			return -ENOMEM;
	}

	ret = hfi_context_register(adreno_dev, drawobj->context);
	if (ret)
		return ret;

	if ((drawobj->type & SYNCOBJ_TYPE) != 0)
		return _submit_hw_fence(adreno_dev, drawobj, cmdbuf);

	cmdobj = CMDOBJ(drawobj);

	/*
	 * If the MARKER object is retired, it doesn't need to be dispatched to GMU. Simply trigger
	 * any pending fences that are less than/equal to this object's timestamp.
	 */
	if (test_bit(CMDOBJ_MARKER_EXPIRED, &cmdobj->priv)) {
		spin_lock(&drawctxt->lock);
		process_hw_fence_queue(adreno_dev, drawctxt, drawobj->timestamp);
		spin_unlock(&drawctxt->lock);
		return 0;
	}

	/* Add a *issue_ib struct for each IB */
	if (cmdobj->numibs > HWSCHED_MAX_DISPATCH_NUMIBS ||
		test_bit(CMDOBJ_SKIP, &cmdobj->priv))
		cmd_sizebytes = sizeof(*cmd);
	else
		cmd_sizebytes = sizeof(*cmd) +
			(sizeof(struct hfi_issue_ib) * cmdobj->numibs);

	if (WARN_ON(cmd_sizebytes > HFI_MAX_MSG_SIZE))
		return -EMSGSIZE;

	memset(cmdbuf, 0x0, cmd_sizebytes);

	cmd = cmdbuf;

	cmd->ctxt_id = drawobj->context->id;
	cmd->flags = HFI_CTXT_FLAG_NOTIFY;
	if (drawobj->flags & KGSL_DRAWOBJ_END_OF_FRAME)
		cmd->flags |= CMDBATCH_EOF;

	cmd->ts = drawobj->timestamp;

	if (test_bit(CMDOBJ_SKIP, &cmdobj->priv))
		goto skipib;

	populate_ibs(adreno_dev, cmd, cmdobj);

	if ((drawobj->flags & KGSL_DRAWOBJ_PROFILING) &&
		cmdobj->profiling_buf_entry) {

		time.drawobj = drawobj;

		cmd->profile_gpuaddr_lo =
			lower_32_bits(cmdobj->profiling_buffer_gpuaddr);
		cmd->profile_gpuaddr_hi =
			upper_32_bits(cmdobj->profiling_buffer_gpuaddr);

		/* Indicate to GMU to do user profiling for this submission */
		cmd->flags |= CMDBATCH_PROFILING;
	}

skipib:
	adreno_drawobj_set_constraint(KGSL_DEVICE(adreno_dev), drawobj);

	seqnum = atomic_inc_return(&adreno_dev->hwsched.submission_seqnum);
	cmd->hdr = CREATE_MSG_HDR(H2F_MSG_ISSUE_CMD, HFI_MSG_CMD);
	cmd->hdr = MSG_HDR_SET_SEQNUM_SIZE(cmd->hdr, seqnum, cmd_sizebytes >> 2);

	if (adreno_hwsched_context_queue_enabled(adreno_dev))
		ret = gen7_gmu_context_queue_write(adreno_dev,
			&drawctxt->gmu_context_queue, (u32 *)cmd, cmd_sizebytes, drawobj, &time);
	else
		ret = gen7_hfi_dispatch_queue_write(adreno_dev,
			HFI_DSP_ID_0 + drawobj->context->gmu_dispatch_queue,
			(u32 *)cmd, cmd_sizebytes, cmdobj, &time);
	if (ret)
		return ret;

	/* Send interrupt to GMU to receive the message */
	gmu_core_regwrite(KGSL_DEVICE(adreno_dev), GEN7_GMU_HOST2GMU_INTR_SET,
		DISPQ_IRQ_BIT(get_irq_bit(adreno_dev, drawobj)));

	spin_lock(&drawctxt->lock);
	process_hw_fence_queue(adreno_dev, drawctxt, drawobj->timestamp);
	/*
	 * We need to update the internal timestamp while holding the drawctxt lock since we have to
	 * check it in the hardware fence creation path, where we are not taking the device mutex.
	 */
	drawctxt->internal_timestamp = drawobj->timestamp;
	spin_unlock(&drawctxt->lock);

	return 0;
}

int gen7_hwsched_send_recurring_cmdobj(struct adreno_device *adreno_dev,
	struct kgsl_drawobj_cmd *cmdobj)
{
	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
	struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj);
	struct hfi_submit_cmd *cmd;
	struct kgsl_memobj_node *ib;
	u32 cmd_sizebytes;
	int ret;
	static bool active;

	if (adreno_gpu_halt(adreno_dev) || adreno_hwsched_gpu_fault(adreno_dev))
		return -EBUSY;

	if (test_bit(CMDOBJ_RECURRING_STOP, &cmdobj->priv)) {
		cmdobj->numibs = 0;
	} else {
		list_for_each_entry(ib, &cmdobj->cmdlist, node)
			cmdobj->numibs++;
	}

	if (cmdobj->numibs > HWSCHED_MAX_IBS)
		return -EINVAL;

	if (cmdobj->numibs > HWSCHED_MAX_DISPATCH_NUMIBS)
		cmd_sizebytes = sizeof(*cmd);
	else
		cmd_sizebytes = sizeof(*cmd) +
			(sizeof(struct hfi_issue_ib) * cmdobj->numibs);

	if (WARN_ON(cmd_sizebytes > HFI_MAX_MSG_SIZE))
		return -EMSGSIZE;

	cmd = kzalloc(cmd_sizebytes, GFP_KERNEL);
	if (cmd == NULL)
		return -ENOMEM;

	if (test_bit(CMDOBJ_RECURRING_START, &cmdobj->priv)) {
		if (!active) {
			ret = adreno_active_count_get(adreno_dev);
			if (ret) {
				kfree(cmd);
				return ret;
			}
			active = true;
		}
		cmd->flags |= CMDBATCH_RECURRING_START;
		populate_ibs(adreno_dev, cmd, cmdobj);
	} else
		cmd->flags |= CMDBATCH_RECURRING_STOP;

	cmd->ctxt_id = drawobj->context->id;

	ret = hfi_context_register(adreno_dev, drawobj->context);
	if (ret) {
		adreno_active_count_put(adreno_dev);
		active = false;
		kfree(cmd);
		return ret;
	}

	cmd->hdr = CREATE_MSG_HDR(H2F_MSG_ISSUE_RECURRING_CMD, HFI_MSG_CMD);

	ret = gen7_hfi_send_cmd_async(adreno_dev, cmd, sizeof(*cmd));

	kfree(cmd);

	if (ret) {
		adreno_active_count_put(adreno_dev);
		active = false;
		return ret;
	}

	if (test_bit(CMDOBJ_RECURRING_STOP, &cmdobj->priv)) {
		adreno_hwsched_retire_cmdobj(hwsched, hwsched->recurring_cmdobj);
		del_timer_sync(&hwsched->lsr_timer);
		hwsched->recurring_cmdobj = NULL;
		if (active)
			adreno_active_count_put(adreno_dev);
		active = false;
		return ret;
	}

	hwsched->recurring_cmdobj = cmdobj;
	/* Star LSR timer for power stats collection */
	mod_timer(&hwsched->lsr_timer, jiffies + msecs_to_jiffies(10));
	return ret;
}

void gen7_trigger_hw_fence_cpu(struct adreno_device *adreno_dev,
	struct adreno_hw_fence_entry *entry)
{
	int ret = msm_hw_fence_update_txq(adreno_dev->hwsched.hw_fence.handle,
			entry->cmd.hash_index, 0, 0);

	if (ret) {
		dev_err_ratelimited(adreno_dev->dev.dev,
			"Failed to trigger hw fence via cpu: ctx:%d ts:%d ret:%d\n",
			entry->drawctxt->base.id, (u32)entry->cmd.ts, ret);
		return;
	}

	msm_hw_fence_trigger_signal(adreno_dev->hwsched.hw_fence.handle, IPCC_CLIENT_GPU,
		IPCC_CLIENT_APSS, 0);
}

/* We don't want to unnecessarily wake the GMU to trigger hardware fences */
static void drain_context_hw_fence_cpu(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt)
{
	struct adreno_hw_fence_entry *entry, *tmp;

	list_for_each_entry_safe(entry, tmp, &drawctxt->hw_fence_list, node) {

		gen7_trigger_hw_fence_cpu(adreno_dev, entry);

		gen7_remove_hw_fence_entry(adreno_dev, entry);
	}
}

int gen7_hwsched_drain_context_hw_fences(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt)
{
	struct adreno_hw_fence_entry *entry, *tmp;
	int ret = 0;

	/* We don't need the drawctxt lock here as this context has already been invalidated */
	list_for_each_entry_safe(entry, tmp, &drawctxt->hw_fence_list, node) {

		/* Any error here is fatal */
		ret = gen7_send_hw_fence_hfi_wait_ack(adreno_dev, entry,
			HW_FENCE_FLAG_SKIP_MEMSTORE);
		if (ret)
			break;

		gen7_remove_hw_fence_entry(adreno_dev, entry);
	}

	return ret;
}

static void trigger_context_unregister_fault(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt)
{
	gmu_core_fault_snapshot(KGSL_DEVICE(adreno_dev));

	/* Make sure we send all fences from this context to the TxQueue after recovery */
	move_detached_context_hardware_fences(adreno_dev, drawctxt);
	gen7_hwsched_fault(adreno_dev, ADRENO_GMU_FAULT);
}

static int send_context_unregister_hfi(struct adreno_device *adreno_dev,
	struct kgsl_context *context, u32 ts)
{
	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
	struct pending_cmd pending_ack;
	struct hfi_unregister_ctxt_cmd cmd;
	u32 seqnum;
	int ret;

	/* Only send HFI if device is not in SLUMBER */
	if (!context->gmu_registered ||
		!test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) {
		drain_context_hw_fence_cpu(adreno_dev, drawctxt);
		return 0;
	}

	ret = CMD_MSG_HDR(cmd, H2F_MSG_UNREGISTER_CONTEXT);
	if (ret)
		return ret;

	cmd.ctxt_id = context->id,
	cmd.ts = ts,

	/*
	 * Although we know device is powered on, we can still enter SLUMBER
	 * because the wait for ack below is done without holding the mutex. So
	 * take an active count before releasing the mutex so as to avoid a
	 * concurrent SLUMBER sequence while GMU is un-registering this context.
	 */
	ret = gen7_hwsched_active_count_get(adreno_dev);
	if (ret) {
		trigger_context_unregister_fault(adreno_dev, drawctxt);
		return ret;
	}

	seqnum = atomic_inc_return(&gmu->hfi.seqnum);
	cmd.hdr = MSG_HDR_SET_SEQNUM_SIZE(cmd.hdr, seqnum, sizeof(cmd) >> 2);
	add_waiter(hfi, cmd.hdr, &pending_ack);

	ret = gen7_hfi_cmdq_write(adreno_dev, (u32 *)&cmd, sizeof(cmd));
	if (ret) {
		trigger_context_unregister_fault(adreno_dev, drawctxt);
		goto done;
	}

	ret = adreno_hwsched_ctxt_unregister_wait_completion(adreno_dev,
		&gmu->pdev->dev, &pending_ack, gen7_hwsched_process_msgq, &cmd);
	if (ret) {
		trigger_context_unregister_fault(adreno_dev, drawctxt);
		goto done;
	}

	ret = check_detached_context_hardware_fences(adreno_dev, drawctxt);
	if (!ret)
		ret = check_ack_failure(adreno_dev, &pending_ack);

done:
	gen7_hwsched_active_count_put(adreno_dev);
	del_waiter(hfi, &pending_ack);

	return ret;
}

void gen7_hwsched_context_detach(struct adreno_context *drawctxt)
{
	struct kgsl_context *context = &drawctxt->base;
	struct kgsl_device *device = context->device;
	struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
	int ret = 0;

	mutex_lock(&device->mutex);

	ret = send_context_unregister_hfi(adreno_dev, context,
		drawctxt->internal_timestamp);

	if (!ret) {
		kgsl_sharedmem_writel(device->memstore,
			KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
			drawctxt->timestamp);

		kgsl_sharedmem_writel(device->memstore,
			KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
			drawctxt->timestamp);

		adreno_profile_process_results(adreno_dev);
	}

	context->gmu_registered = false;

	mutex_unlock(&device->mutex);
}

u32 gen7_hwsched_preempt_count_get(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

	if (device->state != KGSL_STATE_ACTIVE)
		return 0;

	return gen7_hwsched_hfi_get_value(adreno_dev, HFI_VALUE_PREEMPT_COUNT);
}

void gen7_hwsched_context_destroy(struct adreno_device *adreno_dev,
	struct adreno_context *drawctxt)
{
	if (!adreno_hwsched_context_queue_enabled(adreno_dev))
		return;

	if (drawctxt->gmu_context_queue.gmuaddr)
		gen7_free_gmu_block(to_gen7_gmu(adreno_dev), &drawctxt->gmu_context_queue);

	if (drawctxt->gmu_hw_fence_queue.gmuaddr)
		gen7_free_gmu_block(to_gen7_gmu(adreno_dev), &drawctxt->gmu_hw_fence_queue);
}

static int register_global_ctxt(struct adreno_device *adreno_dev)
{
	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	struct hfi_register_ctxt_cmd rcmd = {0};
	struct hfi_context_pointers_cmd pcmd = {0};
	int ret;

	if (hwsched->global_ctxt_gmu_registered)
		return 0;

	if (adreno_hwsched_context_queue_enabled(adreno_dev) && !hwsched->global_ctxtq.hostptr) {
		struct gmu_context_queue_header *hdr;

		ret = gen7_alloc_gmu_kernel_block(to_gen7_gmu(adreno_dev), &hwsched->global_ctxtq,
			SZ_4K, GMU_NONCACHED_KERNEL, IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV);
		if (ret) {
			memset(&hwsched->global_ctxtq, 0x0, sizeof(hwsched->global_ctxtq));
			return ret;
		}

		hdr = hwsched->global_ctxtq.hostptr;
		hdr->start_addr = hwsched->global_ctxtq.gmuaddr + sizeof(*hdr);
		hdr->queue_size = (hwsched->global_ctxtq.size - sizeof(*hdr)) >> 2;
	}

	ret = CMD_MSG_HDR(rcmd, H2F_MSG_REGISTER_CONTEXT);
	if (ret)
		return ret;

	rcmd.ctxt_id = KGSL_GLOBAL_CTXT_ID;
	rcmd.flags = (KGSL_CONTEXT_PRIORITY_HIGH << KGSL_CONTEXT_PRIORITY_SHIFT);

	ret = gen7_hfi_send_cmd_async(adreno_dev, &rcmd, sizeof(rcmd));
	if (ret)
		return ret;

	ret = CMD_MSG_HDR(pcmd, H2F_MSG_CONTEXT_POINTERS);
	if (ret)
		return ret;

	pcmd.ctxt_id = KGSL_GLOBAL_CTXT_ID;
	pcmd.sop_addr = MEMSTORE_ID_GPU_ADDR(device, KGSL_GLOBAL_CTXT_ID, soptimestamp);
	pcmd.eop_addr = MEMSTORE_ID_GPU_ADDR(device, KGSL_GLOBAL_CTXT_ID, eoptimestamp);

	if (adreno_hwsched_context_queue_enabled(adreno_dev))
		pcmd.gmu_context_queue_addr = hwsched->global_ctxtq.gmuaddr;

	ret = gen7_hfi_send_cmd_async(adreno_dev, &pcmd, sizeof(pcmd));
	if (!ret)
		hwsched->global_ctxt_gmu_registered = true;

	return ret;
}

static int submit_global_ctxt_cmd(struct adreno_device *adreno_dev, u64 gpuaddr, u32 size)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
	struct {
		struct hfi_submit_cmd submit_cmd;
		struct hfi_issue_ib issue_ib;
	} cmd = {0};
	u32 seqnum, cmd_size = sizeof(cmd);
	static u32 ts;
	int ret;

	cmd.submit_cmd.ctxt_id = KGSL_GLOBAL_CTXT_ID;
	cmd.submit_cmd.ts = ++ts;
	cmd.submit_cmd.numibs = 1;

	cmd.issue_ib.addr = gpuaddr;
	cmd.issue_ib.size = size;

	seqnum = atomic_inc_return(&hwsched->submission_seqnum);
	cmd.submit_cmd.hdr = CREATE_MSG_HDR(H2F_MSG_ISSUE_CMD, HFI_MSG_CMD);
	cmd.submit_cmd.hdr = MSG_HDR_SET_SEQNUM_SIZE(cmd.submit_cmd.hdr, seqnum, cmd_size >> 2);

	if (adreno_hwsched_context_queue_enabled(adreno_dev))
		ret = gen7_gmu_context_queue_write(adreno_dev,
			  &hwsched->global_ctxtq, (u32 *)&cmd, cmd_size, NULL, NULL);
	else
		ret = gen7_hfi_dispatch_queue_write(adreno_dev, HFI_DSP_ID_0,
			(u32 *)&cmd, cmd_size, NULL, NULL);

	/* Send interrupt to GMU to receive the message */
	if (!ret)
		gmu_core_regwrite(device, GEN7_GMU_HOST2GMU_INTR_SET, DISPQ_IRQ_BIT(0));

	return ret;
}

int gen7_hwsched_counter_inline_enable(struct adreno_device *adreno_dev,
		const struct adreno_perfcount_group *group,
		u32 counter, u32 countable)
{
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	struct adreno_perfcount_register *reg = &group->regs[counter];
	u32 val, *cmds, count = 0;
	int ret;

	ret = register_global_ctxt(adreno_dev);
	if (ret)
		goto err;

	ret = adreno_allocate_global(device, &hfi->perfctr_scratch,
		PAGE_SIZE, 0, KGSL_MEMFLAGS_GPUREADONLY, 0, "perfctr_scratch");
	if (ret)
		goto err;

	if (group->flags & ADRENO_PERFCOUNTER_GROUP_RESTORE)
		gen7_perfcounter_update(adreno_dev, reg, false,
				FIELD_PREP(GENMASK(13, 12), PIPE_NONE), group->flags);

	cmds = hfi->perfctr_scratch->hostptr;

	cmds[count++] = cp_type7_packet(CP_WAIT_FOR_IDLE, 0);
	cmds[count++] = cp_type4_packet(reg->select, 1);
	cmds[count++] = countable;

	ret = submit_global_ctxt_cmd(adreno_dev, hfi->perfctr_scratch->gpuaddr, count << 2);
	if (ret)
		goto err;

	/* Wait till the register is programmed with the countable */
	ret = kgsl_regmap_read_poll_timeout(&device->regmap, reg->select, val,
				val == countable, 100, ADRENO_IDLE_TIMEOUT);
	if (!ret) {
		reg->value = 0;
		return ret;
	}

err:
	dev_err(device->dev, "Perfcounter %s/%u/%u start via commands failed\n",
			group->name, counter, countable);

	return ret;
}

int gen7_hwsched_disable_hw_fence_throttle(struct adreno_device *adreno_dev)
{
	struct gen7_hwsched_hfi *hfi = to_gen7_hwsched_hfi(adreno_dev);
	struct adreno_context *drawctxt = NULL;
	u32 ts = 0;
	int ret = 0;

	if (!test_bit(ADRENO_HWSCHED_HW_FENCE, &adreno_dev->hwsched.flags))
		return 0;

	spin_lock(&hfi->hw_fence.lock);

	drawctxt = hfi->hw_fence.defer_drawctxt;
	ts = hfi->hw_fence.defer_ts;

	spin_unlock(&hfi->hw_fence.lock);

	if (!drawctxt)
		goto done;

	ret = process_hw_fence_deferred_ctxt(adreno_dev, drawctxt, ts);

	kgsl_context_put(&drawctxt->base);
	gen7_hwsched_active_count_put(adreno_dev);

done:
	_disable_hw_fence_throttle(adreno_dev, true);

	return ret;
}